pax_global_header00006660000000000000000000000064130034301010014473gustar00rootroot0000000000000052 comment=7f420cb031847988f51bd69905650a2e031adde6 rosdistro-0.5.0/000077500000000000000000000000001300343010100135255ustar00rootroot00000000000000rosdistro-0.5.0/.gitignore000066400000000000000000000000641300343010100155150ustar00rootroot00000000000000_build build deb_dist dist *.pyc rosdistro.egg-info rosdistro-0.5.0/.travis.yml000066400000000000000000000005641300343010100156430ustar00rootroot00000000000000language: python sudo: false python: - "2.6" - "2.7" - "3.3" # command to install dependencies install: # develop seems to be required by travis since 02/2013 - python setup.py build develop - pip install PyYAML argparse rospkg setuptools - pip install nose coverage # command to run tests script: - nosetests --with-xunit test notifications: email: false rosdistro-0.5.0/LICENSE.txt000066400000000000000000000030001300343010100153410ustar00rootroot00000000000000Software License Agreement (BSD License) Copyright (c) 2012, Willow Garage, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Willow Garage, Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.rosdistro-0.5.0/Makefile000077500000000000000000000010641300343010100151710ustar00rootroot00000000000000.PHONY: all setup clean_dist distro clean install NAME=rosdistro VERSION=`./setup.py --version` UNAME := $(shell uname) all: echo "noop for debbuild" setup: echo "building version ${VERSION}" clean_dist: -rm -f MANIFEST -rm -rf dist -rm -rf deb_dist distro: setup clean_dist python setup.py sdist clean: clean_dist echo "clean" install: distro sudo checkinstall python setup.py install testsetup: echo "running rosdistro! tests" test: testsetup python setup.py nosetests test--pdb-failures: testsetup python setup.py nosetests --pdb-failures rosdistro-0.5.0/README.md000066400000000000000000000001021300343010100147750ustar00rootroot00000000000000rosdistro ========= Tools to work with catkinized rosdistro filesrosdistro-0.5.0/README.rst000066400000000000000000000007331300343010100152170ustar00rootroot00000000000000rosdistro ===== Code & tickets -------------- +----------------------+-----------------------------------------------------------+ | rosdistro | http://github.com/ros-infrastructure/rosdistro | +----------------------+-----------------------------------------------------------+ | Issues | http://github.com/ros-infrastructure/rosdistro/issues | +----------------------+-----------------------------------------------------------+ rosdistro-0.5.0/doc/000077500000000000000000000000001300343010100142725ustar00rootroot00000000000000rosdistro-0.5.0/doc/Makefile000066400000000000000000000005001300343010100157250ustar00rootroot00000000000000.PHONY: html apidoc clean upload SPHINXAPIDOC=sphinx-apidoc html: apidoc $(MAKE) -C _build html apidoc: $(SPHINXAPIDOC) -F -o _build ../src/rosdistro sed -i "s/_build/./g" _build/Makefile clean: -rm -rf _build upload: html scp -r _build/html/ rosbot@ros.osuosl.org:/home/rosbot/docs/independent/api/rosdistro/ rosdistro-0.5.0/scripts/000077500000000000000000000000001300343010100152145ustar00rootroot00000000000000rosdistro-0.5.0/scripts/rosdistro000077500000000000000000000041511300343010100171730ustar00rootroot00000000000000#!/usr/bin/env python import rosdistro import optparse import sys def depends(args): distro = rosdistro.RosDistro(args[1]) deps = distro.get_depends(args[2:], int(args[0])) for t, dep in deps.iteritems(): print('%s depends:'%t) print(', '.join(dep)) def depends_on(args): distro = rosdistro.RosDistro(args[1]) deps = distro.get_depends_on(args[2:], int(args[0])) for t, dep in deps.iteritems(): print('%s depends_on:'%t) print(', '.join(dep)) def help(cmd=None): print("Usage: %s [command]"%sys.argv[0]) print(" With [command] one of the following") if cmd and type(cmd) == list: cmd = cmd[0] if cmd and not cmds.has_key(cmd): print("Unknown command %s"%cmd) cmd = None if cmd and cmd == 'help': cmd = None for c, ops in cmds.iteritems(): res = " '%s'"%c res += (' ' * (30-len(res))) res += "with arguments: '%s'"%(' '.join(ops['args'])) if not cmd or cmd == c: print(res) cmds = {'help': {'args': ['command'], 'cmd': help}, 'depends': {'args': ['depth', 'ros_version', 'package_list'], 'cmd': depends}, 'depends_on': {'args': ['depth', 'ros_version', 'package_list'], 'cmd': depends_on}} def main(): parser = optparse.OptionParser() parser.add_option("--cache", action="store", default=None) (options, commands) = parser.parse_args() # usage if len(commands) == 0: help() return # command specific usage cmd = commands[0] if cmd in cmds: args = cmds[cmd]['args'] if len(commands)-1 < len(args): help(cmd) return else: help(cmd) return # execute command try: cmds[cmd]['cmd'](commands[1:]) except Exception as e: print(e) print("Failure") return distro = rosdistro.RosDistro(args(0), options.cache) for r in distro.get_repositories(): print("Caching %s"%r) distro.get_depends1(r) print("Cache written to %s"%(distro.depends_file.local_url)) if __name__ == "__main__": main() rosdistro-0.5.0/scripts/rosdistro_build_cache000077500000000000000000000101141300343010100214710ustar00rootroot00000000000000#!/usr/bin/env python # Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function import argparse import gzip import logging import sys import yaml from rosdistro import logger from rosdistro.distribution_cache_generator import CacheYamlDumper, generate_distribution_caches logging.basicConfig(level=logging.INFO) def parse_args(args=sys.argv[1:]): parser = argparse.ArgumentParser( description='Build the cache for rosdistro distributions.' ) add = parser.add_argument add('index', help='The path or url of the index.yaml file (must be either a local file or a file:// url)' ) add('dist_names', nargs='*', help='The names of the distributions (default: all)' ) add('--debug', action='store_true', default=False, help='Output debug messages' ) add('--preclean', action='store_true', default=False, help='Build the cache from scratch instead of reusing cached data' ) add('--ignore-local', action='store_true', default=False, help='Ignore locally available cache' ) add('--include-source', action='store_true', default=False, help='Also include source branch package XMLs in the cache' ) return parser.parse_args(args) def main(): args = parse_args() if args.debug: logger.level = logging.DEBUG try: caches = generate_distribution_caches(args.index, dist_names=args.dist_names, preclean=args.preclean, ignore_local=args.ignore_local, include_source=args.include_source, debug=args.debug) except RuntimeError as e: print(str(e), file=sys.stderr) sys.exit(1) for dist_name, cache in caches.items(): if args.dist_names and dist_name not in args.dist_names: continue # write the cache data = yaml.dump(cache.get_data(), Dumper=CacheYamlDumper) with open('%s-cache.yaml' % dist_name, 'w') as f: print('- write cache file "%s-cache.yaml"' % dist_name) f.write(data) with gzip.open('%s-cache.yaml.gz' % dist_name, 'wb') as f: # On Python 3, we must encode the unicode yaml str prior to gzipping it, # whereas for Python 2, the yaml output is a str which cannot be further # encoded (compared with a unicode object). if sys.version_info[0] >= 3: data = data.encode('utf-8') print('- write compressed cache file "%s-cache.yaml.gz"' % dist_name) f.write(data) if __name__ == '__main__': main() rosdistro-0.5.0/scripts/rosdistro_convert000077500000000000000000000246311300343010100207400ustar00rootroot00000000000000#!/usr/bin/env python # Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function import os import shutil import subprocess import sys import tempfile try: from urllib.error import HTTPError except ImportError: from urllib2 import HTTPError from rosdistro.loader import load_url from rosdistro.release_file import ReleaseFile BASE_SRC_URL = 'https://raw.github.com/ros/rosdistro/master' def get_targets(): url = BASE_SRC_URL + '/releases/targets.yaml' print('Load "%s"' % url) yaml_str = load_url(url) data = yaml.load(yaml_str) targets = {} for d in data: targets[d.keys()[0]] = d.values()[0] return targets def convert_release(dist_name, targets): url = BASE_SRC_URL + '/releases/%s.yaml' % dist_name print('Load "%s"' % url) yaml_str = load_url(url) input_ = yaml.load(yaml_str) # improve conversion performance by reusing results from last run last_dist = None if os.path.exists(dist_name + '/release.yaml'): with open(dist_name + '/release.yaml', 'r') as f: last_data = yaml.load(f.read()) last_dist = ReleaseFile(dist_name, last_data) output = {} output['type'] = 'release' output['version'] = 1 output['platforms'] = {'ubuntu': targets[dist_name]} output['repositories'] = {} for repo_name in sorted(input_['repositories']): input_repo = input_['repositories'][repo_name] output_repo = {} output_repo['url'] = input_repo['url'] output_repo['version'] = input_repo['version'] if output_repo['version'] is None: print('- "' + repo_name + '" has no version') del output_repo['version'] pkg_name = repo_name if 'packages' in input_repo: output_repo['packages'] = {} unary_repo = len(input_repo['packages']) == 1 for pkg_name in input_repo['packages']: if unary_repo: if input_repo['packages'][pkg_name] is None: if pkg_name == repo_name: del output_repo['packages'] else: output_repo['packages'][pkg_name] = {'subfolder': pkg_name} else: output_repo['packages'][pkg_name] = input_repo['packages'][pkg_name] break output_repo['packages'][pkg_name] = None if input_repo['packages'][pkg_name] is not None and input_repo['packages'][pkg_name] != pkg_name: output_repo['packages'][pkg_name] = {'subfolder': input_repo['packages'][pkg_name]} output['repositories'][repo_name] = output_repo if 'version' in output_repo and output_repo['version'] is not None: tag_template = _get_tag_template(dist_name, output_repo, pkg_name, last_dist.repositories[repo_name] if last_dist and repo_name in last_dist.repositories else None) output_repo['tags'] = {'release': tag_template} yaml_str = yaml.dump(output, default_flow_style=False) yaml_str = yaml_str.replace(': null', ':') with open(dist_name + '/release.yaml', 'w') as f: f.write('%YAML 1.1\n') f.write('# ROS release file\n') f.write('# see REP 137: http://ros.org/reps/rep-0137.html\n') f.write('---\n') f.write(yaml_str) def _get_tag_template(dist_name, repo, pkg_name, last_repo=None): # reuse tag template if fetched before for the same repo and version if last_repo and last_repo.version == repo['version']: if 'release' in last_repo.tags: return last_repo.tags['release'] assert 'github.com' in repo['url'] release_tag = 'release/{0}/{1}/{2}'.format(dist_name, pkg_name, repo['version']) url = _github_raw_url(repo['url'], release_tag) print('- ' + pkg_name + ': ' + url) try: try: load_url(url) return 'release/%s/{package}/{version}' % dist_name except HTTPError: # try alternative tag upstream_version = repo['version'].split('-')[0] release_tag = 'release/{0}/{1}'.format(pkg_name, upstream_version) url = _github_raw_url(repo['url'], release_tag) print('- ' + pkg_name + ': ' + url) load_url(url) return 'release/{package}/{upstream_version}' except HTTPError as e: raise RuntimeError('Could not determine tag using %s, %s, %s: %s' % (dist_name, repo, pkg_name, e)) def _github_raw_url(url, tag): url = url.replace('.git', '/tree/%s/' % tag) url = url.replace('git://', 'https://') #url = url.replace('https://', 'https://raw.') return url def convert_source(dist_name): url = BASE_SRC_URL + '/releases/%s-devel.yaml' % dist_name print('Load "%s"' % url) yaml_str = load_url(url) input_ = yaml.load(yaml_str) output = {} output['type'] = 'source' output['version'] = 1 output['repositories'] = {} for repo_name in sorted(input_['repositories']): input_repo = input_['repositories'][repo_name] output_repo = {} output_repo['type'] = input_repo['type'] output_repo['url'] = input_repo['url'] output_repo['version'] = input_repo['version'] if output_repo['type'] == 'svn' and output_repo['version'] is None: output_repo['version'] = 'HEAD' if output_repo['version'] is None: print('- "' + repo_name + '" has no version') del output_repo['version'] output['repositories'][repo_name] = output_repo yaml_str = yaml.dump(output, default_flow_style=False) yaml_str = yaml_str.replace(': null', ':') with open(dist_name + '/source.yaml', 'w') as f: f.write('%YAML 1.1\n') f.write('# ROS source file\n') f.write('# see REP 137: http://ros.org/reps/rep-0137.html\n') f.write('---\n') f.write(yaml_str) def convert_doc(dist_name): base = tempfile.mktemp() rc = subprocess.call(['git', 'clone', '--depth', '0', 'git://github.com/ros/rosdistro.git', base]) if rc: print('Failed to checkout rosdistro repo', file=sys.stderr) shutil.rmtree(base) return doc_base = os.path.join(base, 'doc', dist_name) rosinstalls = {} rosinstall_depends = {} for filename in os.listdir(doc_base): if filename.endswith('.rosinstall'): name = os.path.splitext(os.path.basename(filename))[0] with open(os.path.join(doc_base, filename)) as f: data = yaml.load(f) if name.endswith('_depends'): rosinstall_depends[name] = data else: rosinstalls[name] = data shutil.rmtree(base) # currently no doc has depends in groovy and hydro #assert not rosinstall_depends output = {} output['type'] = 'doc' output['version'] = 1 output['repositories'] = {} for repo_name in sorted(rosinstalls.keys()): for input_repo in rosinstalls[repo_name]: output_repo = {} output_repo['type'] = input_repo.keys()[0] input_data = input_repo[output_repo['type']] output_repo['url'] = input_data['uri'] if 'version' in input_data: output_repo['version'] = input_data['version'] if output_repo['type'] == 'svn' and ('version' not in output_repo or output_repo['version'] is None): output_repo['version'] = 'HEAD' if 'version' not in output_repo or output_repo['version'] is None: print('- "' + repo_name + '" has no version') if 'version' in output_repo: del output_repo['version'] local_name = input_data['local-name'] if local_name in output['repositories']: if output_repo == output['repositories'][local_name]: print('- skip duplicate "%s"' % local_name) continue key = local_name i = 1 while key in output['repositories']: i += 1 key = local_name + '_%d' % i print('- duplicate: %s %s -> %s' % (repo_name, local_name, key)) if i > 1: # ignore duplicates print(' ignoring duplicate') continue output['repositories'][key] = output_repo yaml_str = yaml.dump(output, default_flow_style=False) yaml_str = yaml_str.replace(': null', ':') with open(dist_name + '/doc.yaml', 'w') as f: f.write('%YAML 1.1\n') f.write('# ROS doc file\n') f.write('# see REP 137: http://ros.org/reps/rep-0137.html\n') f.write('---\n') f.write(yaml_str) if __name__ == '__main__': targets = get_targets() for distro in ['groovy', 'hydro']: print('\nConverting "%s"\n' % distro) convert_release(distro, targets) convert_source(distro) convert_doc(distro) rosdistro-0.5.0/scripts/rosdistro_freeze_source000077500000000000000000000101371300343010100221140ustar00rootroot00000000000000#!/usr/bin/env python # Software License Agreement (BSD License) # # Copyright (c) 2016, Clearpath Robotics # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function import argparse import os.path import sys import yaml from rosdistro import get_distribution from rosdistro.index import Index from rosdistro.freeze_source import freeze_distribution_sources, CONCURRENT_DEFAULT from rosdistro.writer import yaml_from_distribution_file def parse_args(args=sys.argv[1:]): parser = argparse.ArgumentParser( description=''' Freeze a rosdistro\'s source branch versions to hashes or tags. If neither --release-version nor --release-tag are specified, the hashes of the current devel branches are used. ''') parser.add_argument('index', type=argparse.FileType(), help='Path to a local index.yaml file.') parser.add_argument('dist_names', nargs='*', help='The names of the distributions (default: all)') parser.add_argument('-j', '--jobs', type=int, default=CONCURRENT_DEFAULT, help='How many worker threads to use.') parser.add_argument('-q', '--quiet', action='store_true', help='Suppress updating status bar (for script/CI usage).') mode = parser.add_mutually_exclusive_group() mode.add_argument('--release-version', action='store_true', help='Freeze to the hash of current release tag.') mode.add_argument('--release-tag', action='store_true', help='Freeze to name of current release tag.') return parser.parse_args(args) def main(): args = parse_args() index = Index(yaml.safe_load(args.index), 'file://%s' % os.path.dirname(os.path.abspath(args.index.name))) if not args.dist_names: args.dist_names = sorted(index.distributions.keys()) for dist_name in args.dist_names: dist_url = index.distributions[dist_name]['distribution'] if isinstance(dist_url, list): if len(dist_url) != 1: print('This script only works for distributions with a single distribution.yaml, ' 'skipping distribution "%s"' % dist_name, file=sys.stderr) continue dist_url = dist_url[0] dist = get_distribution(index, dist_name) freeze_distribution_sources(dist, release_version=args.release_version, release_tag=args.release_tag, concurrent_ops=args.jobs, quiet=args.quiet) with open(dist_url.split('://')[1], 'w') as f: f.write(yaml_from_distribution_file(dist)) if __name__ == '__main__': main() rosdistro-0.5.0/scripts/rosdistro_generate_cache000077500000000000000000000014151300343010100221700ustar00rootroot00000000000000#!/usr/bin/env python import rosdistro import optparse import sys def gen_cache(options, args): # touch everything to create the new cache distro = rosdistro.RosDistro(args[0], options.cache) for r in distro.get_repositories(): print("Caching %s"%r) distro.get_depends1(r) print("Cache written to %s"%(distro.depends_file.local_url)) def main(): parser = optparse.OptionParser() parser.add_option("--cache", action="store", default=None) (options, args) = parser.parse_args() if len(args) != 1: print("Usage: %s ros_distro"%sys.argv[0]) return try: gen_cache(options, args) except Exception as e: print(e) print("Failure") return if __name__ == "__main__": main() rosdistro-0.5.0/scripts/rosdistro_migrate_to_rep_141000077500000000000000000000134741300343010100226500ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import print_function import argparse import gzip import os import yaml try: from cStringIO import StringIO except ImportError: from io import BytesIO as StringIO from rosdistro.loader import load_url from rosdistro.verify import _to_yaml, _yaml_header_lines def migrate(index_yaml): data = yaml.load(open(index_yaml, 'r')) assert data['type'] == 'index' assert data['version'] == 1 data['version'] = 2 for distro_name in data['distributions']: distro_data = data['distributions'][distro_name] doc_file = distro_data['doc'] release_file = distro_data['release'] source_file = distro_data['source'] del distro_data['doc'] del distro_data['release'] del distro_data['source'] distribution_file = '%s/distribution.yaml' % distro_name distro_data['distribution'] = distribution_file generate_repos_file(index_yaml, distribution_file, doc_file, release_file, source_file) cache_url = distro_data['release_cache'] del distro_data['release_cache'] rel_cache_file = os.path.join(distro_name, '%s-cache.yaml' % distro_name) distro_data['distribution_cache'] = rel_cache_file + '.gz' base = os.path.dirname(index_yaml) cache_file = os.path.join(base, rel_cache_file) update_cache(index_yaml, distro_name, cache_url, cache_file, distribution_file) data = index_to_yaml(data) data = '\n'.join(_yaml_header_lines('index', data['version'])) + '\n' + data with open(index_yaml + '.new', 'w') as f: f.write(data) def index_to_yaml(data): yaml_str = yaml.dump(data, default_flow_style=None) return yaml_str def generate_repos_file(index_yaml, repos_file, doc_file, release_file, source_file): base = os.path.dirname(index_yaml) repos_url = os.path.join(base, repos_file) doc_url = os.path.join(base, doc_file) release_url = os.path.join(base, release_file) source_url = os.path.join(base, source_file) generate_repos_url(repos_url, doc_url, release_url, source_url) def generate_repos_url(repos_url, doc_url, release_url, source_url): data = {} data['type'] = 'distribution' data['version'] = 1 data['repositories'] = {} # migrate release stuff release_data = yaml.load(open(release_url, 'r')) assert release_data['type'] == 'release' assert release_data['version'] == 1 data['release_platforms'] = release_data['platforms'] for repo_name in release_data['repositories']: repo_data = {} release_repo_data = release_data['repositories'][repo_name] repo_data['release'] = get_dict_parts(release_repo_data, ['tags', 'url', 'version']) if 'packages' in release_repo_data: pkgs = release_repo_data['packages'] pkg_names = pkgs.keys() if len(pkg_names) > 1 or (len(pkg_names) == 1 and pkg_names[0] != repo_name): repo_data['release']['packages'] = sorted(pkg_names) for pkg_name in pkgs: if pkgs[pkg_name] is not None: assert not 'status' in pkgs[pkg_name] assert not 'status_description' in pkgs[pkg_name] repo_data.update(get_dict_parts(release_repo_data, ['status', 'status_description'])) data['repositories'][repo_name] = repo_data # migrate doc stuff doc_data = yaml.load(open(doc_url, 'r')) assert doc_data['type'] == 'doc' assert doc_data['version'] == 1 for repo_name in doc_data['repositories']: doc_repo_data = doc_data['repositories'][repo_name] if not repo_name in data['repositories']: data['repositories'][repo_name] = {} data['repositories'][repo_name]['doc'] = get_dict_parts(doc_repo_data, ['type', 'url', 'version']) # migrate source stuff source_data = yaml.load(open(source_url, 'r')) assert source_data['type'] == 'source' assert source_data['version'] == 1 for repo_name in source_data['repositories']: source_repo_data = source_data['repositories'][repo_name] if not repo_name in data['repositories']: data['repositories'][repo_name] = {} data['repositories'][repo_name]['source'] = get_dict_parts(source_repo_data, ['type', 'url', 'version']) data = _to_yaml(data) data = '\n'.join(_yaml_header_lines('distribution', data['version'])) + '\n' + data with open(repos_url, 'w') as f: f.write(data) def update_cache(index_yaml, distro_name, cache_url, cache_file, distribution_file): base = os.path.dirname(index_yaml) assert cache_url.endswith('.gz') yaml_gz_str = load_url(cache_url, skip_decode=True) yaml_gz_stream = StringIO(yaml_gz_str) f = gzip.GzipFile(fileobj=yaml_gz_stream, mode='rb') yaml_str = f.read() if not isinstance(yaml_str, str): yaml_str = yaml_str.decode('utf-8') f.close() cache_data = yaml.load(yaml_str) del cache_data['release_file'] distribution_data = yaml.load(open(os.path.join(base, distribution_file), 'r')) cache_data['distribution_file'] = distribution_data cache_data['release_package_xmls'] = cache_data['package_xmls'] del cache_data['package_xmls'] cache_data['version'] = 2 data = _to_yaml(cache_data) data = '\n'.join(_yaml_header_lines('cache'), cache_data['version']) + '\n' + data with open(cache_file, 'w') as f: f.write(data) with gzip.open(cache_file + '.gz', 'wb') as f: f.write(data) def get_dict_parts(d, keys): data = {} for key in keys: if key in d: data[key] = d[key] return data if __name__ == "__main__": parser = argparse.ArgumentParser(description='Migrate the distros from REP 137 to REP 141.') parser.add_argument('index', help='The index.yaml file to migrate') args = parser.parse_args() migrate(args.index) rosdistro-0.5.0/scripts/rosdistro_migrate_to_rep_143000077500000000000000000000022211300343010100226360ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import print_function import argparse import yaml from rosdistro.verify import _yaml_header_lines def migrate(index_yaml): data = yaml.load(open(index_yaml, 'r')) assert data['type'] == 'index' assert data['version'] == 2 data['version'] = 3 for distro_name in data['distributions']: distro_data = data['distributions'][distro_name] distro_data['distribution'] = [distro_data['distribution']] for key in ['doc_builds', 'release_builds', 'source_builds']: if key in distro_data: del distro_data[key] yaml_str = index_to_yaml(data) yaml_str = '\n'.join(_yaml_header_lines('index', data['version'])) + '\n' + yaml_str with open(index_yaml + '.new', 'w') as f: f.write(yaml_str) def index_to_yaml(data): yaml_str = yaml.dump(data, default_flow_style=None) return yaml_str if __name__ == "__main__": parser = argparse.ArgumentParser(description='Migrate the distros from REP 141 to REP 143.') parser.add_argument('index', help='The index.yaml file to migrate') args = parser.parse_args() migrate(args.index) rosdistro-0.5.0/scripts/rosdistro_reformat000077500000000000000000000051111300343010100210670ustar00rootroot00000000000000#!/usr/bin/env python # Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. import argparse import os import sys from rosdistro.verify import reformat_files from rosdistro.verify import verify_files_identical def parse_args(args=sys.argv[1:]): parser = argparse.ArgumentParser( description='Reformat the rosdistro files by reading and overwriting each file referenced from the index.' ) add = parser.add_argument add('index', help='The path or url of the index.yaml file (must be either a local file or a file:// url)' ) add('-n', '--dry-run', action='store_true', help="Don't actually overwrite any files, just show what would be done." ) return parser.parse_args(args) if __name__ == '__main__': args = parse_args() index = args.index if os.path.isfile(index): index = 'file://' + os.path.abspath(index) if args.dry_run: success = verify_files_identical(index) else: success = reformat_files(index) sys.exit(0 if success else 1) rosdistro-0.5.0/setup.cfg000066400000000000000000000002541300343010100153470ustar00rootroot00000000000000[build_sphinx] source-dir = doc/ build-dir = doc/build all_files = 1 [upload_sphinx] upload-dir = doc/build/html [nosetests] with-coverage=true cover-package=rosdistro rosdistro-0.5.0/setup.py000077500000000000000000000027111300343010100152430ustar00rootroot00000000000000#!/usr/bin/env python import os from setuptools import setup, find_packages import sys exec(open(os.path.join(os.path.dirname(__file__), 'src', 'rosdistro', '_version.py')).read()) install_requires = ['catkin_pkg', 'rospkg', 'PyYAML', 'setuptools'] if sys.version_info[0] == 2 and sys.version_info[1] < 7: install_requires.append('argparse') setup( name='rosdistro', version=__version__, install_requires=install_requires, packages=find_packages('src'), package_dir={'': 'src'}, scripts=[ # 'scripts/rosdistro', 'scripts/rosdistro_build_cache', 'scripts/rosdistro_freeze_source', # 'scripts/rosdistro_convert', # 'scripts/rosdistro_generate_cache', 'scripts/rosdistro_migrate_to_rep_141', 'scripts/rosdistro_migrate_to_rep_143', 'scripts/rosdistro_reformat' ], author='Wim Meeussen, Dirk Thomas', author_email='wim@hidof.com, dthomas@osrfoundation.org', maintainer='Dirk Thomas', maintainer_email='dthomas@osrfoundation.org', url='http://wiki.ros.org/rosdistro', download_url='http://download.ros.org/downloads/rosdistro/', keywords=['ROS'], classifiers=['Programming Language :: Python', 'License :: OSI Approved :: BSD License', 'License :: OSI Approved :: MIT License'], description="A tool to work with rosdistro files", long_description="""A tool to work with rosdistro files""", license='BSD, MIT' ) rosdistro-0.5.0/src/000077500000000000000000000000001300343010100143145ustar00rootroot00000000000000rosdistro-0.5.0/src/rosdistro/000077500000000000000000000000001300343010100163445ustar00rootroot00000000000000rosdistro-0.5.0/src/rosdistro/__init__.py000066400000000000000000000331571300343010100204660ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function # legacy imports from . import common from .rosdistro import walks from .rosdistro import RosDistro from .develdistro import DevelDistro from .aptdistro import AptDistro import gzip import logging import os try: from cStringIO import StringIO except ImportError: from io import BytesIO as StringIO import sys try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse import yaml logger = logging.getLogger('rosdistro') from ._version import __version__ from .distribution import Distribution from .distribution_cache import DistributionCache from .distribution_file import create_distribution_file from .distribution_file import DistributionFile from .doc_build_file import DocBuildFile from .doc_file import DocFile from .external.appdirs import user_config_dir, site_config_dir from .index import Index from .loader import load_url from .manifest_provider.cache import CachedManifestProvider, CachedSourceManifestProvider from .release import Release from .release_build import ReleaseBuild from .release_build_file import ReleaseBuildFile from .release_cache import ReleaseCache from .release_file import ReleaseFile from .source_build_file import SourceBuildFile from .source_file import SourceFile ### index information DEFAULT_INDEX_URL = 'https://raw.githubusercontent.com/ros/rosdistro/master/index.yaml' def get_index_url(): # environment variable has precedence over configuration files if 'ROSDISTRO_INDEX_URL' in os.environ: return os.environ['ROSDISTRO_INDEX_URL'] def read_cfg_index_url(fname): try: with open(fname) as f: return yaml.load(f.read())['index_url'] except (IOError, KeyError, yaml.YAMLError): return None cfg_file = 'config.yaml' # first, look for the user configuration (usually ~/.config/rosdistro) user_cfg_path = os.path.join(user_config_dir('rosdistro'), cfg_file) index_url = read_cfg_index_url(user_cfg_path) if index_url is not None: return index_url # if not found, look for the global configuration *usually /etc/xdg/rosdistro) site_cfg_paths = os.path.join(site_config_dir('rosdistro', multipath=True), cfg_file).split(os.pathsep) for site_cfg_path in site_cfg_paths: index_url = read_cfg_index_url(site_cfg_path) if index_url is not None: return index_url # if nothing is found, use the default return DEFAULT_INDEX_URL def get_index(url): logger.debug('Load index from "%s"' % url) yaml_str = load_url(url) data = yaml.load(yaml_str) base_url = os.path.dirname(url) url_parts = urlparse(url) return Index(data, base_url, url_query=url_parts.query) ### distribution information def get_distribution(index, dist_name): dist_file = get_distribution_file(index, dist_name) return Distribution(dist_file) def get_distribution_file(index, dist_name): data = _get_dist_file_data(index, dist_name, 'distribution') return create_distribution_file(dist_name, data) def get_distribution_files(index, dist_name): data = _get_dist_file_data(index, dist_name, 'distribution') if not isinstance(data, list): data = [data] dist_files = [] for d in data: dist_file = DistributionFile(dist_name, d) dist_files.append(dist_file) return dist_files def get_cached_distribution(index, dist_name, cache=None, allow_lazy_load=False): if cache is None: try: cache = get_distribution_cache(index, dist_name) except Exception: if not allow_lazy_load: raise # create empty cache instance dist_file_data = _get_dist_file_data(index, dist_name, 'distribution') cache = DistributionCache(dist_name, distribution_file_data=dist_file_data) dist = Distribution( cache.distribution_file, [CachedManifestProvider(cache, Distribution.default_manifest_providers if allow_lazy_load else None)], [CachedSourceManifestProvider(cache, Distribution.default_source_manifest_providers if allow_lazy_load else None)]) assert cache.distribution_file.name == dist_name return dist def get_distribution_cache(index, dist_name): if dist_name not in index.distributions.keys(): raise RuntimeError("Unknown distribution: '{0}'. Valid distribution names are: {1}".format(dist_name, ', '.join(sorted(index.distributions.keys())))) dist = index.distributions[dist_name] if 'distribution_cache' not in dist.keys(): raise RuntimeError("Distribution has no cache: '{0}'".format(dist_name)) url = dist['distribution_cache'] logger.debug('Load cache from "%s"' % url) if url.endswith('.yaml'): yaml_str = load_url(url) elif url.endswith('.yaml.gz'): yaml_gz_str = load_url(url, skip_decode=True) yaml_gz_stream = StringIO(yaml_gz_str) f = gzip.GzipFile(fileobj=yaml_gz_stream, mode='rb') yaml_str = f.read() if not isinstance(yaml_str, str): yaml_str = yaml_str.decode('utf-8') f.close() else: raise NotImplementedError('The url of the cache must end with either ".yaml" or ".yaml.gz"') data = yaml.load(yaml_str) return DistributionCache(dist_name, data) ### release information def get_cached_release(index, dist_name, cache=None, allow_lazy_load=False): print('# rosdistro.get_cached_release() has been deprecated in favor of the new function rosdistro.get_cached_distribution() - please check that you have the latest versions of the Python tools (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-bloom python-rosdep python-rosinstall python-rosinstall-generator)', file=sys.stderr) if cache is None: try: cache = get_release_cache(index, dist_name) except Exception: if not allow_lazy_load: raise # create empty cache instance rel_file_data = _get_dist_file_data(index, dist_name, 'release') cache = ReleaseCache(dist_name, rel_file_data=rel_file_data) rel = Release( cache.release_file, [CachedManifestProvider(cache, Release.default_manifest_providers if allow_lazy_load else None)]) assert cache.release_file.name == dist_name return rel def get_release(index, dist_name): print('# rosdistro.get_release() has been deprecated in favor of the new function rosdistro.get_cached_distribution() - please check that you have the latest versions of the Python tools (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-bloom python-rosdep python-rosinstall python-rosinstall-generator)', file=sys.stderr) rel_file = get_release_file(index, dist_name) return Release(rel_file) def get_release_file(index, dist_name): print('# rosdistro.get_release_file() has been deprecated in favor of the new function rosdistro.get_distribution_file() - please check that you have the latest versions of the Python tools (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-bloom python-rosdep python-rosinstall python-rosinstall-generator)', file=sys.stderr) data = _get_dist_file_data(index, dist_name, 'distribution') return ReleaseFile(dist_name, data) def get_release_cache(index, dist_name): print('# rosdistro.get_release_cache() has been deprecated in favor of the new function rosdistro.get_distribution_cache() - please check that you have the latest versions of the Python tools (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-bloom python-rosdep python-rosinstall python-rosinstall-generator)', file=sys.stderr) if dist_name not in index.distributions.keys(): raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(sorted(index.distributions.keys())))) dist = index.distributions[dist_name] if 'distribution_cache' not in dist.keys(): raise RuntimeError("Release has no cache: '{0}'".format(dist_name)) url = dist['distribution_cache'] logger.debug('Load cache from "%s"' % url) if url.endswith('.yaml'): yaml_str = load_url(url) elif url.endswith('.yaml.gz'): yaml_gz_str = load_url(url, skip_decode=True) yaml_gz_stream = StringIO(yaml_gz_str) f = gzip.GzipFile(fileobj=yaml_gz_stream, mode='rb') yaml_str = f.read() if not isinstance(yaml_str, str): yaml_str = yaml_str.decode('utf-8') f.close() else: raise NotImplementedError('The url of the cache must end with either ".yaml" or ".yaml.gz"') data = yaml.load(yaml_str) return ReleaseCache(dist_name, data) def get_release_builds(index, release_file): print("# rosdistro.get_release_builds() has been deprecated and its functionality is now handled by the 'ros_buildfarm.config' module", file=sys.stderr) build_files = get_release_build_files(index, release_file.name) builds = [] for build_file in build_files: builds.append(ReleaseBuild(release_file, build_file)) return builds def get_release_build_files(index, dist_name): print("# rosdistro.get_release_build_files() has been deprecated and its functionality is now handled by the 'ros_buildfarm.config' module", file=sys.stderr) data = _get_dist_file_data(index, dist_name, 'release_builds') build_files = [] for d in data: build_files.append(ReleaseBuildFile(dist_name, d)) return build_files ### source information def get_source_file(index, dist_name): print('# rosdistro.get_source_file() has been deprecated in favor of the new function rosdistro.get_distribution_file() - please check that you have the latest versions of the Python tools (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-bloom python-rosdep python-rosinstall python-rosinstall-generator)', file=sys.stderr) data = _get_dist_file_data(index, dist_name, 'distribution') return SourceFile(dist_name, data) def get_source_build_files(index, dist_name): print("# rosdistro.get_source_build_files() has been deprecated and its functionality is now handled by the 'ros_buildfarm.config' module", file=sys.stderr) data = _get_dist_file_data(index, dist_name, 'source_builds') build_files = [] for d in data: build_files.append(SourceBuildFile(dist_name, d)) return build_files ### doc information def get_doc_file(index, dist_name): print('# rosdistro.get_doc_file() has been deprecated in favor of the new function rosdistro.get_distribution_file() - please check that you have the latest versions of the Python tools (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-bloom python-rosdep python-rosinstall python-rosinstall-generator)', file=sys.stderr) data = _get_dist_file_data(index, dist_name, 'distribution') return DocFile(dist_name, data) def get_doc_build_files(index, dist_name): print("# rosdistro.get_doc_build_files() has been deprecated and its functionality is now handled by the 'ros_buildfarm.config' module", file=sys.stderr) data = _get_dist_file_data(index, dist_name, 'doc_builds') build_files = [] for d in data: build_files.append(DocBuildFile(dist_name, d)) return build_files ### internal def _get_dist_file_data(index, dist_name, type_): if dist_name not in index.distributions.keys(): raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(sorted(index.distributions.keys())))) dist = index.distributions[dist_name] if type_ not in dist.keys(): raise RuntimeError('unknown release type "%s"' % type_) url = dist[type_] def _load_yaml_data(url): logger.debug('Load file from "%s"' % url) yaml_str = load_url(url) return yaml.load(yaml_str) if not isinstance(url, list): data = _load_yaml_data(url) else: data = [] for u in url: data.append(_load_yaml_data(u)) return data rosdistro-0.5.0/src/rosdistro/_version.py000066400000000000000000000000261300343010100205400ustar00rootroot00000000000000__version__ = '0.5.0' rosdistro-0.5.0/src/rosdistro/aptdistro.py000066400000000000000000000034511300343010100207320ustar00rootroot00000000000000try: from urllib2 import urlopen except ImportError: from urllib.request import urlopen class AptDistro: def __init__(self, ubuntudistro, arch, shadow=True): if shadow: url = 'http://packages.ros.org/ros-shadow-fixed/ubuntu/dists/{0}/main/binary-{1}/Packages' url = urlopen(url.format(ubuntudistro, arch)) else: url = 'http://packages.ros.org/ros/ubuntu/dists/{0}/main/binary-{1}/Packages' url = urlopen(url.format(ubuntudistro, arch)) self.dep = {} package = None for l in url.read().split('\n'): if l.startswith('Package: '): package = l[len('Package: '):] if l.startswith('Depends: '): if not package: raise RuntimeError("Found 'Depends: ' but not 'Package: ' while parsing the apt repository index file") self.dep[package] = [d.split(' ')[0] for d in (l[len('Depends: '):].split(', '))] package = None def has_package(self, package): return package in self.dep def depends1(self, package): return self.depends(package, one=True) def depends(self, package, res=[], one=False): if package in self.dep: for d in self.dep[package]: if not d in res: res.append(d) if not one: self.depends(d, res, one) return res def depends_on1(self, package): return self.depends_on(package, one=True) def depends_on(self, package, res=[], one=False): for p, dep in self.dep.iteritems(): if package in dep: if not p in res: res.append(p) if not one: self.depends_on(p, res, one) return res rosdistro-0.5.0/src/rosdistro/common.py000066400000000000000000000012461300343010100202110ustar00rootroot00000000000000from __future__ import print_function import sys _quiet = False def quiet(state=True): global _quiet _quiet = state def _print_func(msg, end, file): global _quiet if not _quiet: print(msg, end=end, file=file) def override_print(print_func=_print_func): global _print _print = print_func _print = _print_func def info(msg, end=None, file=None): global _print _print(msg, end=end, file=file) def warning(msg, end=None, file=None): global _print _print(msg, end=end, file=file) def error(msg, end=None, file=None): global _print file = sys.stderr if file is None else file _print(msg, end=end, file=file) rosdistro-0.5.0/src/rosdistro/dependency_walker.py000066400000000000000000000154341300343010100224100ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from catkin_pkg.package import InvalidPackage, parse_package_string class DependencyWalker(object): def __init__(self, distribution_instance): self._distribution_instance = distribution_instance self._packages = {} def _get_package(self, pkg_name): if pkg_name not in self._packages: repo = self._distribution_instance.repositories[self._distribution_instance.release_packages[pkg_name].repository_name].release_repository assert repo is not None and repo.version is not None, "Package '%s' in repository '%s' has no version set" % (pkg_name, repo.name) assert 'release' in repo.tags, "Package '%s' in repository '%s' has no 'release' tag set" % (pkg_name, repo.name) pkg_xml = self._distribution_instance.get_release_package_xml(pkg_name) try: pkg = parse_package_string(pkg_xml) except InvalidPackage as e: raise InvalidPackage(pkg_name + ': %s' % str(e)) self._packages[pkg_name] = pkg return self._packages[pkg_name] def _get_package_names(self): return self._distribution_instance.release_packages.keys() def get_depends(self, pkg_name, depend_type, ros_packages_only=False): '''Return a set of package names which the package depends on.''' deps = self._get_dependencies(pkg_name, depend_type) if ros_packages_only: deps &= set(self._get_package_names()) return deps def get_recursive_depends(self, pkg_name, depend_types, ros_packages_only=False, ignore_pkgs=None, limit_depth=None): '''Return a set of package names which the package (transitively) depends on.''' ignore_pkgs = set(ignore_pkgs or []) depends = set([]) # mapping from the scheduled pkg names to their dependency level pkgs_to_check = {pkg_name: 0} while pkgs_to_check: next_pkg_to_check = sorted(pkgs_to_check.keys())[0] current_level = pkgs_to_check.pop(next_pkg_to_check) if next_pkg_to_check in ignore_pkgs or (limit_depth is not None and current_level >= limit_depth): continue for depend_type in depend_types: deps = self.get_depends(next_pkg_to_check, depend_type, ros_packages_only=ros_packages_only) deps -= ignore_pkgs new_deps = deps - depends for new_dep in new_deps: if new_dep not in pkgs_to_check: pkgs_to_check[new_dep] = current_level + 1 else: pkgs_to_check[new_dep] = min(pkgs_to_check[new_dep], current_level + 1) depends |= new_deps return depends def get_depends_on(self, pkg_name, depend_type, ignore_pkgs=None): '''Return a set of package names which depend on the package.''' ignore_pkgs = ignore_pkgs or [] depends_on = set([]) for name in self._get_package_names(): if name in ignore_pkgs: continue pkg = self._get_package(name) repo = self._distribution_instance.repositories[pkg.repository_name].release_repository if repo is None or repo.version is None: continue deps = self._get_dependencies(name, depend_type) if pkg_name in deps: depends_on.add(name) return depends_on def get_recursive_depends_on(self, pkg_name, depend_types, ignore_pkgs=None): '''Return a set of package names which (transitively) depend on the package.''' ignore_pkgs = ignore_pkgs or [] depends_on = set([]) pkgs_to_check = set([pkg_name]) while pkgs_to_check: next_pkg_to_check = pkgs_to_check.pop() for depend_type in depend_types: deps = self.get_depends_on(next_pkg_to_check, depend_type, ignore_pkgs=ignore_pkgs) new_deps = deps - depends_on pkgs_to_check |= new_deps depends_on |= new_deps return depends_on def _get_dependencies(self, pkg_name, dep_type): pkg = self._get_package(pkg_name) deps = { 'buildtool': pkg.buildtool_depends, 'build': pkg.build_depends, 'run': pkg.run_depends, 'test': pkg.test_depends } return set([d.name for d in deps[dep_type]]) class SourceDependencyWalker(DependencyWalker): def _get_package(self, pkg_name): if pkg_name not in self._packages: repo = self._distribution_instance.repositories[self._distribution_instance.source_packages[pkg_name].repository_name].source_repository assert repo is not None, "Package '%s' in repository '%s' is missing a source entry." % (pkg_name, repo.name) pkg_xml = self._distribution_instance.get_source_package_xml(pkg_name) try: pkg = parse_package_string(pkg_xml) except InvalidPackage as e: raise InvalidPackage(pkg_name + ': %s' % str(e)) self._packages[pkg_name] = pkg return self._packages[pkg_name] def _get_package_names(self): return self._distribution_instance.source_packages.keys() rosdistro-0.5.0/src/rosdistro/develdistro.py000066400000000000000000000023571300343010100212510ustar00rootroot00000000000000try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen import yaml class DevelDistro: def __init__(self, name): url = urlopen('https://raw.github.com/ros/rosdistro/master/releases/{0}-devel.yaml'.format(name)) distro = yaml.load(url.read())['repositories'] self.repositories = {} for name, data in distro.iteritems(): repo = DevelDistroRepo(name, data) self.repositories[name] = repo class DevelDistroRepo: def __init__(self, name, data): self.name = name self.type = data['type'] self.url = data['url'] self.version = None if 'version' in data: self.version = data['version'] def get_rosinstall(self): if self.version: return yaml.dump([{ self.type: { 'local-name': self.name, 'uri': '{0}'.format(self.url), 'version': '{0}'.format(self.version) }}], default_style=False) else: return yaml.dump([{ self.type: { 'local-name': self.name, 'uri': '{0}'.format(self.url) }}], default_style=False) rosdistro-0.5.0/src/rosdistro/distribution.py000066400000000000000000000114251300343010100214400ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from .manifest_provider.bitbucket import bitbucket_manifest_provider from .manifest_provider.git import git_manifest_provider, git_source_manifest_provider from .manifest_provider.github import github_manifest_provider, github_source_manifest_provider from .package import Package class Distribution(object): default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider] default_source_manifest_providers = [github_source_manifest_provider, git_source_manifest_provider] def __init__(self, distribution_file, manifest_providers=None, source_manifest_providers=None): self._distribution_file = distribution_file # Use default self._manifest_providers = Distribution.default_manifest_providers self._source_manifest_providers = Distribution.default_source_manifest_providers # Override default if given if manifest_providers is not None: self._manifest_providers = manifest_providers if source_manifest_providers is not None: self._source_manifest_providers = source_manifest_providers self._release_package_xmls = {} self._source_repo_package_xmls = {} def __getattr__(self, name): return getattr(self._distribution_file, name) def get_release_package_xml(self, pkg_name): if pkg_name not in self._release_package_xmls: pkg = self._distribution_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = self._distribution_file.repositories[repo_name] if repo.release_repository is None: return None repo = repo.release_repository if repo.version is None: return None package_xml = None for mp in self._manifest_providers: package_xml = mp(self._distribution_file.name, repo, pkg_name) if package_xml is not None: break self._release_package_xmls[pkg_name] = package_xml return self._release_package_xmls[pkg_name] def get_source_package_xml(self, pkg_name): repo_name = self._distribution_file.source_packages[pkg_name].repository_name repo_cache = self.get_source_repo_package_xmls(repo_name) if repo_cache: return repo_cache[pkg_name][1] else: return None def get_source_repo_package_xmls(self, repo_name): if repo_name in self._source_repo_package_xmls: return self._source_repo_package_xmls[repo_name] else: for mp in self._source_manifest_providers: repo_cache = mp(self.repositories[repo_name].source_repository) if repo_cache is not None: # Update map of package XMLs, and also list of known package names. self._source_repo_package_xmls[repo_name] = repo_cache for pkg_name in repo_cache: if pkg_name[0] != '_': self._distribution_file.source_packages[pkg_name] = Package(pkg_name, repo_name) return self._source_repo_package_xmls[repo_name] return None rosdistro-0.5.0/src/rosdistro/distribution_cache.py000066400000000000000000000226261300343010100225700ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function from . import logger from .distribution_file import create_distribution_file from .package import Package from .vcs import Git, ref_is_hash import sys class DistributionCache(object): _type = 'cache' def __init__(self, name, data=None, distribution_file_data=None): assert data or distribution_file_data if data: assert 'type' in data, "Expected file type is '%s'" % DistributionCache._type assert data['type'] == DistributionCache._type, "Expected file type is '%s', not '%s'" % (DistributionCache._type, data['type']) assert 'version' in data, "Distribution cache file for '%s' lacks required version information" % name self.version = int(data['version']) assert self.version > 1, "Unable to handle '%s' format version '%d' anymore, please update your '%s' file to version '2'" % (DistributionCache._type, self.version, DistributionCache._type) assert self.version == 2, "Unable to handle '%s' format version '%d', please update rosdistro (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-rosdistro)" % (DistributionCache._type, self.version) assert 'name' in data, "Distribution cache file for '%s' lacks required name information" % name assert data['name'] == name, "Distribution cache file for '%s' does not match the name '%s'" % (name, data['name']) else: self.version = 2 self._distribution_file_data = data['distribution_file'] if data else distribution_file_data self.distribution_file = create_distribution_file(name, self._distribution_file_data) self.release_package_xmls = data['release_package_xmls'] if data else {} self.source_repo_package_xmls = data['source_repo_package_xmls'] if data and 'source_repo_package_xmls' in data else {} self.distribution_file.source_packages = self.get_source_packages() # if Python 2 has converted the xml to unicode, convert it back for k, v in self.release_package_xmls.items(): if not isinstance(v, str) and not isinstance(v, bytes): self.release_package_xmls[k] = v.encode('utf-8') def get_data(self): data = {} data['type'] = 'cache' data['version'] = 2 data['name'] = self.distribution_file.name data['distribution_file'] = self._distribution_file_data data['release_package_xmls'] = self.release_package_xmls data['source_repo_package_xmls'] = self.source_repo_package_xmls return data def update_distribution(self, distribution_file_data): # remove packages which are not in the old distribution file self._remove_obsolete_entries() # determine differences in doc and source entries if len(distribution_file_data) == len(self._distribution_file_data): for old_data, new_data in zip(self._distribution_file_data, distribution_file_data): for repo_name in sorted(new_data['repositories'].keys()): repo = new_data['repositories'][repo_name] for section in ['doc', 'source']: if section not in repo: continue if repo_name in old_data['repositories'] and \ section in old_data['repositories'][repo_name] and \ old_data['repositories'][repo_name][section] == repo[section]: continue # section is either different or does't exist before print(" - updated '%s' entry for repository '%s'" % (section, repo_name)) self._distribution_file_data = distribution_file_data dist_file = create_distribution_file(self.distribution_file.name, self._distribution_file_data) # remove all release package xmls where the package version has changed. print("- removing invalid release package cache entries.") for pkg_name in sorted(dist_file.release_packages.keys()): if pkg_name not in self.distribution_file.release_packages: continue if pkg_name in self.release_package_xmls and self._get_repo_info(dist_file, pkg_name) != self._get_repo_info(self.distribution_file, pkg_name): logger.debug("Dropping release package XML cache for %s" % pkg_name) del self.release_package_xmls[pkg_name] # Remove all source package xmls where the devel branch is pointing to a different commit than # the one we have associated with our cache. This requires calling git ls-remote on all affected repos. if self.source_repo_package_xmls: print("- checking invalid source repo cache entries.") for repo in sorted(self.source_repo_package_xmls.keys()): sys.stdout.write('.') sys.stdout.flush() try: source_repository = dist_file.repositories[repo].source_repository except (KeyError, AttributeError): # The repo entry has been dropped, or the source stanza from it has been dropped, # either way, remove the cache entries associated with this repository. logger.debug('Unable to find source repository info for repo "%s".' % repo) del self.source_repo_package_xmls[repo] continue if ref_is_hash(source_repository.version): source_hash = source_repository.version else: result = Git().command('ls-remote', source_repository.url, source_repository.version) if result['returncode'] != 0 or not result['output']: # Error checking remote, or unable to find remote reference. Drop the cache entry. logger.debug("Unable to check hash for branch %s of %s, dropping cache entry." % (source_repository.version, source_repository.url)) del self.source_repo_package_xmls[repo] continue # Split by line first and take the last line, to squelch any preamble output, for example # a known host key validation notice. source_hash = result['output'].split('\n')[-1].split('\t')[0] cached_hash = self.source_repo_package_xmls[repo]['_ref'] if source_hash != cached_hash: logger.debug('Repo "%s" has moved from %s to %s, dropping cache.' % (repo, cached_hash, source_hash)) del self.source_repo_package_xmls[repo] sys.stdout.write('\n') self.distribution_file = dist_file self.distribution_file.source_packages = self.get_source_packages() # remove packages which are not in the new distribution file self._remove_obsolete_entries() def get_source_packages(self): """ Returns dictionary mapping source package names to Package() objects. """ package_dict = {} for source_repo_name, source_repo in self.source_repo_package_xmls.items(): for pkg_name in source_repo: if pkg_name[0] != '_': package_dict[pkg_name] = Package(pkg_name, source_repo_name) return package_dict def _get_repo_info(self, dist_file, pkg_name): pkg = dist_file.release_packages[pkg_name] repo = dist_file.repositories[pkg.repository_name].release_repository return (repo.version, repo.url) def _remove_obsolete_entries(self): for pkg_name in list(self.release_package_xmls.keys()): if pkg_name not in self.distribution_file.release_packages: print('- REMOVE', pkg_name) del self.release_package_xmls[pkg_name] rosdistro-0.5.0/src/rosdistro/distribution_cache_generator.py000066400000000000000000000220271300343010100246310ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function import gzip import os import re import sys import yaml from catkin_pkg.package import InvalidPackage, parse_package_string from . import _get_dist_file_data, get_cached_distribution, get_index, get_distribution_cache from .distribution_cache import DistributionCache def generate_distribution_caches(index, dist_names=None, preclean=False, ignore_local=False, include_source=False, debug=False): if os.path.isfile(index): index = 'file://' + os.path.abspath(index) index = get_index(index) if not dist_names: dist_names = sorted(index.distributions.keys()) errors = [] caches = {} for dist_name in dist_names: try: cache = generate_distribution_cache(index, dist_name, preclean=preclean, ignore_local=ignore_local, include_source=include_source, debug=debug) except RuntimeError as e: errors.append(str(e)) continue caches[dist_name] = cache if errors: raise RuntimeError('\n'.join(errors)) return caches def generate_distribution_cache(index, dist_name, preclean=False, ignore_local=False, include_source=False, debug=False): dist, cache = _get_cached_distribution(index, dist_name, preclean=preclean, ignore_local=ignore_local, include_source=include_source) print('- fetch missing release manifests') errors = [] for pkg_name in sorted(dist.release_packages.keys()): repo = dist.repositories[dist.release_packages[pkg_name].repository_name].release_repository if repo.version is None: if debug: print(' - skip "%s" since it has no version' % pkg_name) continue if debug: print(' - fetch "%s"' % pkg_name) else: sys.stdout.write('.') sys.stdout.flush() # check that package.xml is fetchable old_package_xml = None if cache and pkg_name in cache.release_package_xmls: old_package_xml = cache.release_package_xmls[pkg_name] package_xml = dist.get_release_package_xml(pkg_name) if not package_xml: errors.append('%s: missing package.xml file for package "%s"' % (dist_name, pkg_name)) continue # check that package.xml is parseable try: pkg = parse_package_string(package_xml) except InvalidPackage as e: errors.append('%s: invalid package.xml file for package "%s": %s' % (dist_name, pkg_name, e)) continue # check that version numbers match (at least without deb inc) if not re.match('^%s-[\dA-z~\+\.]+$' % re.escape(pkg.version), repo.version): errors.append('%s: different version in package.xml (%s) for package "%s" than for the repository (%s) (after removing the debian increment)' % (dist_name, pkg.version, pkg_name, repo.version)) if package_xml != old_package_xml: print(" - updated manifest of package '%s' to version '%s'" % (pkg_name, pkg.version)) if not debug: print('') if include_source: print('- fetch source repository manifests') for repo_name in sorted(dist.repositories.keys()): if dist.repositories[repo_name].source_repository: source_repo_cache = dist.get_source_repo_package_xmls(repo_name) if debug: print(' - fetch "%s"' % repo_name) else: sys.stdout.write('.') sys.stdout.flush() else: if debug: print(' - skip "%s" since it has no source entry.' % repo_name) continue if not debug: print('') if errors: raise RuntimeError('\n'.join(errors)) return cache class CacheYamlDumper(yaml.SafeDumper): """ A yaml dumper specific to dumping the serialized rosdistro cache file. Allows long lines and direct unicode representation. This avoids writing escape sequences, line continuations, and other noise into the cache file. Also permits long strings to alias each other (by default only objects do). """ def __init__(self, *args, **kwargs): kwargs['width'] = 10000 kwargs['allow_unicode'] = True super(CacheYamlDumper, self).__init__(*args, **kwargs) def ignore_aliases(self, content): """ Allow strings that look like package XML to alias to each other in the YAML output. """ try: basestring except NameError: # Python 3 basestring = str return not (isinstance(content, basestring) and ' for details and usage. """ # Dev Notes: # - MSDN on where to store app data files: # http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120 # - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html # - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html __version_info__ = (1, 3, 0) __version__ = '.'.join(map(str, __version_info__)) import sys import os PY3 = sys.version_info[0] == 3 if PY3: unicode = str def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): r"""Return full path to the user-specific data dir for this application. "appname" is the name of application. If None, just the system directory is returned. "appauthor" (only required and used on Windows) is the name of the appauthor or distributing body for this application. Typically it is the owning company name. This falls back to appname. "version" is an optional version path element to append to the path. You might want to use this if you want multiple versions of your app to be able to run independently. If used, this would typically be ".". Only applied when appname is present. "roaming" (boolean, default False) can be set True to use the Windows roaming appdata directory. That means that for users on a Windows network setup for roaming profiles, this user data will be sync'd on login. See for a discussion of issues. Typical user data directories are: Mac OS X: ~/Library/Application Support/ Unix: ~/.local/share/ # or in $XDG_DATA_HOME, if defined Win XP (not roaming): C:\Documents and Settings\\Application Data\\ Win XP (roaming): C:\Documents and Settings\\Local Settings\Application Data\\ Win 7 (not roaming): C:\Users\\AppData\Local\\ Win 7 (roaming): C:\Users\\AppData\Roaming\\ For Unix, we follow the XDG spec and support $XDG_DATA_HOME. That means, by deafult "~/.local/share/". """ if sys.platform == "win32": if appauthor is None: appauthor = appname const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA" path = os.path.normpath(_get_win_folder(const)) if appname: path = os.path.join(path, appauthor, appname) elif sys.platform == 'darwin': path = os.path.expanduser('~/Library/Application Support/') if appname: path = os.path.join(path, appname) else: path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) if appname: path = os.path.join(path, appname) if appname and version: path = os.path.join(path, version) return path def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): """Return full path to the user-shared data dir for this application. "appname" is the name of application. If None, just the system directory is returned. "appauthor" (only required and used on Windows) is the name of the appauthor or distributing body for this application. Typically it is the owning company name. This falls back to appname. "version" is an optional version path element to append to the path. You might want to use this if you want multiple versions of your app to be able to run independently. If used, this would typically be ".". Only applied when appname is present. "multipath" is an optional parameter only applicable to *nix which indicates that the entire list of data dirs should be returned. By default, the first item from XDG_DATA_DIRS is returned, or '/usr/local/share/', if XDG_DATA_DIRS is not set Typical user data directories are: Mac OS X: /Library/Application Support/ Unix: /usr/local/share/ or /usr/share/ Win XP: C:\Documents and Settings\All Users\Application Data\\ Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) Win 7: C:\ProgramData\\ # Hidden, but writeable on Win 7. For Unix, this is using the $XDG_DATA_DIRS[0] default. WARNING: Do not use this on Windows. See the Vista-Fail note above for why. """ if sys.platform == "win32": if appauthor is None: appauthor = appname path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA")) if appname: path = os.path.join(path, appauthor, appname) elif sys.platform == 'darwin': path = os.path.expanduser('/Library/Application Support') if appname: path = os.path.join(path, appname) else: # XDG default for $XDG_DATA_DIRS # only first, if multipath is False path = os.getenv('XDG_DATA_DIRS', os.pathsep.join(['/usr/local/share', '/usr/share'])) pathlist = [ os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) ] if appname: if version: appname = os.path.join(appname, version) pathlist = [ os.sep.join([x, appname]) for x in pathlist ] if multipath: path = os.pathsep.join(pathlist) else: path = pathlist[0] return path if appname and version: path = os.path.join(path, version) return path def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): r"""Return full path to the user-specific config dir for this application. "appname" is the name of application. If None, just the system directory is returned. "appauthor" (only required and used on Windows) is the name of the appauthor or distributing body for this application. Typically it is the owning company name. This falls back to appname. "version" is an optional version path element to append to the path. You might want to use this if you want multiple versions of your app to be able to run independently. If used, this would typically be ".". Only applied when appname is present. "roaming" (boolean, default False) can be set True to use the Windows roaming appdata directory. That means that for users on a Windows network setup for roaming profiles, this user data will be sync'd on login. See for a discussion of issues. Typical user data directories are: Mac OS X: same as user_data_dir Unix: ~/.config/ # or in $XDG_CONFIG_HOME, if defined Win *: same as user_data_dir For Unix, we follow the XDG spec and support $XDG_DATA_HOME. That means, by deafult "~/.local/share/". """ if sys.platform in [ "win32", "darwin" ]: path = user_data_dir(appname, appauthor, None, roaming) else: path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")) if appname: path = os.path.join(path, appname) if appname and version: path = os.path.join(path, version) return path def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): """Return full path to the user-shared data dir for this application. "appname" is the name of application. If None, just the system directory is returned. "appauthor" (only required and used on Windows) is the name of the appauthor or distributing body for this application. Typically it is the owning company name. This falls back to appname. "version" is an optional version path element to append to the path. You might want to use this if you want multiple versions of your app to be able to run independently. If used, this would typically be ".". Only applied when appname is present. "multipath" is an optional parameter only applicable to *nix which indicates that the entire list of config dirs should be returned. By default, the first item from XDG_CONFIG_DIRS is returned, or '/etc/xdg/', if XDG_CONFIG_DIRS is not set Typical user data directories are: Mac OS X: same as site_data_dir Unix: /etc/xdg/ or $XDG_CONFIG_DIRS[i]/ for each value in $XDG_CONFIG_DIRS Win *: same as site_data_dir Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False WARNING: Do not use this on Windows. See the Vista-Fail note above for why. """ if sys.platform in [ "win32", "darwin" ]: path = site_data_dir(appname, appauthor) if appname and version: path = os.path.join(path, version) else: # XDG default for $XDG_CONFIG_DIRS # only first, if multipath is False path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') pathlist = [ os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) ] if appname: if version: appname = os.path.join(appname, version) pathlist = [ os.sep.join([x, appname]) for x in pathlist ] if multipath: path = os.pathsep.join(pathlist) else: path = pathlist[0] return path def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): r"""Return full path to the user-specific cache dir for this application. "appname" is the name of application. If None, just the system directory is returned. "appauthor" (only required and used on Windows) is the name of the appauthor or distributing body for this application. Typically it is the owning company name. This falls back to appname. "version" is an optional version path element to append to the path. You might want to use this if you want multiple versions of your app to be able to run independently. If used, this would typically be ".". Only applied when appname is present. "opinion" (boolean) can be False to disable the appending of "Cache" to the base app data dir for Windows. See discussion below. Typical user cache directories are: Mac OS X: ~/Library/Caches/ Unix: ~/.cache/ (XDG default) Win XP: C:\Documents and Settings\\Local Settings\Application Data\\\Cache Vista: C:\Users\\AppData\Local\\\Cache On Windows the only suggestion in the MSDN docs is that local settings go in the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming app data dir (the default returned by `user_data_dir` above). Apps typically put cache data somewhere *under* the given dir here. Some examples: ...\Mozilla\Firefox\Profiles\\Cache ...\Acme\SuperApp\Cache\1.0 OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. This can be disabled with the `opinion=False` option. """ if sys.platform == "win32": if appauthor is None: appauthor = appname path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) if appname: path = os.path.join(path, appauthor, appname) if opinion: path = os.path.join(path, "Cache") elif sys.platform == 'darwin': path = os.path.expanduser('~/Library/Caches') if appname: path = os.path.join(path, appname) else: path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) if appname: path = os.path.join(path, appname) if appname and version: path = os.path.join(path, version) return path def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): r"""Return full path to the user-specific log dir for this application. "appname" is the name of application. If None, just the system directory is returned. "appauthor" (only required and used on Windows) is the name of the appauthor or distributing body for this application. Typically it is the owning company name. This falls back to appname. "version" is an optional version path element to append to the path. You might want to use this if you want multiple versions of your app to be able to run independently. If used, this would typically be ".". Only applied when appname is present. "opinion" (boolean) can be False to disable the appending of "Logs" to the base app data dir for Windows, and "log" to the base cache dir for Unix. See discussion below. Typical user cache directories are: Mac OS X: ~/Library/Logs/ Unix: ~/.cache//log # or under $XDG_CACHE_HOME if defined Win XP: C:\Documents and Settings\\Local Settings\Application Data\\\Logs Vista: C:\Users\\AppData\Local\\\Logs On Windows the only suggestion in the MSDN docs is that local settings go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in examples of what some windows apps use for a logs dir.) OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA` value for Windows and appends "log" to the user cache dir for Unix. This can be disabled with the `opinion=False` option. """ if sys.platform == "darwin": path = os.path.join( os.path.expanduser('~/Library/Logs'), appname) elif sys.platform == "win32": path = user_data_dir(appname, appauthor, version); version=False if opinion: path = os.path.join(path, "Logs") else: path = user_cache_dir(appname, appauthor, version); version=False if opinion: path = os.path.join(path, "log") if appname and version: path = os.path.join(path, version) return path class AppDirs(object): """Convenience wrapper for getting application dirs.""" def __init__(self, appname, appauthor=None, version=None, roaming=False, multipath=False): self.appname = appname self.appauthor = appauthor self.version = version self.roaming = roaming self.multipath = multipath @property def user_data_dir(self): return user_data_dir(self.appname, self.appauthor, version=self.version, roaming=self.roaming) @property def site_data_dir(self): return site_data_dir(self.appname, self.appauthor, version=self.version, multipath=self.multipath) @property def user_config_dir(self): return user_config_dir(self.appname, self.appauthor, version=self.version, roaming=self.roaming) @property def site_config_dir(self): return site_data_dir(self.appname, self.appauthor, version=self.version, multipath=self.multipath) @property def user_cache_dir(self): return user_cache_dir(self.appname, self.appauthor, version=self.version) @property def user_log_dir(self): return user_log_dir(self.appname, self.appauthor, version=self.version) #---- internal support stuff def _get_win_folder_from_registry(csidl_name): """This is a fallback technique at best. I'm not sure if using the registry for this guarantees us the correct answer for all CSIDL_* names. """ import _winreg shell_folder_name = { "CSIDL_APPDATA": "AppData", "CSIDL_COMMON_APPDATA": "Common AppData", "CSIDL_LOCAL_APPDATA": "Local AppData", }[csidl_name] key = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders") dir, type = _winreg.QueryValueEx(key, shell_folder_name) return dir def _get_win_folder_with_pywin32(csidl_name): from win32com.shell import shellcon, shell dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0) # Try to make this a unicode path because SHGetFolderPath does # not return unicode strings when there is unicode data in the # path. try: dir = unicode(dir) # Downgrade to short path name if have highbit chars. See # . has_high_char = False for c in dir: if ord(c) > 255: has_high_char = True break if has_high_char: try: import win32api dir = win32api.GetShortPathName(dir) except ImportError: pass except UnicodeError: pass return dir def _get_win_folder_with_ctypes(csidl_name): import ctypes csidl_const = { "CSIDL_APPDATA": 26, "CSIDL_COMMON_APPDATA": 35, "CSIDL_LOCAL_APPDATA": 28, }[csidl_name] buf = ctypes.create_unicode_buffer(1024) ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) # Downgrade to short path name if have highbit chars. See # . has_high_char = False for c in buf: if ord(c) > 255: has_high_char = True break if has_high_char: buf2 = ctypes.create_unicode_buffer(1024) if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): buf = buf2 return buf.value if sys.platform == "win32": try: import win32com.shell _get_win_folder = _get_win_folder_with_pywin32 except ImportError: try: import ctypes _get_win_folder = _get_win_folder_with_ctypes except ImportError: _get_win_folder = _get_win_folder_from_registry #---- self test code if __name__ == "__main__": appname = "MyApp" appauthor = "MyCompany" props = ("user_data_dir", "site_data_dir", "user_config_dir", "site_config_dir", "user_cache_dir", "user_log_dir") print("-- app dirs (with optional 'version')") dirs = AppDirs(appname, appauthor, version="1.0") for prop in props: print("%s: %s" % (prop, getattr(dirs, prop))) print("\n-- app dirs (without optional 'version')") dirs = AppDirs(appname, appauthor) for prop in props: print("%s: %s" % (prop, getattr(dirs, prop))) print("\n-- app dirs (without optional 'appauthor')") dirs = AppDirs(appname) for prop in props: print("%s: %s" % (prop, getattr(dirs, prop))) rosdistro-0.5.0/src/rosdistro/freeze_source.py000066400000000000000000000102361300343010100215600ustar00rootroot00000000000000#!/usr/bin/env python # Software License Agreement (BSD License) # # Copyright (c) 2016, Clearpath Robotics # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function import sys import subprocess import threading import time import yaml try: import queue except ImportError: import Queue as queue CONCURRENT_DEFAULT = 16 def freeze_distribution_sources(dist, release_version=False, release_tag=False, concurrent_ops=CONCURRENT_DEFAULT, quiet=False): # Populate this queue with tuples of repositories instances to be updated, # so that this work can be spread across multiple threads. work_queue = queue.Queue() for repo_name, repo in dist.repositories.iteritems(): # Only manipulate distribution entries with a source repo listed. if repo.source_repository: # Decide which git ref string we'll be using as the replacement match. if repo.release_repository and (release_version or release_tag): version = repo.release_repository.version.split('-')[0] else: version = repo.source_repository.version work_queue.put((repo.source_repository, version, release_tag)) total_items = work_queue.qsize() for i in range(concurrent_ops): threading.Thread(target=_worker, args=[work_queue]).start() # Wait until the threads have done all the work and exited. while not work_queue.empty(): time.sleep(0.1) if not quiet: sys.stdout.write("Updating source repo versions (%d/%d) \r" % (total_items - work_queue.qsize(), total_items)) sys.stdout.flush() work_queue.join() # Clear past the updating line. if not quiet: print("") def _worker(work_queue): while True: try: source_repo, freeze_version, freeze_to_tag = work_queue.get(block=False) cmd = ['git', 'ls-remote', source_repo.url] ls_remote_lines = subprocess.check_output(cmd).splitlines() for line in ls_remote_lines: hash, ref = line.split('\t', 1) if freeze_to_tag and ref == 'refs/tags/%s' % freeze_version: source_repo.version = ref.split('refs/tags/')[1] break elif ref in ('refs/heads/%s' % freeze_version, 'refs/tags/%s' % freeze_version): source_repo.version = hash break work_queue.task_done() except subprocess.CalledProcessError: print("Non-zero return code for: %s" % ' '.join(cmd), file=sys.stderr) work_queue.task_done() except queue.Empty: break rosdistro-0.5.0/src/rosdistro/index.py000066400000000000000000000126431300343010100200330ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. import os try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse class Index(object): _type = 'index' def __init__(self, data, base_url, url_query=''): assert 'type' in data, "Expected file type is '%s'" % Index._type assert data['type'] == Index._type, "Expected file type is '%s', not '%s'" % (Index._type, data['type']) assert 'version' in data, 'Index file lacks required version information' assert int(data['version']) > 1, "Unable to handle '%s' format version '%d' anymore, please update your '%s' file to version '2' or '3'" % (Index._type, int(data['version']), Index._type) assert int(data['version']) in [2, 3], "Unable to handle '%s' format version '%d', please update rosdistro (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-rosdistro)" % (Index._type, int(data['version'])) self.version = int(data['version']) self.distributions = {} if 'distributions' in data and data['distributions']: # if distributions is not a dict raise an exception including the value # this can be used to notify users (e.g. if an index.yaml file has been deleted / moved) if not isinstance(data['distributions'], dict): raise RuntimeError("Distributions type is invalid: expected 'dict', but got '%s': %s" % (type(data['distributions']).__name__, data['distributions'])) for distro_name in sorted(data['distributions']): self.distributions[distro_name] = {} distro_data = data['distributions'][distro_name] for key in distro_data: if key in ['distribution']: if self.version == 2: list_value = False elif self.version == 3: list_value = True else: assert False elif key in ['distribution_cache']: list_value = False elif key in ['release_builds', 'source_builds', 'doc_builds']: if self.version == 2: list_value = True elif self.version == 3: assert False, "'%s' format version '3' does not allow a '%s' entry anymore" % (Index._type, key) else: assert False else: assert False, 'unknown key "%s"' % key self.distributions[distro_name][key] = [] value = distro_data[key] if list_value != isinstance(value, list): assert False, 'wrong type of key "%s"' % key if not list_value: value = [value] for v in value: parts = urlparse(v) if not parts[0]: # schema v = base_url + '/' + v if url_query: v += '?' + url_query self.distributions[distro_name][key].append(v) if not list_value: self.distributions[distro_name][key] = self.distributions[distro_name][key][0] # for backward compatibility only if key == 'distribution': self.distributions[distro_name]['release'] = self.distributions[distro_name][key] # for backward compatibility only for key in ['release_builds', 'source_builds', 'doc_builds']: if key not in self.distributions[distro_name]: self.distributions[distro_name][key] = [] rosdistro-0.5.0/src/rosdistro/loader.py000066400000000000000000000054031300343010100201660ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. import socket import time try: from urllib.request import urlopen from urllib.error import HTTPError from urllib.error import URLError except ImportError: from urllib2 import urlopen from urllib2 import HTTPError from urllib2 import URLError def load_url(url, retry=2, retry_period=1, timeout=10, skip_decode=False): try: fh = urlopen(url, timeout=timeout) except HTTPError as e: if e.code in [500, 502, 503] and retry: time.sleep(retry_period) return load_url(url, retry=retry - 1, retry_period=retry_period, timeout=timeout) e.msg += ' (%s)' % url raise except URLError as e: if isinstance(e.reason, socket.timeout) and retry: time.sleep(retry_period) return load_url(url, retry=retry - 1, retry_period=retry_period, timeout=timeout) raise URLError(str(e) + ' (%s)' % url) except socket.timeout as e: raise socket.timeout(str(e) + ' (%s)' % url) # Python 2/3 Compatibility contents = fh.read() if isinstance(contents, str) or skip_decode: return contents else: return contents.decode('utf-8') rosdistro-0.5.0/src/rosdistro/manifest_provider/000077500000000000000000000000001300343010100220645ustar00rootroot00000000000000rosdistro-0.5.0/src/rosdistro/manifest_provider/__init__.py000066400000000000000000000035221300343010100241770ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # This function remains here as a compatibility shim because there is code # in older rosinstall_generator versions which uses it in this location. def get_release_tag(repo, pkg_name): return repo.get_release_tag(pkg_name) rosdistro-0.5.0/src/rosdistro/manifest_provider/bitbucket.py000066400000000000000000000066271300343010100244250ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2016, Clearpath Robotics # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. try: from urllib.request import urlopen, Request from urllib.error import URLError except ImportError: from urllib2 import urlopen, Request from urllib2 import URLError import base64 import os from rosdistro import logger # This Bitbucket provider can optionally send basic auth to fetch from # private repositories if you supply the environment variables # BITBUCKET_USER and BITBUCKET_PASSWORD. This can be the credentials # for an actual Bitbucket user, or it can be a team name with API key. # # Generate an API key for your team here: # https://bitbucket.org/account/user//api-key/ BITBUCKET_USER = os.getenv('BITBUCKET_USER', None) BITBUCKET_PASSWORD = os.getenv('BITBUCKET_PASSWORD', None) def bitbucket_manifest_provider(_dist_name, repo, pkg_name): assert repo.version server, path = repo.get_url_parts() if server != 'bitbucket.org': logger.debug('Skip non-bitbucket url "%s"' % repo.url) raise RuntimeError('Cannot handle non bitbucket url.') release_tag = repo.get_release_tag(pkg_name) if not repo.has_remote_tag(release_tag): raise RuntimeError('specified tag "%s" is not a git tag' % release_tag) url = 'https://bitbucket.org/%s/raw/%s/package.xml' % (path, release_tag) try: logger.debug('Load package.xml file from url "%s"' % url) req = Request(url) if BITBUCKET_USER and BITBUCKET_PASSWORD: logger.debug('- using http basic auth from supplied environment variables.') authheader = 'Basic %s' % base64.b64encode('%s:%s' % (BITBUCKET_USER, BITBUCKET_PASSWORD)) req.add_header('Authorization', authheader) package_xml = urlopen(req).read().decode('utf-8') return package_xml except URLError as e: logger.debug('- failed (%s)' % e) raise RuntimeError() rosdistro-0.5.0/src/rosdistro/manifest_provider/cache.py000066400000000000000000000133741300343010100235110ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from rosdistro import logger from xml.dom import minidom def sanitize_xml(xml_string): """ Returns a version of the supplied XML string with comments and all whitespace stripped, including runs of spaces internal to text nodes. The returned string will be encoded, so str (Python 2) or bytes (Python 3). """ def _squash(node): drop_nodes = [] for x in node.childNodes: if x.nodeType == minidom.Node.TEXT_NODE: if x.nodeValue: x.nodeValue = ' '.join(x.nodeValue.strip().split()) elif x.nodeType == minidom.Node.ELEMENT_NODE: _squash(x) elif x.nodeType is minidom.Node.COMMENT_NODE: drop_nodes.append(x) for x in drop_nodes: node.removeChild(x) return node try: # Python 2. The minidom module parses as ascii, so we have to pre-encode. if isinstance(xml_string, unicode): xml_string = xml_string.encode('utf-8') except NameError: # Python 3. Strings are native unicode. pass xml_node = _squash(minidom.parseString(xml_string)) try: # Python 2. Encode the resultant XML as a str. unicode return xml_node.toxml('utf-8') except NameError: # Python 3. Return native bytes. return xml_node.toxml() class CachedManifestProvider(object): def __init__(self, distribution_cache, manifest_providers=None): self._distribution_cache = distribution_cache self._manifest_providers = manifest_providers def __call__(self, dist_name, repo, pkg_name): assert repo.version package_xml = self._distribution_cache.release_package_xmls.get(pkg_name, None) if package_xml: package_xml = sanitize_xml(package_xml) self._distribution_cache.release_package_xmls[pkg_name] = package_xml logger.debug('Loading package.xml for package "%s" from cache' % pkg_name) else: # use manifest providers to lazy load for mp in self._manifest_providers or []: try: package_xml = sanitize_xml(mp(dist_name, repo, pkg_name)) break except Exception as e: # pass and try next manifest provider logger.debug('Skipped "%s()": %s' % (mp.__name__, e)) if package_xml is None: return None # populate the cache self._distribution_cache.release_package_xmls[pkg_name] = package_xml return package_xml class CachedSourceManifestProvider(object): def __init__(self, distribution_cache, source_manifest_providers=None): self._distribution_cache = distribution_cache self._source_manifest_providers = source_manifest_providers def __call__(self, repo): assert repo.url repo_cache = self._distribution_cache.source_repo_package_xmls.get(repo.name, None) if not repo_cache: # Use manifest providers to lazy load for mp in self._source_manifest_providers or []: try: repo_cache = mp(repo) except Exception as e: # pass and try next manifest provider logger.debug('Skipped "%s()": %s' % (mp.__name__, e)) continue self._distribution_cache.source_repo_package_xmls[repo.name] = repo_cache break else: logger.debug('Load package XMLs for repo "%s" from cache' % repo.name) # De-duplicate with the release package XMLs. This will cause the YAML writer # to use references, saving a lot of space in the cache file. for key in repo_cache or {}: if key[0] != '_': repo_cache[key][1] = sanitize_xml(repo_cache[key][1]) if key in self._distribution_cache.release_package_xmls: release_package_xml = self._distribution_cache.release_package_xmls[key] if repo_cache[key][1] == release_package_xml: repo_cache[key][1] = release_package_xml return repo_cache rosdistro-0.5.0/src/rosdistro/manifest_provider/git.py000066400000000000000000000112611300343010100232220ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from catkin_pkg.package import InvalidPackage, parse_package_string from catkin_pkg.packages import find_package_paths from contextlib import contextmanager import os import re import shutil import tempfile from rosdistro import logger from rosdistro.vcs import Git, ref_is_hash def git_manifest_provider(_dist_name, repo, pkg_name): assert repo.version try: release_tag = repo.get_release_tag(pkg_name) with _temp_git_clone(repo.url, release_tag) as git_repo_path: filename = os.path.join(git_repo_path, 'package.xml') if not os.path.exists(filename): raise RuntimeError('Could not find package.xml in repository "%s"' % repo.url) with open(filename, 'r') as f: return f.read() except Exception as e: raise RuntimeError('Unable to fetch package.xml: %s' % e) def git_source_manifest_provider(repo): try: with _temp_git_clone(repo.url, repo.version) as git_repo_path: # Include the git hash in our cache dictionary. result = Git(git_repo_path).command('rev-parse', 'HEAD') cache = { '_ref': result['output'] } # Find package.xml files inside the repo. for package_path in find_package_paths(git_repo_path): if package_path == '.': package_path = '' with open(os.path.join(git_repo_path, package_path, 'package.xml'), 'r') as f: package_xml = f.read() try: name = parse_package_string(package_xml).name except InvalidPackage: raise RuntimeError('Unable to parse package.xml file found in %s' % repo.url) cache[name] = [ package_path, package_xml ] except Exception as e: raise RuntimeError('Unable to fetch source package.xml files: %s' % e) return cache @contextmanager def _temp_git_clone(url, ref): base = tempfile.mkdtemp('rosdistro') git = Git(cwd=base) try: if git.version_gte('1.8.0') and not ref_is_hash(ref): # Directly clone the required ref with least amount of additional history. # Available since git 1.8.0, but only works for tags and branches, not hashes: # https://git.kernel.org/cgit/git/git.git/tree/Documentation/git-clone.txt?h=v1.8.0#n158 result = git.command('clone', url, '.', '--depth', '1', '--branch', ref) if result['returncode'] != 0: raise RuntimeError('Could not clone repository "%s" at reference "%s"' % (url, ref)) else: # Old git doesn't support cloning a tag/branch directly, so full clone and checkout. result = git.command('clone', url, '.') if result['returncode'] != 0: raise RuntimeError('Could not clone repository "%s"' % url) result = git.command('checkout', ref) if result['returncode'] != 0: raise RuntimeError('Could not checkout ref "%s" of repository "%s"' % (ref, url)) yield base finally: shutil.rmtree(base) rosdistro-0.5.0/src/rosdistro/manifest_provider/github.py000066400000000000000000000116251300343010100237250ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. try: from urllib.request import urlopen, Request from urllib.error import URLError except ImportError: from urllib2 import urlopen, Request from urllib2 import URLError import base64 from catkin_pkg.package import InvalidPackage, parse_package_string import json import os from rosdistro import logger GITHUB_USER = os.getenv('GITHUB_USER', None) GITHUB_PASSWORD = os.getenv('GITHUB_PASSWORD', None) def github_manifest_provider(_dist_name, repo, pkg_name): assert repo.version server, path = repo.get_url_parts() if server != 'github.com': logger.debug('Skip non-github url "%s"' % repo.url) raise RuntimeError('can not handle non github urls') release_tag = repo.get_release_tag(pkg_name) if not repo.has_remote_tag(release_tag): raise RuntimeError('specified tag "%s" is not a git tag' % release_tag) url = 'https://raw.githubusercontent.com/%s/%s/package.xml' % (path, release_tag) try: logger.debug('Load package.xml file from url "%s"' % url) package_xml = urlopen(url).read().decode('utf-8') return package_xml except URLError as e: logger.debug('- failed (%s), trying "%s"' % (e, url)) raise RuntimeError() def github_source_manifest_provider(repo): server, path = repo.get_url_parts() if server != 'github.com': logger.debug('Skip non-github url "%s"' % repo.url) raise RuntimeError('can not handle non github urls') tree_url = 'https://api.github.com/repos/%s/git/trees/%s?recursive=1' % (path, repo.version) req = Request(tree_url) if GITHUB_USER and GITHUB_PASSWORD: logger.debug('- using http basic auth from supplied environment variables.') authheader = 'Basic %s' % base64.b64encode('%s:%s' % (GITHUB_USER, GITHUB_PASSWORD)) req.add_header('Authorization', authheader) try: tree_json = json.loads(urlopen(req).read().decode('utf-8')) logger.debug('- load repo tree from %s' % tree_url) except URLError as e: raise RuntimeError('Unable to fetch JSON tree from %s: %s' % (tree_url, e)) if tree_json['truncated']: raise RuntimeError('JSON tree is truncated, must perform full clone.') package_xml_paths = set() for obj in tree_json['tree']: if obj['path'].split('/')[-1] == 'package.xml': package_xml_paths.add(os.path.dirname(obj['path'])) # Filter out ones that are inside other packages (eg, part of tests) def package_xml_in_parent(path): if path == '': return True parent = path while True: parent = os.path.dirname(parent) if parent in package_xml_paths: return False if parent == '': return True package_xml_paths = list(filter(package_xml_in_parent, package_xml_paths)) cache = { '_ref': tree_json['sha'] } for package_xml_path in package_xml_paths: url = 'https://raw.githubusercontent.com/%s/%s/%s' % \ (path, cache['_ref'], package_xml_path + '/package.xml' if package_xml_path else 'package.xml') logger.debug('- load package.xml from %s' % url) package_xml = urlopen(url).read().decode('utf-8') name = parse_package_string(package_xml).name cache[name] = [ package_xml_path, package_xml ] return cache rosdistro-0.5.0/src/rosdistro/package.py000066400000000000000000000036451300343010100203210ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from .status import valid_statuses class Package: def __init__(self, name, repository_name): self.name = name self.repository_name = repository_name # for backward compatibility only self.subfolder = None self.status = None self.status_description = None rosdistro-0.5.0/src/rosdistro/release.py000066400000000000000000000064571300343010100203520ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from .manifest_provider.bitbucket import bitbucket_manifest_provider from .manifest_provider.git import git_manifest_provider from .manifest_provider.github import github_manifest_provider class Release(object): default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider] def __init__(self, rel_file, manifest_providers=None): self._rel_file = rel_file # Use default self._manifest_providers = Release.default_manifest_providers # Override default if given if manifest_providers is not None: self._manifest_providers = manifest_providers self._package_xmls = {} def __getattr__(self, name): # for backward compatibility only if name == 'get_release_package_xml': return self.get_package_xml if name == 'release_packages': name = 'packages' if name == 'release_platforms': name = 'platforms' return getattr(self._rel_file, name) def get_package_xml(self, pkg_name): if pkg_name not in self._package_xmls: pkg = self._rel_file.packages[pkg_name] repo_name = pkg.repository_name repo = self._rel_file.repositories[repo_name] if repo.version is None: return None package_xml = None for mp in self._manifest_providers: #try: package_xml = mp(self._rel_file.name, repo, pkg_name) #except: # pass if package_xml is not None: break self._package_xmls[pkg_name] = package_xml return self._package_xmls[pkg_name] rosdistro-0.5.0/src/rosdistro/release_build.py000066400000000000000000000061751300343010100215260ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. class ReleaseBuild(object): def __init__(self, dist, release_build_file): self._dist = dist self._build_file = release_build_file self._verify_package_names(self._build_file.package_whitelist) self._verify_package_names(self._build_file.package_blacklist) for os_name in self._build_file.get_target_os_names(): assert os_name in self._dist.platforms.keys(), "Distribution '%s' specifies to build for os_name '%s' which is not listed in the release file" % (self._dist.name, os_name) for os_code_name in self._build_file.get_target_os_code_names(os_name): assert os_code_name in self._dist.platforms[os_name], "Distribution '%s' specifies to build for os_code_name '%s' which is not listed in the release file" % (self._dist.name, os_code_name) self._verify_package_names(self._build_file.sync_packages) def _verify_package_names(self, pkg_names): if pkg_names: for pkg_name in pkg_names: assert pkg_name in self._dist.packages, "Distribution '%s' specifies to build the package '%s' which is not listed in the release file" % (self._dist.name, pkg_name) def __getattr__(self, name): return getattr(self._build_file, name) def get_package_names(self): if not self._build_file.package_whitelist and not self._build_file.package_blacklist: return self._dist.get_package_names() raise NotImplementedError('Package whitelisting/blacklisting has not been implemented yet') rosdistro-0.5.0/src/rosdistro/release_build_file.py000066400000000000000000000172621300343010100225240ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. class ReleaseBuildFile(object): _type = 'release-build' def __init__(self, name, data): self.name = name assert 'type' in data, "Expected file type is '%s'" % ReleaseBuildFile._type assert data['type'] == ReleaseBuildFile._type, "Expected file type is '%s', not '%s'" % (ReleaseBuildFile._type, data['type']) assert 'version' in data, "Release build file for '%s' lacks required version information" % self.name assert int(data['version']) == 1, "Unable to handle '%s' format version '%d', please update rosdistro (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-rosdistro)" % (ReleaseBuildFile._type, int(data['version'])) self.version = int(data['version']) self.package_whitelist = [] if 'package_whitelist' in data: self.package_whitelist = data['package_whitelist'] assert isinstance(self.package_whitelist, list) self.package_blacklist = [] if 'package_blacklist' in data: self.package_blacklist = data['package_blacklist'] assert isinstance(self.package_blacklist, list) self.notify_emails = [] self.notify_maintainers = None self.notify_committers = None if 'notifications' in data: if 'emails' in data['notifications']: self.notify_emails = data['notifications']['emails'] assert isinstance(self.notify_emails, list) if 'maintainers' in data['notifications'] and data['notifications']['maintainers']: self.notify_maintainers = True if 'committers' in data['notifications'] and data['notifications']['committers']: self.notify_committers = True assert 'targets' in data self._targets = {} for os_name in data['targets'].keys(): if os_name == '_config': self._targets[os_name] = data['targets'][os_name] continue self._targets[os_name] = {} for os_code_name in data['targets'][os_name]: if os_code_name == '_config': self._targets[os_name][os_code_name] = data['targets'][os_name][os_code_name] continue self._targets[os_name][os_code_name] = {} for arch in data['targets'][os_name][os_code_name]: self._targets[os_name][os_code_name][arch] = data['targets'][os_name][os_code_name][arch] assert 'jenkins_url' in data self.jenkins_url = str(data['jenkins_url']) self.jenkins_sourcedeb_job_timeout = None if 'jenkins_sourcedeb_job_timeout' in data: self.jenkins_sourcedeb_job_timeout = int(data['jenkins_sourcedeb_job_timeout']) self.jenkins_binarydeb_job_timeout = None if 'jenkins_binarydeb_job_timeout' in data: self.jenkins_binarydeb_job_timeout = int(data['jenkins_binarydeb_job_timeout']) self.sync_package_count = None self.sync_packages = [] if 'sync' in data: if 'package_count' in data['sync']: self.sync_package_count = int(data['sync']['package_count']) if 'packages' in data['sync']: self.notify_maintainers = data['sync']['packages'] assert isinstance(self.sync_packages, list) def get_target_os_names(self): return [t for t in self._targets.keys() if t != '_config'] def get_target_os_code_names(self, os_name): os_code_names = self._targets[os_name] return [t for t in os_code_names.keys() if t != '_config'] def get_target_arches(self, os_name, os_code_name): arches = self._targets[os_name][os_code_name] return [t for t in arches.keys() if t != '_config'] def get_target_configuration(self, os_name=None, os_code_name=None, arch=None): assert os_code_name is not None or arch is None assert os_name is not None or os_code_name is None config = {} if '_config' in self._targets: config.update(self._targets['_config']) if os_name is not None: if '_config' in self._targets[os_name]: config.update(self._targets[os_name]['_config']) if os_code_name is not None: if '_config' in self._targets[os_name][os_code_name]: config.update(self._targets[os_name][os_code_name]['_config']) if arch is not None: if '_config' in self._targets[os_name][os_code_name][arch]: config.update(self._targets[os_name][os_code_name][arch]['_config']) return config def get_data(self): data = {} data['type'] = ReleaseBuildFile._type data['version'] = 1 if self.package_whitelist: data['package_whitelist'] = self.package_whitelist if self.package_blacklist: data['package_blacklist'] = self.package_blacklist if self.notify_emails or self.notify_maintainers or self.notify_committers: data['notifications'] = {} if self.notify_emails: data['notifications']['emails'] = self.notify_emails if self.notify_maintainers is not None: data['notifications']['maintainers'] = bool(self.notify_maintainers) if self.notify_committers is not None: data['notifications']['committers'] = bool(self.notify_committers) data['targets'] = self._targets data['jenkins_url'] = self.jenkins_url if self.jenkins_sourcedeb_job_timeout: data['jenkins_sourcedeb_job_timeout'] = self.jenkins_sourcedeb_job_timeout if self.jenkins_binarydeb_job_timeout: data['jenkins_binarydeb_job_timeout'] = self.jenkins_binarydeb_job_timeout if self.sync_package_count or self.sync_packages: data['sync'] = {} if self.sync_package_count is not None: data['sync']['package_count'] = self.sync_package_count if self.sync_packages: data['sync']['packages'] = self.sync_packages return data rosdistro-0.5.0/src/rosdistro/release_cache.py000066400000000000000000000116321300343010100214640ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function from .release_file import ReleaseFile class ReleaseCache(object): _type = 'cache' def __init__(self, name, data=None, distribution_file_data=None): assert data or distribution_file_data if data: assert 'type' in data, "Expected file type is '%s'" % ReleaseCache._type assert data['type'] == ReleaseCache._type, "Expected file type is '%s', not '%s'" % (ReleaseCache._type, data['type']) assert 'version' in data, "Release cache file for '%s' lacks required version information" % name self.version = int(data['version']) assert self.version > 1, "Unable to handle '%s' format version '%d' anymore, please update your '%s' file to version '2'" % (ReleaseCache._type, self.version, ReleaseCache._type) assert self.version == 2, "Unable to handle '%s' format version '%d', please update rosdistro (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-rosdistro)" % (ReleaseCache._type, self.version) assert 'name' in data, "Release cache file for '%s' lacks required name information" % name assert data['name'] == name, "Release cache file for '%s' does not match the name '%s'" % (name, data['name']) else: self.version = 2 self._distribution_file_data = data['distribution_file'] if data else distribution_file_data self.release_file = ReleaseFile(name, self._distribution_file_data) self.package_xmls = data['release_package_xmls'] if data else {} # for backward compatibility only def __getattr__(self, name): if name == 'release_package_xmls': return self.package_xmls raise AttributeError def get_data(self): data = {} data['type'] = 'cache' data['version'] = 2 data['name'] = self.release_file.name data['distribution_file'] = self._distribution_file_data data['package_xmls'] = self.package_xmls return data def update_distribution(self, distribution_file_data): # remove packages which are not in the old distribution file self._remove_obsolete_entries() self._distribution_file_data = distribution_file_data rel_file = ReleaseFile(self.distribution_file.name, self._distribution_file_data) # remove all package xmls if repository information has changed for pkg_name in sorted(rel_file.packages.keys()): if pkg_name not in self.release_file.packages: continue if pkg_name in self.package_xmls and self._get_repo_info(rel_file, pkg_name) != self._get_repo_info(self.release_file, pkg_name): del self.package_xmls[pkg_name] self.release_file = rel_file # remove packages which are not in the new distribution file self._remove_obsolete_entries() def _get_repo_info(self, dist_file, pkg_name): pkg = dist_file.packages[pkg_name] repo = dist_file.repositories[pkg.repository_name] return (repo.version, repo.url) def _remove_obsolete_entries(self): for pkg_name in self.package_xmls.keys(): if pkg_name not in self.release_file.packages: print('- REMOVE', pkg_name) del self.package_xmls[pkg_name] rosdistro-0.5.0/src/rosdistro/release_cache_generator.py000066400000000000000000000137431300343010100235370ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function import gzip import os import re import sys import yaml from catkin_pkg.package import InvalidPackage, parse_package_string from . import _get_dist_file_data, get_cached_release, get_index, get_release_cache from .release_cache import ReleaseCache def generate_release_caches(index, dist_names=None, preclean=False, debug=False): if os.path.isfile(index): index = 'file://' + os.path.abspath(index) index = get_index(index) if not dist_names: dist_names = sorted(index.distributions.keys()) errors = [] caches = {} for dist_name in dist_names: try: cache = generate_release_cache(index, dist_name, preclean, debug) except RuntimeError as e: errors.append(str(e)) continue caches[dist_name] = cache if errors: raise RuntimeError('\n'.join(errors)) return caches def generate_release_cache(index, dist_name, preclean=False, debug=False): dist, cache = _get_cached_release(index, dist_name, preclean) # fetch all manifests print('- fetch missing manifests') errors = [] for pkg_name in sorted(dist.packages.keys()): repo = dist.repositories[dist.packages[pkg_name].repository_name] if repo.version is None: if debug: print(' - skip "%s" since it has no version' % pkg_name) continue if debug: print(' - fetch "%s"' % pkg_name) else: sys.stdout.write('.') sys.stdout.flush() # check that package.xml is fetchable package_xml = dist.get_package_xml(pkg_name) if not package_xml: errors.append('%s: missing package.xml file for package "%s"' % (dist_name, pkg_name)) continue # check that package.xml is parseable try: pkg = parse_package_string(package_xml) except InvalidPackage as e: errors.append('%s: invalid package.xml file for package "%s"' % (dist_name, pkg_name)) continue # check that version numbers match (at least without deb inc) if not re.match('^%s-[\dA-z~\+\.]+$' % re.escape(pkg.version), repo.version): errors.append('%s: different version in package.xml (%s) for package "%s" than for the repository (%s) (after removing the debian increment)' % (dist_name, pkg.version, pkg_name, repo.version)) if not debug: print('') if errors: raise RuntimeError('\n'.join(errors)) return cache def _get_cached_release(index, dist_name, preclean=False): print('Build cache for "%s"' % dist_name) cache = None try: if not preclean: print('- trying to use local cache') yaml_str = None if os.path.exists('%s-cache.yaml.gz' % dist_name): print('- use local file "%s-cache.yaml.gz"' % dist_name) with gzip.open('%s-cache.yaml.gz' % dist_name, 'rb') as f: yaml_str = f.read() elif os.path.exists('%s-cache.yaml' % dist_name): print('- use local file "%s-cache.yaml"' % dist_name) with open('%s-cache.yaml' % dist_name, 'r') as f: yaml_str = f.read() if yaml_str is not None: data = yaml.load(yaml_str) cache = ReleaseCache(dist_name, data) if not cache: print('- trying to fetch cache') # get release cache cache = get_release_cache(index, dist_name) except: print('- failed to fetch old cache') if cache: print('- update cache') # get current release file rel_file_data = _get_dist_file_data(index, dist_name, 'release') # since format 2 of the index file might contain a single value rather then a list if not isinstance(rel_file_data, list): rel_file_data = [rel_file_data] # update cache with current release file cache.update_distribution(rel_file_data) else: print('- build cache from scratch') # get empty cache with distribution file distribution_file_data = _get_dist_file_data(index, dist_name, 'distribution') cache = ReleaseCache(dist_name, distribution_file_data=distribution_file_data) # get distribution return get_cached_release(index, dist_name, cache=cache, allow_lazy_load=True), cache rosdistro-0.5.0/src/rosdistro/release_file.py000066400000000000000000000114431300343010100213400ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from .package import Package from .release_repository_specification import ReleaseRepositorySpecification class ReleaseFile(object): _type = 'distribution' def __init__(self, name, data): self.name = name assert 'type' in data and data['type'] != 'release', "Unable to handle 'release' format anymore, please update your 'release' file to the latest specification" assert 'type' in data, "Expected file type is '%s'" % ReleaseFile._type assert data['type'] == ReleaseFile._type, "Expected file type is '%s', not '%s'" % (ReleaseFile._type, data['type']) assert 'version' in data, "Release file for '%s' lacks required version information" % self.name assert int(data['version']) == 1, "Unable to handle '%s' format version '%d', please update rosdistro (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-rosdistro)" % (ReleaseFile._type, int(data['version'])) self.version = int(data['version']) self.repositories = {} self.packages = {} if 'repositories' in data: for repo_name in sorted(data['repositories'].keys()): if 'release' not in data['repositories'][repo_name]: continue repo_status_data = data['repositories'][repo_name] repo_data = repo_status_data['release'] try: repo = ReleaseRepositorySpecification(repo_name, repo_data) # for backward compatibility only repo.status = repo_status_data.get('status', None) repo.status_description = repo_status_data.get('status_description', None) except AssertionError as e: e.args = [("Release file '%s': %s" % (self.name, a) if i == 0 else a) for i, a in enumerate(e.args)] raise e self.repositories[repo_name] = repo if repo.package_names: for pkg_name in repo.package_names: assert pkg_name not in self.packages, "Duplicate package name '%s' exists in repository '%s' as well as in repository '%s'" % (pkg_name, repo_name, self.packages[pkg_name].repository_name) self._add_package(pkg_name, repo, unary_repo=len(repo.package_names) == 1 and pkg_name == repo_name) else: # no package means a single package in the root of the repository self._add_package(repo_name, repo, {'subfolder': '.'}, True) self.platforms = {} if 'release_platforms' in data: for os_name in data['release_platforms'].keys(): self.platforms[os_name] = [] for os_code_name in data['release_platforms'][os_name]: assert os_code_name not in self.platforms[os_name], "Distribution '%s' specifies the os_code_name '%s' multiple times for the os_name '%s'" % (self.name, os_code_name, os_name) self.platforms[os_name].append(os_code_name) def _add_package(self, pkg_name, repo, unary_repo): assert pkg_name not in self.packages self.packages[pkg_name] = Package(pkg_name, repo.name) rosdistro-0.5.0/src/rosdistro/release_repository_specification.py000066400000000000000000000064511300343010100255430ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from .repository_specification import RepositorySpecification class ReleaseRepositorySpecification(RepositorySpecification): def __init__(self, name, data): super(ReleaseRepositorySpecification, self).__init__(name, data) assert self.type == 'git' self.tags = {} if self.version is not None: assert 'tags' in data assert 'release' in data['tags'] if 'tags' in data: for tag_type in data['tags']: tag_data = data['tags'][tag_type] self.tags[tag_type] = str(tag_data) self.package_names = [] if 'packages' in data and data['packages']: self.package_names = sorted(data['packages']) else: # no packages means a single package self.package_names = [self.name] # for backward compatibility only self.release_repository = self def get_release_tag(self, pkg_name): data = { 'package': pkg_name } if self.version: data['version'] = self.version data['upstream_version'] = self.version.split('-')[0] release_tag = self.tags['release'] for k, v in data.items(): release_tag = release_tag.replace('{%s}' % k, v) return release_tag def get_data(self): data = self._get_data(skip_git_type=True) if self.tags: data['tags'] = {} for tag in self.tags: data['tags'][tag] = str(self.tags[tag]) if len(self.package_names) > 1 or (len(self.package_names) == 1 and self.package_names[0] != self.name): data['packages'] = sorted(self.package_names) return data rosdistro-0.5.0/src/rosdistro/repository.py000066400000000000000000000104351300343010100211400ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from .doc_repository_specification import DocRepositorySpecification from .release_repository_specification import ReleaseRepositorySpecification from .source_repository_specification import SourceRepositorySpecification from .status import valid_statuses class Repository(object): def __init__(self, name, doc_data, release_data, source_data, status_data): self.name = name self.doc_repository = DocRepositorySpecification(self.name, doc_data) if doc_data else None self.release_repository = ReleaseRepositorySpecification(self.name, release_data) if release_data else None self.source_repository = SourceRepositorySpecification(self.name, source_data) if source_data else None self.status = status_data.get('status', None) if self.status is not None: assert self.status in valid_statuses self.status_description = status_data.get('status_description', None) self.status_per_package = {} for pkg_name in status_data.get('status_per_package', {}): data = {} status = status_data.get('status_per_package')[pkg_name].get('status', None) if status: assert status in valid_statuses data['status'] = status status_description = status_data.get('status_per_package')[pkg_name].get('status_description', None) if status_description: data['status_description'] = status_description if data: self.status_per_package[pkg_name] = data def get_data(self): data = {} if self.doc_repository: data['doc'] = self.doc_repository.get_data() if self.release_repository: data['release'] = self.release_repository.get_data() if self.source_repository: data['source'] = self.source_repository.get_data() if self.status: data['status'] = self.status if self.status_description: data['status_description'] = self.status_description if self.status_per_package: data['status_per_package'] = {} for pkg_name in sorted(self.status_per_package.keys()): pkg_status = self.status_per_package[pkg_name] pkg_data = {} status = pkg_status.get('status', None) if status: pkg_data['status'] = status status_description = pkg_status.get('status_description', None) if status_description: pkg_data['status_description'] = status_description if pkg_data: data['status_per_package'][pkg_name] = pkg_data return data rosdistro-0.5.0/src/rosdistro/repository_specification.py000066400000000000000000000072571300343010100240500ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. import re from .vcs import Git class RepositorySpecification(object): # Match groups are server and path on server. VCS_REGEX = re.compile('(?:https?:\/\/|ssh:\/\/|git:\/\/|git@)([\w.-]+)[:/]([\w/-]*)(?:\.git)?$') def __init__(self, name, data): self.name = name self.type = data.get('type', 'git') assert 'url' in data and data['url'], "Repository '%s' lacks required URL information" % name self.url = data['url'] self.version = data.get('version', None) self._remote_refs = None # for backward compatibility only self.status = None self.status_description = None def get_data(self): return self._get_data() def get_url_parts(self): """ Returns a tuple for the server and path, eg ('github.com', 'ros/catkin') """ match = self.VCS_REGEX.match(self.url) if not match: raise RuntimeError('VCS url "%s" does not match expected format.' % self.url) return match.groups() def has_remote_tag(self, tag): return tag in self.remote_tags @property def remote_refs(self): if not self._remote_refs: result = Git().command('ls-remote', self.url) if result['returncode'] != 0: raise RuntimeError('Could not git ls-remote repository "%s"' % self.url) self._remote_refs = {} for row in result['output'].strip().splitlines(): sha, name = row.split('\t') self._remote_refs[name] = sha return self._remote_refs @property def remote_tags(self): result = {} for name, sha in self.remote_refs.items(): if name.startswith('refs/tags/'): result[name.split('refs/tags/')[1]] = sha return result def _get_data(self, skip_git_type=False): data = {} if self.type != 'git' or not skip_git_type: data['type'] = str(self.type) data['url'] = str(self.url) if self.version is not None: data['version'] = self.version return data rosdistro-0.5.0/src/rosdistro/rosdistro.py000066400000000000000000000353031300343010100207520ustar00rootroot00000000000000import copy import os import sys import tarfile import tempfile import threading import urllib try: from urllib.request import urlopen from urllib.error import HTTPError except ImportError: from urllib2 import urlopen from urllib2 import HTTPError from .common import info from .common import warning from .common import error RES_DICT = {'build': [], 'buildtool': [], 'test': [], 'run': []} RES_TREE = {'build': {}, 'buildtool': {}, 'test': {}, 'run': {}} CACHE_VERSION = 1 walks = { 'FULL_WALK': {'build': ['build', 'run', 'buildtool', 'test'], 'run': ['build', 'run', 'buildtool', 'test'], 'buildtool': ['build', 'run', 'buildtool', 'test'], 'test': ['build', 'run', 'buildtool', 'test']}, 'SPIRAL_OF_DOOM': {'build': ['run'], 'run': ['buildtool'], 'buildtool': ['test'], 'test': ['build']} } def invert_dict(d): inverted = {} for key, value in d.iteritems(): for v in value: v_keys = inverted.setdefault(v, []) if key not in v_keys: v_keys.append(key) return inverted class RosDistro: def __init__(self, name, cache_location=None): self.depends_on1_cache = copy.deepcopy(RES_TREE) t1 = threading.Thread(target=self._construct_rosdistro_file, args=(name,)) t2 = threading.Thread(target=self._construct_rosdistro_dependencies, args=(name, cache_location,)) t1.start() t2.start() t1.join() t2.join() def _construct_rosdistro_file(self, name): self.distro_file = RosDistroFile(name) def _construct_rosdistro_dependencies(self, name, cache_location): self.depends_file = RosDependencies(name, cache_location) def get_repositories(self): return self.distro_file.repositories def get_repository(self, repo): return self.get_repositories()[repo] def get_packages(self): return self.distro_file.packages def get_package(self, pkg): return self.get_packages()[pkg] def get_rosinstall(self, items, version='last_release', source='vcs'): rosinstall = "" for p in self._convert_to_pkg_list(items): rosinstall += p.get_rosinstall(version, source, self.distro_file.name) return rosinstall def _get_depends_on1(self, package_name): if package_name in self.depends_on1_cache: return self.depends_on1_cache[package_name] res = copy.deepcopy(RES_DICT) for pkg in self.get_packages(): for key, depends in self._get_depends1(pkg).iteritems(): if package_name in depends: res[key].append(pkg) self.depends_on1_cache[package_name] = res return res def get_depends_on1(self, items): return self.get_depends_on(items, 1) def get_depends_on(self, items, depth=0, dep_dict=walks['FULL_WALK']): res = copy.deepcopy(RES_DICT) for p in self._convert_to_pkg_list(items): for dep_type, dep_list in res.iteritems(): self._get_depends_on_recursive(p.name, dep_type, invert_dict(dep_dict), dep_list, depth, 1) return res def _get_depends_on_recursive(self, package_name, dep_type, dep_dict, res, depth, curr_depth): deps_on = self._get_depends_on1(package_name) # merge and recurse for d in deps_on[dep_type]: if not d in res: res.append(d) if depth == 0 or curr_depth < depth: for next_dep_type in dep_dict[dep_type]: self._get_depends_on_recursive(d, next_dep_type, dep_dict, res, depth, curr_depth+1) def _get_depends1(self, package_name): p = self.distro_file.packages[package_name] return self.depends_file.get_dependencies(p, self.distro_file.name) def get_depends1(self, items): return self.get_depends(items, 1) def get_depends(self, items, depth=0, dep_dict=walks['FULL_WALK']): res = copy.deepcopy(RES_DICT) for p in self._convert_to_pkg_list(items): for dep_type, dep_list in res.iteritems(): self._get_depends_recursive(p.name, dep_type, dep_dict, dep_list, depth, 1) return res def _get_depends_recursive(self, package_name, dep_type, dep_dict, res, depth, curr_depth): deps1 = self._get_depends1(package_name) # merge and recurse for d in deps1[dep_type]: if not d in res: res.append(d) if depth == 0 or curr_depth < depth: if d in self.get_packages(): # recurse on packages only for next_dep_type in dep_dict[dep_type]: self._get_depends_recursive(d, next_dep_type, dep_dict, res, depth, curr_depth+1) def _convert_to_pkg_list(self, items): if type(items) != list: items = [items] pkgs = [] for i in items: if i in self.distro_file.repositories: for p in self.distro_file.repositories[i].packages: if not p in pkgs: pkgs.append(p) elif i in self.distro_file.packages: if not self.distro_file.packages[i] in pkgs: pkgs.append(self.distro_file.packages[i]) else: raise RuntimeError("!!! {0} is not a package name nor a repository name".format(i)) return pkgs class RosDistroFile: def __init__(self, name): self.packages = {} self.repositories = {} self.name = name # parse ros distro file distro_url = urlopen('https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' % name) distro = yaml.load(distro_url.read())['repositories'] # loop over all repo's for repo_name, data in distro.iteritems(): repo = RosRepository(repo_name, data['version'], data['url']) self.repositories[repo_name] = repo if 'packages' not in data: # support unary disto's data['packages'] = {repo_name: ''} # loop over all packages for pkg_name in data['packages'].keys(): pkg = RosPackage(pkg_name, repo) repo.packages.append(pkg) self.packages[pkg_name] = pkg class RosRepository: def __init__(self, name, version, url): self.name = name self.version = version self.url = url self.packages = [] def get_rosinstall(self, version, source): return "\n".join([p.get_rosinstall(version, source) for p in self.packages]) class RosPackage: def __init__(self, name, repository): self.name = name self.repository = repository self._package_xmls = {} self._release_tags = {} def _fetch_package_xml(self, rosdistro): repo = self.repository if 'github.com' in repo.url: url = repo.url upstream_version = repo.version.split('-')[0] release_tag = 'release/{0}/{1}'.format(self.name, upstream_version) url = url.replace('.git', '/{0}/package.xml'.format(release_tag)) url = url.replace('git://', 'https://') url = url.replace('https://', 'https://raw.') try: try: package_xml = urlopen(url).read() except Exception as e: msg = "Failed to read package.xml file from url '{0}': {1}".format(url, e) warning(msg) url = repo.url release_tag = 'release/{0}/{1}/{2}'.format(rosdistro, self.name, repo.version) tail = '/{0}/package.xml'.format(release_tag) url = url.replace('.git', tail) url = url.replace('git://', 'https://') url = url.replace('https://', 'https://raw.') info("Trying to read from url '{0}' instead".format(url)) package_xml = urlopen(url).read() except Exception as e: msg += '\nAND\n' msg += "Failed to read package.xml file from url '{0}': {1}".format(url, e) raise RuntimeError(msg) self._package_xmls[rosdistro] = package_xml self._release_tags[rosdistro] = release_tag return package_xml, release_tag else: raise Exception("Non-github repositories are net yet supported by the rosdistro tool") def get_package_xml(self, rosdistro): if rosdistro not in self._package_xmls: self._fetch_package_xml(rosdistro) return self._package_xmls[rosdistro] def get_release_tag(self, rosdistro): if rosdistro not in self._release_tags: self._fetch_package_xml(rosdistro) return self._release_tags[rosdistro] def get_rosinstall(self, version, source, rosdistro): # can't get last release of unreleased repository if version == 'last_release' and not self.repository.version: raise RuntimeError("Can't get the last release of unreleased repository {0}".format(self.repository.name)) # set specific version of last release of needed if version == 'last_release': version = self.repository.version.split('-')[0] # generate the rosinstall file release_tag = self.get_release_tag(rosdistro) if version == 'master': return yaml.dump([{ 'git': { 'local-name': self.name, 'uri': self.repository.url, 'version': '/'.join(release_tag.split('/')[:-1]) }}], default_style=False) else: if source == 'vcs': return yaml.safe_dump([{ 'git': { 'local-name': self.name, 'uri': self.repository.url, 'version': release_tag }}], default_style=False) elif source == 'tar': uri = self.repository.url uri = uri.replace('git://', 'https://') uri = uri.replace('.git', '/archive/{0}.tar.gz'.format(release_tag)) return yaml.safe_dump([{ 'tar': { 'local-name': self.name, 'uri': uri, 'version': '{0}-release-{1}'.format(self.repository.name, release_tag.replace('/', '-')) }}], default_style=False) else: raise RuntimeError("Invalid source type {0}".format(source)) class RosDependencies: def __init__(self, name, cache_location): # url's self.file_name = '%s-dependencies.yaml' % name if cache_location: self.local_url = os.path.join(cache_location, self.file_name) else: from rospkg import environment self.local_url = os.path.join(environment.get_ros_home(), self.file_name) self.server_url = 'http://www.ros.org/rosdistro/%s-dependencies.tar.gz' % name self.dependencies = {} # initialize with the local or server cache deps = self._read_local_cache() if deps == {}: deps = self._read_server_cache() for key, value in deps.iteritems(): self.dependencies[key] = value if self.cache == 'server': self._write_local_cache() def get_dependencies(self, package, rosdistro): repo = package.repository # support unreleased stacks if not repo.version: return copy.deepcopy(RES_DICT) key = '%s?%s?%s' % (repo.name, repo.version, package.name) # check in memory first if key in self.dependencies: return self.dependencies[key] # read server cache if needed if self.cache != 'server': deps = self._read_server_cache() for key, value in deps.iteritems(): self.dependencies[key] = value self._write_local_cache() if key in self.dependencies: return self.dependencies[key] # retrieve dependencies deps = retrieve_dependencies(package.get_package_xml(rosdistro)) self.dependencies[key] = deps self._write_local_cache() return deps def _read_server_cache(self): self.cache = 'server' try: resp = urlopen(self.server_url) except HTTPError as ex: warning("Failed to read server cache: %s" % ex) return {} with tempfile.NamedTemporaryFile('w') as fh: fh.write(resp.read()) fh.flush() tar = tarfile.open(fh.name, 'r') data = tar.extractfile(self.file_name) deps = yaml.load(data.read()) if not deps \ or not 'cache_version' in deps \ or deps['cache_version'] != CACHE_VERSION \ or not 'repositories' in deps: raise return deps['repositories'] def _read_local_cache(self): try: self.cache = 'local' with open(self.local_url) as f: deps = yaml.safe_load(f.read()) if not deps \ or not 'cache_version' in deps \ or deps['cache_version'] != CACHE_VERSION \ or not 'repositories' in deps: raise return deps['repositories'] except Exception: return {} def _write_local_cache(self): try: try: os.makedirs(os.path.dirname(self.local_url)) except: pass with open(self.local_url, 'w') as f: yaml.dump({'cache_version': CACHE_VERSION, 'repositories': self.dependencies}, f) except Exception as ex: error("Failed to write local dependency cache to %s: %s" % (self.local_url, ex)) def retrieve_dependencies(package_xml): try: return get_package_dependencies(package_xml) except Exception: raise RuntimeError("Failed to get dependencies from package_xml:\n```\n{0}\n```".format(package_xml)) def get_package_dependencies(package_xml): if not os.path.abspath("/usr/lib/pymodules/python2.7") in sys.path: sys.path.append("/usr/lib/pymodules/python2.7") from catkin_pkg import package as catkin_pkg pkg = catkin_pkg.parse_package_string(package_xml) depends1 = {'build': [d.name for d in pkg.build_depends], 'buildtool': [d.name for d in pkg.buildtool_depends], 'test': [d.name for d in pkg.test_depends], 'run': [d.name for d in pkg.run_depends]} return depends1 rosdistro-0.5.0/src/rosdistro/source_build_file.py000066400000000000000000000145051300343010100224010ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. import copy class SourceBuildFile(object): _type = 'source-build' def __init__(self, name, data): self.name = name assert 'type' in data, "Expected file type is '%s'" % SourceBuildFile._type assert data['type'] == SourceBuildFile._type, "Expected file type is '%s', not '%s'" % (SourceBuildFile._type, data['type']) assert 'version' in data, "Source build file for '%s' lacks required version information" % self.name assert int(data['version']) == 1, "Unable to handle '%s' format version '%d', please update rosdistro (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-rosdistro)" % (SourceBuildFile._type, int(data['version'])) self.version = int(data['version']) self.repository_whitelist = [] if 'repository_whitelist' in data: self.repository_whitelist = data['repository_whitelist'] assert isinstance(self.repository_whitelist, list) self.repository_blacklist = [] if 'repository_blacklist' in data: self.repository_blacklist = data['repository_blacklist'] assert isinstance(self.repository_blacklist, list) self.notify_emails = [] self.notify_maintainers = None self.notify_committers = None if 'notifications' in data: if 'emails' in data['notifications']: self.notify_emails = data['notifications']['emails'] assert isinstance(self.notify_emails, list) if 'maintainers' in data['notifications'] and data['notifications']['maintainers']: self.notify_maintainers = True if 'committers' in data['notifications'] and data['notifications']['committers']: self.notify_committers = True assert 'targets' in data self._targets = {} for os_name in data['targets'].keys(): if os_name == '_config': self._targets[os_name] = data['targets'][os_name] continue self._targets[os_name] = {} for os_code_name in data['targets'][os_name]: if os_code_name == '_config': self._targets[os_name][os_code_name] = data['targets'][os_name][os_code_name] continue self._targets[os_name][os_code_name] = {} for arch in data['targets'][os_name][os_code_name]: self._targets[os_name][os_code_name][arch] = data['targets'][os_name][os_code_name][arch] assert 'jenkins_url' in data self.jenkins_url = str(data['jenkins_url']) self.jenkins_job_timeout = None if 'jenkins_job_timeout' in data: self.jenkins_job_timeout = int(data['jenkins_job_timeout']) def filter_repositories(self, repos): res = copy.copy(set(repos)) if self.repository_whitelist: res &= set(self.repository_whitelist) res -= set(self.repository_blacklist) return res def get_target_os_names(self): return [t for t in self._targets.keys() if t != '_config'] def get_target_os_code_names(self, os_name): os_code_names = self._targets[os_name] return [t for t in os_code_names.keys() if t != '_config'] def get_target_arches(self, os_name, os_code_name): arches = self._targets[os_name][os_code_name] return [t for t in arches.keys() if t != '_config'] def get_target_configuration(self, os_name=None, os_code_name=None, arch=None): assert os_code_name is not None or arch is None assert os_name is not None or os_code_name is None arches = self._targets[os_name][os_code_name] return [t for t in arches.keys() if t != '_config'] def get_data(self): data = {} data['type'] = SourceBuildFile._type data['version'] = 1 if self.repository_whitelist: data['repository_whitelist'] = self.repository_whitelist if self.repository_blacklist: data['repository_blacklist'] = self.repository_blacklist if self.notify_emails or self.notify_maintainers or self.notify_committers: data['notifications'] = {} if self.notify_emails: data['notifications']['emails'] = self.notify_emails if self.notify_maintainers is not None: data['notifications']['maintainers'] = bool(self.notify_maintainers) if self.notify_committers is not None: data['notifications']['committers'] = bool(self.notify_committers) data['targets'] = self._targets data['jenkins_url'] = self.jenkins_url if self.jenkins_job_timeout: data['jenkins_job_timeout'] = self.jenkins_job_timeout return data rosdistro-0.5.0/src/rosdistro/source_file.py000066400000000000000000000064451300343010100212260ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from .repository_specification import RepositorySpecification class SourceFile(object): _type = 'distribution' def __init__(self, name, data): self.name = name assert 'type' in data and data['type'] != 'source', "Unable to handle 'source' format anymore, please update your 'source' file to the latest specification" assert 'type' in data, "Expected file type is '%s'" % SourceFile._type assert data['type'] == SourceFile._type, "Expected file type is '%s', not '%s'" % (SourceFile._type, data['type']) assert 'version' in data, "Source file for '%s' lacks required version information" % self.name assert int(data['version']) == 1, "Unable to handle '%s' format version '%d', please update rosdistro (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-rosdistro)" % (SourceFile._type, int(data['version'])) self.version = int(data['version']) self.repositories = {} if 'repositories' in data: for repo_name in sorted(data['repositories']): repo_data = data['repositories'][repo_name] if 'source' not in repo_data: continue repo_data = repo_data['source'] try: assert 'version' in repo_data, "Repository '%s' lacks required version information" % repo_name repo = RepositorySpecification(repo_name, repo_data) except AssertionError as e: e.args = [("Source file '%s': %s" % (self.name, a) if i == 0 else a) for i, a in enumerate(e.args)] raise e self.repositories[repo_name] = repo rosdistro-0.5.0/src/rosdistro/source_repository_specification.py000066400000000000000000000046411300343010100254220ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2014, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from .repository_specification import RepositorySpecification class SourceRepositorySpecification(RepositorySpecification): def __init__(self, name, data): super(SourceRepositorySpecification, self).__init__(name, data) self.test_commits = None if 'test_commits' in data: self.test_commits = bool(data['test_commits']) self.test_pull_requests = None if 'test_pull_requests' in data: self.test_pull_requests = bool(data['test_pull_requests']) def get_data(self): data = self._get_data(skip_git_type=False) if self.test_commits is not None: data['test_commits'] = self.test_commits if self.test_pull_requests is not None: data['test_pull_requests'] = self.test_pull_requests return data rosdistro-0.5.0/src/rosdistro/status.py000066400000000000000000000035521300343010100202460ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. STATUS_DEVELOPED = 'developed' STATUS_MAINTAINED = 'maintained' STATUS_UNMAINTAINED = 'unmaintained' STATUS_END_OF_LIFE = 'end-of-life' valid_statuses = [ STATUS_DEVELOPED, STATUS_MAINTAINED, STATUS_UNMAINTAINED, STATUS_END_OF_LIFE ] rosdistro-0.5.0/src/rosdistro/vcs.py000066400000000000000000000064721300343010100175220ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # Copyright (c) 2016, Clearpath Robotics # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from distutils.version import LooseVersion import os import re import subprocess class Git(object): _client_executable = None _client_version = None def __init__(self, cwd=None): self.cwd = cwd if not self._client_executable: self.__class__._client_executable = _find_executable('git') def command(self, *args): assert self._client_executable is not None, "'git' not found" return _run_command((self._client_executable,) + args, self.cwd) @classmethod def version_gte(cls, version): if not cls._client_version: result = cls().command('--version') cls._client_version = result['output'].split()[-1] return LooseVersion(cls._client_version) >= LooseVersion(version) def ref_is_hash(ref): return re.match('^[0-9a-f]{40}$', ref) != None def _run_command(cmd, cwd=None, env=None): result = {'cmd': ' '.join(cmd), 'cwd': cwd} try: proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) output, _ = proc.communicate() result['output'] = output.rstrip() result['returncode'] = proc.returncode except subprocess.CalledProcessError as e: result['output'] = e.output result['returncode'] = e.returncode if not isinstance(result['output'], str): result['output'] = result['output'].decode('utf-8') return result def _find_executable(file_name): for path in os.getenv('PATH').split(os.path.pathsep): file_path = os.path.join(path, file_name) if os.path.isfile(file_path) and os.access(file_path, os.X_OK): return file_path return None rosdistro-0.5.0/src/rosdistro/verify.py000066400000000000000000000142061300343010100202250ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function import difflib import sys import yaml from . import get_distribution_file, get_distribution_files, get_doc_build_files, get_doc_file, get_index, get_release_build_files, get_release_file, get_source_build_files, get_source_file from .loader import load_url def verify_files_parsable(index_url): return verify_files(index_url, _check_files_parsable, include_deprecated=False) def _check_files_parsable(index, dist_name, loader_function, _yaml_url, _file_type): try: loader_function(index, dist_name) except Exception as e: print(str(e), file=sys.stderr) return False return True def reformat_files(index_url): return verify_files(index_url, _reformat_files) def verify_files_identical(index_url): return verify_files(index_url, _check_files_identical) def verify_files(index_url, callback, include_deprecated=False): identical = True index = get_index(index_url) for dist_name in sorted(index.distributions.keys()): dist = index.distributions[dist_name] if index.version < 3: providers = [get_distribution_file] else: providers = [get_distribution_files] if include_deprecated: providers.extend([get_release_file, get_source_file, get_doc_file]) file_providers = { 'distribution': (providers, 'distribution'), } if index.version < 3: file_providers.update({ 'release_builds': (get_release_build_files, 'release-build'), 'source_builds': (get_source_build_files, 'source-build'), 'doc_builds': (get_doc_build_files, 'doc-build') }) for key in sorted(file_providers.keys()): provider, file_type = file_providers[key] yaml_url = dist[key] if isinstance(provider, list): for p in provider: identical &= callback(index, dist_name, p, yaml_url, file_type) else: identical &= callback(index, dist_name, provider, yaml_url, file_type) return identical def _reformat_files(index, dist_name, loader_function, yaml_url, file_type): all_identical = True files = loader_function(index, dist_name) if not isinstance(files, list): files = [files] yaml_url = [yaml_url] for i, f in enumerate(files): url = yaml_url[i] if not url.startswith('file://'): print('Skipping non-file url: %s' % url) continue identical = _check_file_identical(f, yaml_url[i], file_type) all_identical &= identical path = url[7:] if identical: print('Skipping identical file: %s' % path) continue print('Updating file: %s' % path) data = f.get_data() dist_file_data = _to_yaml(data) dist_file_data = '\n'.join(_yaml_header_lines(file_type, data['version'])) + '\n' + dist_file_data with open(path, 'w') as f: f.write(dist_file_data) return all_identical def _check_files_identical(index, dist_name, loader_function, yaml_url, file_type): identical = True files = loader_function(index, dist_name) if not isinstance(files, list): files = [files] yaml_url = [yaml_url] for i, f in enumerate(files): identical &= _check_file_identical(f, yaml_url[i], file_type) return identical def _check_file_identical(dist_file, yaml_url, file_type): yaml_str = load_url(yaml_url) yaml_lines = yaml_str.splitlines() dist_file_data = dist_file.get_data() dist_file_lines = _to_yaml(dist_file_data).splitlines() dist_file_lines[0:0] = _yaml_header_lines(file_type, dist_file_data['version']) if yaml_lines != dist_file_lines: diff = difflib.unified_diff( yaml_lines, dist_file_lines, yaml_url, 'loaded-and-saved', n=1, lineterm='') for line in diff: print(line, file=sys.stderr) return False return True def _to_yaml(data): yaml_str = yaml.dump(data, default_flow_style=False) yaml_str = yaml_str.replace(': null', ':') yaml_str = yaml_str.replace(': {}', ':') return yaml_str def _yaml_header_lines(file_type, version): rep = '141' if file_type == 'index' and version == 3: rep = '143' if file_type == 'distribution' and version == 2: rep = '143' return [ '%YAML 1.1', '# ROS %s file' % file_type, '# see REP %s: http://ros.org/reps/rep-0%s.html' % (rep, rep), '---' ] rosdistro-0.5.0/src/rosdistro/writer.py000066400000000000000000000037641300343010100202440ustar00rootroot00000000000000# Software License Agreement (BSD License) # # Copyright (c) 2013, Open Source Robotics Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # * Neither the name of Open Source Robotics Foundation, Inc. nor # the names of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function from .verify import _to_yaml, _yaml_header_lines def yaml_from_distribution_file(distribution_file): return '\n'.join(_yaml_header_lines(distribution_file._type, distribution_file.version)) + '\n' + _to_yaml(distribution_file.get_data()) # for backward compatibility only def yaml_from_release_file(release_file): raise NotImplementedError rosdistro-0.5.0/stdeb.cfg000077500000000000000000000006141300343010100153130ustar00rootroot00000000000000[DEFAULT] Depends: ca-certificates, python-argparse, python-catkin-pkg, python-rospkg, python-setuptools, python-yaml Depends3: ca-certificates, python3-catkin-pkg, python3-rospkg, python3-setuptools, python3-yaml Conflicts: python3-rosdistro Conflicts3: python-rosdistro Suite: oneiric precise quantal raring saucy trusty utopic vivid wily xenial yakkety wheezy jessie X-Python3-Version: >= 3.2 rosdistro-0.5.0/test/000077500000000000000000000000001300343010100145045ustar00rootroot00000000000000rosdistro-0.5.0/test/files/000077500000000000000000000000001300343010100156065ustar00rootroot00000000000000rosdistro-0.5.0/test/files/foo/000077500000000000000000000000001300343010100163715ustar00rootroot00000000000000rosdistro-0.5.0/test/files/foo/distribution.yaml000066400000000000000000000026751300343010100220060ustar00rootroot00000000000000%YAML 1.1 # ROS distribution file # see REP 141: http://ros.org/reps/rep-0141.html --- release_platforms: debian: - wheezy ubuntu: - precise - quantal - raring repositories: bar_repo: doc: type: git url: https://github.com/example-test/bar_repo.git version: master release: tags: release: release/foo/{package}/{version} url: https://github.com/example-release/bar_repo.git version: 0.1.2 source: type: git url: https://github.com/example-test/bar_repo.git version: master baz-repo: doc: blacklist_packages: - baz_pkg2 depends: - bar_repo type: hg url: https://bitbucket.org/baz-test/baz-repo version: default release: packages: - baz_pkg1 - baz_pkg2 tags: release: release/foo/{package}/{version} url: https://example.com/release/baz-repo.git version: 7.7.7-7 source: type: hg url: https://bitbucket.org/baz-test/baz-repo version: default status: maintained status_description: With a lot of passion status_per_package: baz_pkg2: status: end-of-life status_description: The package has not been updated since 1995 release_only_repo: release: tags: release: release/foo/{package}/{version} url: https://github.com/example-release/release_only_repo.git version: 1.2.3 type: distribution version: 1 rosdistro-0.5.0/test/files/foo/distribution2.yaml000066400000000000000000000002541300343010100220570ustar00rootroot00000000000000%YAML 1.1 # ROS distribution file # see REP 141: http://ros.org/reps/rep-0141.html --- release_platforms: ubuntu: - precise repositories: type: distribution version: 1 rosdistro-0.5.0/test/files/foo/distribution3.yaml000066400000000000000000000002531300343010100220570ustar00rootroot00000000000000%YAML 1.1 # ROS distribution file # see REP 141: http://ros.org/reps/rep-0141.html --- release_platforms: ubuntu: - trusty repositories: type: distribution version: 1 rosdistro-0.5.0/test/files/foo/doc-build.yaml000066400000000000000000000007511300343010100211220ustar00rootroot00000000000000%YAML 1.1 # ROS doc-build file # see REP 141: http://ros.org/reps/rep-0141.html --- doc_tag_index_repository: url: git@github.com/ros-infrastructure/rosdoc_tag_index version: master jenkins_job_timeout: 23 jenkins_url: http://farm.example.com:8080 notifications: emails: - admin@example.com maintainers: true targets: _config: apt_mirrors: - http://repo.example.com/ - http://archive.ubuntu.com/ubuntu ubuntu: precise: amd64: type: doc-build version: 1 rosdistro-0.5.0/test/files/foo/release-build.yaml000066400000000000000000000011401300343010100217660ustar00rootroot00000000000000%YAML 1.1 # ROS release-build file # see REP 141: http://ros.org/reps/rep-0141.html --- jenkins_binarydeb_job_timeout: 42 jenkins_sourcedeb_job_timeout: 5 jenkins_url: http://farm.example.com:8080 notifications: emails: - admin@example.com sync: package_count: 123 targets: _config: apt_target_repository: http://repo.example.com/ foo: bar ubuntu: _config: bar: baz precise: _config: ping: pong amd64: _config: foo: baz i386: quantal: amd64: i386: raring: amd64: i386: type: release-build version: 1 rosdistro-0.5.0/test/files/foo/release-cache.yaml000066400000000000000000000174371300343010100217520ustar00rootroot00000000000000distribution_file: release_platforms: debian: - wheezy ubuntu: - precise - quantal - raring repositories: bar_repo: doc: type: git url: https://github.com/example-test/bar_repo.git version: master release: tags: release: release/foo/{package}/{version} url: https://github.com/example-release/bar_repo.git version: 0.1.2 source: type: git url: https://github.com/example-test/bar_repo.git version: master baz-repo: doc: type: hg url: https://bitbucket.org/baz-test/baz-repo version: default release: packages: - baz_pkg1 - baz_pkg2 tags: release: release/foo/{package}/{version} url: https://example.com/release/baz-repo.git version: 7.7.7-7 source: type: hg url: https://bitbucket.org/baz-test/baz-repo version: default status: maintained status_description: With a lot of passion status_per_package: baz_pkg2: status: end-of-life status_description: The package has not been updated since 1995 type: distribution version: 1 name: foo release_package_xmls: {actionlib_msgs: "\n actionlib_msgs\n 1.9.14\n\ \ \n actionlib_msgs defines the common messages to interact\ \ with an\n action server and an action client. For full documentation of\n\ \ the actionlib API see\n the actionlib\n package.\n \n Tully Foote\n BSD\n\n http://ros.org/wiki/actionlib_msgs\n\ \ Vijay Pradeep\n\n catkin\n\ \n message_generation\n std_msgs\n\ \n message_generation \n message_runtime\n std_msgs\n\ \n", catkin: "\n\n catkin\n\ \ 0.5.63\n Low-level build system macros and\ \ infrastructure for ROS.\n Dirk Thomas\n BSD\n\n http://www.ros.org/wiki/catkin\n https://github.com/ros/catkin/issues\n\ \ https://github.com/ros/catkin\n\n Troy\ \ Straszheim\n Morten Kjaergaard\n Brian Gerkey\n\ \ Dirk Thomas\n\n gtest\n python-argparse\n\ \ python-catkin-pkg\n python-empy\n\ \ python-nose\n\n cmake\n\ \n cmake\n gtest\n python-argparse\n\ \ python-catkin-pkg\n python-empy\n\ \ python-nose\n\n \n \n \n\n", common_msgs: "\n common_msgs\n\ \ 1.9.14\n \n common_msgs contains messages\ \ that are widely used by other ROS packages.\n These includes messages for\n\ \ actions (actionlib_msgs),\n\ \ diagnostics (diagnostic_msgs),\n\ \ geometric primitives (geometry_msgs),\n\ \ robot navigation (nav_msgs),\n\ \ and common sensors (sensor_msgs),\ \ such as laser range finders, cameras, point clouds.\n \n Tully Foote\n BSD\n\ \n http://www.ros.org/wiki/common_msgs\n\n actionlib_msgs\n\ \ diagnostic_msgs\n geometry_msgs\n\ \ nav_msgs\n sensor_msgs\n\ \ shape_msgs\n stereo_msgs\n\ \ trajectory_msgs\n visualization_msgs\n\ \n \n \n \n\n", control: "\n\ \ control\n 1.1.6\n \n This\ \ package is essentially an alias for control_msgs, for backward compatibility.\ \ Do not depend on this package.\n \n Stuart Glaser\n Willow Garage\n\n BSD\n\n http://ros.org/wiki/control_msgs\n\ \n control_msgs\n\n control_msgs\n\ \n \n \n \n\n\n", control_msgs: "\n\ \ control_msgs\n 1.1.6\n \n \ \ control_msgs contains base messages and actions useful for\n controlling\ \ robots. It provides representations for controller\n setpoints and joint\ \ and cartesian trajectories.\n \n Stuart Glaser\n Willow Garage\n\n BSD\n\n http://ros.org/wiki/control_msgs\n\ \n catkin\n\n message_generation\n\ \ std_msgs\n trajectory_msgs\n\ \ geometry_msgs\n actionlib_msgs\n\ \n\n message_runtime\n std_msgs\n\ \ trajectory_msgs\n geometry_msgs\n\ \ actionlib_msgs\n\n\n", diagnostic_msgs: "\n\ \ diagnostic_msgs\n 1.9.14\n \n\ \ This package holds the diagnostic messages which provide the\n standardized\ \ interface for the diagnostic and runtime monitoring\n systems in ROS. These\ \ messages are currently used by\n the diagnostics\n Stack, which provides libraries for simple ways to set and\ \ access\n the messages, as well as automated ways to process the diagnostic\n\ \ data.\n\n These messages are used for long term logging and will not be\n\ \ changed unless there is a very important reason.\n \n Tully Foote\n BSD\n\ \n http://ros.org/wiki/diagnostic_msgs\n Tully Foote\n\n catkin\n\n message_generation\n\ \ std_msgs\n\n message_runtime\n\ \ std_msgs\n\n"} type: cache version: 2 rosdistro-0.5.0/test/files/foo/source-build.yaml000066400000000000000000000007401300343010100216530ustar00rootroot00000000000000%YAML 1.1 # ROS source-build file # see REP 141: http://ros.org/reps/rep-0141.html --- jenkins_job_timeout: 23 jenkins_url: http://farm.example.com:8080 notifications: emails: - admin@example.com maintainers: true targets: _config: apt_mirrors: - http://repo.example.com/ - http://archive.ubuntu.com/ubuntu ubuntu: precise: amd64: i386: quantal: amd64: i386: raring: amd64: i386: type: source-build version: 1 rosdistro-0.5.0/test/files/index_v2.yaml000066400000000000000000000005211300343010100202060ustar00rootroot00000000000000%YAML 1.1 # ROS index file # see REP 141: http://ros.org/reps/rep-0141.html --- distributions: foo: distribution: foo/distribution.yaml distribution_cache: foo/release-cache.yaml doc_builds: [foo/doc-build.yaml] release_builds: [foo/release-build.yaml] source_builds: [foo/source-build.yaml] type: index version: 2 rosdistro-0.5.0/test/files/index_v3.yaml000066400000000000000000000003561300343010100202150ustar00rootroot00000000000000%YAML 1.1 # ROS index file # see REP 143: http://ros.org/reps/rep-0143.html --- distributions: foo: distribution: [foo/distribution.yaml, foo/distribution2.yaml] distribution_cache: foo/release-cache.yaml type: index version: 3 rosdistro-0.5.0/test/files/index_v3_invalid.yaml000066400000000000000000000003561300343010100217230ustar00rootroot00000000000000%YAML 1.1 # ROS index file # see REP 143: http://ros.org/reps/rep-0143.html --- distributions: foo: distribution: [foo/distribution.yaml, foo/distribution3.yaml] distribution_cache: foo/release-cache.yaml type: index version: 3 rosdistro-0.5.0/test/test_cache.py000066400000000000000000000004631300343010100171630ustar00rootroot00000000000000import os from rosdistro import get_index, get_distribution_cache FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_get_release_cache(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) get_distribution_cache(i, 'foo') rosdistro-0.5.0/test/test_distribution.py000066400000000000000000000024711300343010100206400ustar00rootroot00000000000000import os import yaml from rosdistro import get_index, get_distribution_file, get_distribution_files from rosdistro.distribution_file import DistributionFile from rosdistro.loader import load_url FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_distribution_file(): url = 'file://' + FILES_DIR + '/foo/distribution.yaml' yaml_str = load_url(url) data = yaml.load(yaml_str) dist_file = DistributionFile('foo', data) _validate_dist_file(dist_file) def test_get_distribution_file(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) dist_file = get_distribution_file(i, 'foo') _validate_dist_file(dist_file) dist_files = get_distribution_files(i, 'foo') assert len(dist_files) == 1 def _validate_dist_file(dist_file): assert('bar_repo' in dist_file.repositories) repo = dist_file.repositories['bar_repo'] assert repo.release_repository.package_names == ['bar_repo'] assert 'bar_repo' in dist_file.release_packages assert'baz-repo' in dist_file.repositories repo = dist_file.repositories['baz-repo'] assert set(repo.release_repository.package_names) == set(['baz_pkg1', 'baz_pkg2']) assert 'baz_pkg1' in dist_file.release_packages assert 'baz_pkg2' in dist_file.release_packages rosdistro-0.5.0/test/test_doc.py000066400000000000000000000021221300343010100166570ustar00rootroot00000000000000import os import yaml from rosdistro import get_doc_file, get_index from rosdistro.doc_file import DocFile from rosdistro.loader import load_url FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_doc_file(): url = 'file://' + FILES_DIR + '/foo/distribution.yaml' yaml_str = load_url(url) data = yaml.load(yaml_str) doc_file = DocFile('foo', data) _validate_doc_file(doc_file) def test_get_doc_file(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) doc_file = get_doc_file(i, 'foo') _validate_doc_file(doc_file) def _validate_doc_file(doc_file): assert(set(['bar_repo', 'baz-repo']) == set(doc_file.repositories.keys())) repo = doc_file.repositories['bar_repo'] assert(repo.type == 'git') assert(repo.url == 'https://github.com/example-test/bar_repo.git') assert(repo.version == 'master') repo = doc_file.repositories['baz-repo'] assert(repo.type == 'hg') assert(repo.url == 'https://bitbucket.org/baz-test/baz-repo') assert(repo.version == 'default') rosdistro-0.5.0/test/test_doc_build.py000066400000000000000000000012571300343010100200460ustar00rootroot00000000000000import os import yaml from rosdistro import get_doc_build_files, get_index from rosdistro.doc_build_file import DocBuildFile from rosdistro.loader import load_url FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_doc_build_file(): url = 'file://' + FILES_DIR + '/foo/doc-build.yaml' yaml_str = load_url(url) data = yaml.load(yaml_str) DocBuildFile('foo', data) def test_get_doc_build_files(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) files = get_doc_build_files(i, 'foo') assert len(files) == 1 build_file = files[0] assert build_file.jenkins_job_timeout == 23 rosdistro-0.5.0/test/test_index.py000066400000000000000000000043011300343010100172220ustar00rootroot00000000000000import os from rosdistro import get_distribution_file from rosdistro import get_distribution_files from rosdistro import get_index from rosdistro import get_index_url FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_get_index_url(): get_index_url() def test_get_index_v2(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) assert len(i.distributions.keys()) == 1 assert 'foo' in i.distributions.keys() def test_get_index_v3(): url = 'file://' + FILES_DIR + '/index_v3.yaml' i = get_index(url) assert len(i.distributions.keys()) == 1 assert 'foo' in i.distributions.keys() dist_files = get_distribution_files(i, 'foo') assert len(dist_files) == 2 get_distribution_file(i, 'foo') def test_get_index_v3_invalid(): url = 'file://' + FILES_DIR + '/index_v3_invalid.yaml' i = get_index(url) dist_files = get_distribution_files(i, 'foo') assert len(dist_files) == 2 try: get_distribution_file(i, 'foo') assert False except AssertionError: pass def test_get_index_from_http_with_query_parameters(): import subprocess import sys import time url = 'http://localhost:9876/index_v3.yaml?raw&at=master' # start a http server and wait if sys.version_info < (3, 0, 0): proc = subprocess.Popen([sys.executable, '-m', 'SimpleHTTPServer', '9876'], cwd=FILES_DIR) else: proc = subprocess.Popen([sys.executable, '-m', 'http.server', '9876'], cwd=FILES_DIR) time.sleep(0.5) try: i = get_index(url) assert len(i.distributions.keys()) == 1 assert 'foo' in i.distributions.keys() # test if every url has the same queries for dist_urls in i.distributions['foo'].values(): if not isinstance(dist_urls, list): dist_urls = [dist_urls] for dist_url in dist_urls: assert dist_url.endswith('?raw&at=master') dist_files = get_distribution_files(i, 'foo') assert len(dist_files) == 2 get_distribution_file(i, 'foo') finally: proc.terminate() rosdistro-0.5.0/test/test_manifest_providers.py000066400000000000000000000074061300343010100220270ustar00rootroot00000000000000# -*- coding: utf-8 -*- import os from rosdistro.manifest_provider.bitbucket import bitbucket_manifest_provider from rosdistro.manifest_provider.cache import CachedManifestProvider, sanitize_xml from rosdistro.manifest_provider.git import git_manifest_provider, git_source_manifest_provider from rosdistro.manifest_provider.github import github_manifest_provider, github_source_manifest_provider from rosdistro.release_repository_specification import ReleaseRepositorySpecification from rosdistro.source_repository_specification import SourceRepositorySpecification import rosdistro.vcs def test_bitbucket(): assert '' in bitbucket_manifest_provider('indigo', _rospeex_release_repo(), 'rospeex_msgs') def test_cached(): class FakeDistributionCache(object): def __init__(self): self.release_package_xmls = {} dc = FakeDistributionCache() cache = CachedManifestProvider(dc, [github_manifest_provider]) assert '' in cache('kinetic', _genmsg_release_repo(), 'genmsg') def test_git(): assert '' in git_manifest_provider('kinetic', _genmsg_release_repo(), 'genmsg') def test_git_legacy(): rosdistro.vcs.Git._client_version = '1.7.0' assert '' in git_manifest_provider('kinetic', _genmsg_release_repo(), 'genmsg') rosdistro.vcs.Git._client_version = None def test_github(): assert '' in github_manifest_provider('kinetic', _genmsg_release_repo(), 'genmsg') def test_git_source(): repo_cache = git_source_manifest_provider(_genmsg_source_repo()) # This hash corresponds to the 0.5.7 tag. assert repo_cache['_ref'] == '81b66fe5eb00043c43894ddeee07e738d9b9712f' package_path, package_xml = repo_cache['genmsg'] assert '' == package_path assert '0.5.7' in package_xml def test_github_source(): repo_cache = github_source_manifest_provider(_genmsg_source_repo()) # This hash corresponds to the 0.5.7 tag. assert repo_cache['_ref'] == '81b66fe5eb00043c43894ddeee07e738d9b9712f' package_path, package_xml = repo_cache['genmsg'] assert '' == package_path assert '0.5.7' in package_xml def test_git_source_multi(): repo_cache = git_source_manifest_provider(_ros_source_repo()) assert repo_cache['_ref'] package_path, package_xml = repo_cache['roslib'] assert package_path == 'core/roslib' def test_github_source_multi(): repo_cache = github_source_manifest_provider(_ros_source_repo()) assert repo_cache['_ref'] package_path, package_xml = repo_cache['roslib'] assert package_path == 'core/roslib' def test_sanitize(): assert 'abc' in sanitize_xml('abc') assert 'ab c' in sanitize_xml(' ab c ') # This unicode check should be valid on both Python 2 and 3. assert 'français' in sanitize_xml(' français ') def _genmsg_release_repo(): return ReleaseRepositorySpecification('genmsg', { 'url': 'https://github.com/ros-gbp/genmsg-release.git', 'tags': {'release': 'release/kinetic/{package}/{version}'}, 'version': '0.5.7-1' }) def _genmsg_source_repo(): return SourceRepositorySpecification('genmsg', { 'url': 'https://github.com/ros/genmsg.git', 'version': '0.5.7' }) def _ros_source_repo(): return SourceRepositorySpecification('ros', { 'url': 'https://github.com/ros/ros.git', 'version': 'kinetic-devel' }) def _rospeex_release_repo(): return ReleaseRepositorySpecification('rospeex', { 'packages': ['rospeex', 'rospeex_msgs'], 'tags': {'release': 'release/indigo/{package}/{version}'}, 'url': 'https://bitbucket.org/rospeex/rospeex-release.git', 'version': '2.14.7-0' }) rosdistro-0.5.0/test/test_release.py000066400000000000000000000021361300343010100175370ustar00rootroot00000000000000import os import yaml from rosdistro import get_index, get_release_file from rosdistro.release_file import ReleaseFile from rosdistro.loader import load_url FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_release_file(): url = 'file://' + FILES_DIR + '/foo/distribution.yaml' yaml_str = load_url(url) data = yaml.load(yaml_str) rel_file = ReleaseFile('foo', data) _validate_rel_file(rel_file) def test_get_release_file(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) rel_file = get_release_file(i, 'foo') _validate_rel_file(rel_file) def _validate_rel_file(rel_file): assert('bar_repo' in rel_file.repositories) repo = rel_file.repositories['bar_repo'] assert repo.package_names == ['bar_repo'] assert 'bar_repo' in rel_file.packages assert'baz-repo' in rel_file.repositories repo = rel_file.repositories['baz-repo'] assert set(repo.package_names) == set(['baz_pkg1', 'baz_pkg2']) assert 'baz_pkg1' in rel_file.packages assert 'baz_pkg2' in rel_file.packages rosdistro-0.5.0/test/test_release_build.py000066400000000000000000000034631300343010100207220ustar00rootroot00000000000000import os import yaml from rosdistro import get_index, get_release, get_release_build_files, get_release_builds from rosdistro.loader import load_url from rosdistro.release_build_file import ReleaseBuildFile FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_release_build_file(): url = 'file://' + FILES_DIR + '/foo/release-build.yaml' yaml_str = load_url(url) data = yaml.load(yaml_str) ReleaseBuildFile('foo', data) def test_get_release_build_files(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) get_release_build_files(i, 'foo') def test_get_release_builds(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) d = get_release(i, 'foo') builds = get_release_builds(i, d) assert len(builds) == 1 build = builds[0] assert build.jenkins_sourcedeb_job_timeout == 5 assert build.jenkins_binarydeb_job_timeout == 42 os_names = build.get_target_os_names() assert set(os_names) == set(['ubuntu']) os_code_names = build.get_target_os_code_names('ubuntu') assert set(os_code_names) == set(['precise', 'quantal', 'raring']) arches = build.get_target_arches('ubuntu', 'precise') assert set(arches) == set(['amd64', 'i386']) c = build.get_target_configuration() assert len(c.keys()) == 2 assert set(c.keys()) == set(['apt_target_repository', 'foo']) assert c['apt_target_repository'] == 'http://repo.example.com/' assert c['foo'] == 'bar' c = build.get_target_configuration('ubuntu', 'precise') assert 'foo' in c.keys() assert c['foo'] == 'bar' assert 'ping' in c.keys() assert c['ping'] == 'pong' c = build.get_target_configuration('ubuntu', 'precise', 'amd64') assert 'foo' in c.keys() assert c['foo'] == 'baz' rosdistro-0.5.0/test/test_repository_specification.py000066400000000000000000000013741300343010100232410ustar00rootroot00000000000000from rosdistro.repository_specification import RepositorySpecification def test_repository_specification(): data = {'type': 'git', 'url': 'https://github.com/ros/catkin.git'} r = RepositorySpecification("test", data) assert r.get_data() == data assert r.version is None assert r.get_url_parts() == ('github.com', 'ros/catkin') r.url = 'http://github.com/ros/catkin' assert r.get_url_parts() == ('github.com', 'ros/catkin') r.url = 'ssh://example.com/a/b/c.git' assert r.get_url_parts() == ('example.com', 'a/b/c') r.url = 'git://example.com/a/b/c/d.git' assert r.get_url_parts() == ('example.com', 'a/b/c/d') r.url = 'git@example.com:a/b/c/d/e.git' assert r.get_url_parts() == ('example.com', 'a/b/c/d/e') rosdistro-0.5.0/test/test_source.py000066400000000000000000000021471300343010100174210ustar00rootroot00000000000000import os import yaml from rosdistro import get_index, get_source_file from rosdistro.loader import load_url from rosdistro.source_file import SourceFile FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_source_file(): url = 'file://' + FILES_DIR + '/foo/distribution.yaml' yaml_str = load_url(url) data = yaml.load(yaml_str) src_file = SourceFile('foo', data) _validate_src_file(src_file) def test_get_source_file(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) src_file = get_source_file(i, 'foo') _validate_src_file(src_file) def _validate_src_file(src_file): assert(set(['bar_repo', 'baz-repo']) == set(src_file.repositories.keys())) repo = src_file.repositories['bar_repo'] assert(repo.type == 'git') assert(repo.url == 'https://github.com/example-test/bar_repo.git') assert(repo.version == 'master') repo = src_file.repositories['baz-repo'] assert(repo.type == 'hg') assert(repo.url == 'https://bitbucket.org/baz-test/baz-repo') assert(repo.version == 'default') rosdistro-0.5.0/test/test_source_build.py000066400000000000000000000013071300343010100205750ustar00rootroot00000000000000import os import yaml from rosdistro import get_index, get_source_build_files from rosdistro.loader import load_url from rosdistro.source_build_file import SourceBuildFile FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_source_build_file(): url = 'file://' + FILES_DIR + '/foo/source-build.yaml' yaml_str = load_url(url) data = yaml.load(yaml_str) SourceBuildFile('foo', data) def test_get_source_build_files(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) files = get_source_build_files(i, 'foo') assert len(files) == 1 build_file = files[0] assert build_file.jenkins_job_timeout == 23 rosdistro-0.5.0/test/test_verify.py000066400000000000000000000007131300343010100174220ustar00rootroot00000000000000import os from rosdistro.verify import verify_files_identical, verify_files_parsable FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def test_verify_files_parsable(): index_url = 'file://' + FILES_DIR + '/index_v2.yaml' assert verify_files_parsable(index_url) def test_verify_files_identical(): index_url = 'file://' + FILES_DIR + '/index_v2.yaml' assert verify_files_identical(index_url) rosdistro-0.5.0/test/test_writer.py000066400000000000000000000024261300343010100174350ustar00rootroot00000000000000import difflib import os from rosdistro import get_distribution_file, get_index from rosdistro.writer import yaml_from_distribution_file FILES_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files')) def get_diff(expected, actual): udiff = difflib.unified_diff(expected.splitlines(), actual.splitlines(), fromfile='expected', tofile='actual') udiff_raw = '' for line in udiff: if line.startswith('@@'): udiff_raw += line if line.startswith('+'): if not line.startswith('+++'): line += '\n' udiff_raw += line if line.startswith('-'): if not line.startswith('---'): line += '\n' udiff_raw += line if line.startswith(' '): line += '\n' udiff_raw += line return '\n' + udiff_raw def test_verify_files_parsable(): url = 'file://' + FILES_DIR + '/index_v2.yaml' index = get_index(url) distribution_file = get_distribution_file(index, 'foo') data = yaml_from_distribution_file(distribution_file) with open(os.path.join(FILES_DIR, 'foo', 'distribution.yaml'), 'r') as f: expected = f.read() assert data == expected, get_diff(expected, data)