datalad-0.4.1/000077500000000000000000000000001301115270200130765ustar00rootroot00000000000000datalad-0.4.1/.coveragerc000066400000000000000000000000551301115270200152170ustar00rootroot00000000000000[paths] source = datalad/ */datalad/ datalad-0.4.1/.gitignore000066400000000000000000000003321301115270200150640ustar00rootroot00000000000000.pybuild/ datalad.build/ /.idea .coverage /venv* /cfgs /dist /fixtures /diagrams /build /docs/build /docs/source/generated /.tox /repos tmp .noseids *.egg-info *.py[coe] *.bak .#* .orig .rej .*.swp .travis.yml.evil-bd datalad-0.4.1/.mailmap000066400000000000000000000005301301115270200145150ustar00rootroot00000000000000Benjamin Poldrack Benjamin Poldrack Jason Gors Yaroslav Halchenko Yaroslav Halchenko Debanjum Singh Solanky Debanjum debanjum datalad-0.4.1/.travis.yml000066400000000000000000000136331301115270200152150ustar00rootroot00000000000000# vim ft=yaml # travis-ci.org definition for DataLad build language: python python: - 2.7 - 3.3 - 3.4 cache: - apt env: global: # will be used in the matrix, where neither other variable is used - BOTO_CONFIG=/tmp/nowhere - DATALAD_TESTS_SSH=1 # Special settings/helper for combined coverage from special remotes execution - COVERAGE=coverage matrix: - DATALAD_REPO_DIRECT=1 - DATALAD_REPO_DIRECT= # Disabled since old folks don't want to change workflows of submitting through central authority # - secure: "k2rHdTBjUU3pUUASqfRr7kHaaSmNKLLAR2f66A0fFSulih4CXxwLrR3g8/HP9m+jMve8mAYEiPSI7dT7cCm3WMA/piyLh2wKCGgzDD9oLjtvPAioR8dgLpzbgjxV/Vq6fwwPMlvbqqa+MmAImnAoSufEmI7zVQHCq11Hd5nd6Es=" # - secure: "Az7Pzo5pSdAFTTX6XXzE4VUS3wnlIe1vi/+bfHBzDjxstDvZVkPjPzaIs6v/BLod43AYBl1v9dyJR4qnBnaVrUDLB3tC0emLhJ2qnw+8GKHSSImCwIOeZzg9QpXeVQovZUqQVQ3fg3KIWCIzhmJ59EbMQcI4krNDxk4WcXmyVfk=" matrix: include: # Additional custom ones - python: 2.7 # By default no logs will be output. This one is to test with log output at INFO level env: - _DATALAD_UPSTREAM_GITPYTHON=1 - python: 2.7 # By default no logs will be output. This one is to test with log output at INFO level env: - DATALAD_LOG_LEVEL=INFO - DATALAD_LOG_TRACEBACK=1 # just a smoke test for now - python: 2.7 # By default no logs will be output. This one is to test with low level but dumped to /dev/null env: - DATALAD_LOG_LEVEL=2 - DATALAD_LOG_TARGET=/dev/null - DATALAD_LOG_TRACEBACK=collide # just a smoke test for now - DATALAD_API_ALWAYSRENDER=1 - DATALAD_TESTS_PROTOCOLREMOTE=1 - DATALAD_TESTS_DATALADREMOTE=1 - DATALAD_LOG_OUTPUTS=1 - python: 2.7 env: # to test operation under root since also would consider FS "crippled" due to # ability to rewrite R/O files - NOSE_WRAPPER="sudo -E" # no key authentication for root: - DATALAD_TESTS_SSH=0 - python: 2.7 env: - DATALAD_TESTS_NONETWORK=1 # must operate nicely with those env variables set - http_proxy= - https_proxy= - python: 2.7 env: - PYTHONPATH=$PWD/tools/testing/bad_internals/_scrapy/ - python: 2.7 # To make sure that operates correctly whenever dataset directories have symlinks # in their path. env: # Eventually we will get there, but atm causes a good number of failures # - TMPDIR="/tmp/sym ссылка" - TMPDIR="/tmp/sym link" - python: 2.7 # apparently moving symlink outside has different effects on abspath # see https://github.com/datalad/datalad/issues/878 env: - TMPDIR="/var/tmp/sym link" - python: 2.7 # To make sure that operates correctly whenever dataset directories have symlinks # in their path. env: # To make orthogonal test where it is not a symlink but a dir with spaces - TMPDIR="/tmp/d i r" allow_failures: # For now, don't fail entirely when direct mode fails - env: DATALAD_REPO_DIRECT=1 # Causes complete laptop or travis instance crash atm, but survives in a docker # need to figure it out (looks like some PID explosion) # - python: 3.5 # # we would need to migrate to boto3 to test it fully, but SSH should work # env: # - DATALAD_TESTS_SSH=1 # - UNSET_S3_SECRETS=1 before_install: # The ultimate one-liner setup for NeuroDebian repository - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) - travis_retry sudo apt-get update -qq - sudo tools/ci/prep-travis-forssh-sudo.sh - tools/ci/prep-travis-forssh.sh # Install grunt-cli - npm install grunt-cli # Install optionally upstream current development so we are sure that they break nothing important for us - if [ ! -z "${_DATALAD_UPSTREAM_GITPYTHON:-}" ]; then pip install https://github.com/gitpython-developers/GitPython/archive/master.zip; fi install: # Install standalone build of git-annex for the recent enough version - travis_retry sudo apt-get install git-annex-standalone zip pandoc - travis_retry sudo apt-get install shunit2 - git config --global user.email "test@travis.land" - git config --global user.name "Travis Almighty" - cd ..; pip install -q codecov; cd - - pip install -r requirements-devel.txt # So we could test under sudo -E with PATH pointing to installed location - sudo sed -i -e 's/^Defaults.*secure_path.*$//' /etc/sudoers # TMPDIRs - if [[ "${TMPDIR:-}" =~ .*/sym\ link ]]; then echo "Symlinking $TMPDIR"; ln -s /tmp "$TMPDIR"; fi - if [[ "${TMPDIR:-}" =~ .*/d\ i\ r ]]; then echo "mkdir $TMPDIR"; mkdir -p "$TMPDIR"; fi # S3 - if [ ! -z "$UNSET_S3_SECRETS" ]; then echo "usetting"; unset DATALAD_datalad_test_s3_key_id DATALAD_datalad_test_s3_secret_id; fi # Install grunt to test run javascript frontend tests - npm install grunt - npm install grunt-contrib-qunit script: # Verify that setup.py build doesn't puke - python setup.py build # Run tests - PATH=$PWD/tools/coverage-bin:$PATH $NOSE_WRAPPER `which nosetests` -s -v --with-doctest --doctest-tests --with-cov --cover-package datalad --logging-level=INFO # Generate documentation and run doctests - PYTHONPATH=$PWD make -C docs html doctest # Run javascript tests - grunt test --verbose # do not run them with custom TMPDIR with spaces for now : https://github.com/datalad/datalad/issues/893 - if [[ ${TMPDIR:-} =~ ".* .*" ]]; then tools/testing/run_doc_examples; else echo "not running since has spaces"; fi after_success: - coverage combine -a /tmp/.coverage-entrypoints-* - codecov # makes it only more difficult to comprehend the failing output. Enable only when necessary # for a particular debugging #after_failure: # - if [ ! -z "$DATALAD_TESTS_NONETWORK" ]; then sudo route add -net 0.0.0.0 netmask 0.0.0.0 dev lo; fi # - DATALAD_LOG_LEVEL=DEBUG $NOSE_WRAPPER `which nosetests` -s -v --with-doctest --doctest-tests --with-cov --cover-package datalad --logging-level=DEBUG # - if [ ! -z "$DATALAD_TESTS_NONETWORK" ]; then sudo route del -net 0.0.0.0 netmask 0.0.0.0 dev lo; fi datalad-0.4.1/CHANGELOG.md000066400000000000000000000160641301115270200147160ustar00rootroot00000000000000 ____ _ _ _ | _ \ __ _ | |_ __ _ | | __ _ __| | | | | | / _` | | __| / _` | | | / _` | / _` | | |_| | | (_| | | |_ | (_| | | |___ | (_| | | (_| | |____/ \__,_| \__| \__,_| |_____| \__,_| \__,_| Change Log This is a very high level and scarce summary of the changes between releases. We would recommend to consult log of the [DataLad git repository](http://github.com/datalad/datalad) for more details ATM. ## 0.4.1 (Nov 10, 2016) -- CA release Requires now GitPython >= 2.1.0 ### Fixes - [save] - to not save staged files if explicit paths were provided - improved (but not yet complete) support for direct mode - [update] to not crash if some sub-datasets are not installed - do not log calls to `git config` to avoid leakage of possibly sensitive settings to the logs ### Enhancements and new features - New [rfc822-compliant metadata] format - [save] - -S to save the change also within all super-datasets - [add] now has progress-bar reporting - [create-sibling-github] to create a :term:`sibling` of a dataset on github - [OpenfMRI] crawler and datasets were enriched with URLs to separate files where also available from openfmri s3 bucket (if upgrading your datalad datasets, you might need to run `git annex enableremote datalad` to make them available) - various enhancements to log messages - web interface - populates "install" box first thus making UX better over slower connections ## 0.4 (Oct 22, 2016) -- Paris is waiting Primarily it is a bugfix release but because of significant refactoring of the [install] and [get] implementation, it gets a new minor release. ### Fixes - be able to [get] or [install] while providing paths while being outside of a dataset - remote annex datasets get properly initialized - robust detection of outdated [git-annex] ### Enhancements and new features - interface changes - [get] `--recursion-limit=existing` to not recurse into not-installed subdatasets - [get] `-n` to possibly install sub-datasets without getting any data - [install] `--jobs|-J` to specify number of parallel jobs for annex [get] call could use (ATM would not work when data comes from archives) - more (unit-)testing - documentation: see http://docs.datalad.org/en/latest/basics.html for basic principles and useful shortcuts in referring to datasets - various webface improvements: breadcrumb paths, instructions how to install dataset, show version from the tags, etc. ## 0.3.1 (Oct 1, 2016) -- what a wonderful week Primarily bugfixes but also a number of enhancements and core refactorings ### Fixes - do not build manpages and examples during installation to avoid problems with possibly previously outdated dependencies - [install] can be called on already installed dataset (with `-r` or `-g`) ### Enhancements and new features - complete overhaul of datalad configuration settings handling (see [Configuration documentation]), so majority of the environment. Now uses git format and stores persistent configuration settings under `.datalad/config` and local within `.git/config` variables we have used were renamed to match configuration names - [create-sibling] does not now by default upload web front-end - [export] command with a plug-in interface and `tarball` plugin to export datasets - in Python, `.api` functions with rendering of results in command line got a _-suffixed sibling, which would render results as well in Python as well (e.g., using `search_` instead of `search` would also render results, not only output them back as Python objects) - [get] - `--jobs` option (passed to `annex get`) for parallel downloads - total and per-download (with git-annex >= 6.20160923) progress bars (note that if content is to be obtained from an archive, no progress will be reported yet) - [install] `--reckless` mode option - [search] - highlights locations and fieldmaps for better readability - supports `-d^` or `-d///` to point to top-most or centrally installed meta-datasets - "complete" paths to the datasets are reported now - `-s` option to specify which fields (only) to search - various enhancements and small fixes to [meta-data] handling, [ls], custom remotes, code-base formatting, downloaders, etc - completely switched to `tqdm` library (`progressbar` is no longer used/supported) ## 0.3 (Sep 23, 2016) -- winter is coming Lots of everything, including but not limited to - enhanced index viewer, as the one on http://datasets.datalad.org - initial new data providers support: [Kaggle], [BALSA], [NDA], [NITRC] - initial [meta-data support and management] - new and/or improved crawler pipelines for [BALSA], [CRCNS], [OpenfMRI] - refactored [install] command, now with separate [get] - some other commands renaming/refactoring (e.g., [create-sibling]) - datalad [search] would give you an option to install datalad's super-dataset under ~/datalad if ran outside of a dataset ### 0.2.3 (Jun 28, 2016) -- busy OHBM New features and bugfix release - support of /// urls to point to http://datasets.datalad.org - variety of fixes and enhancements throughout ### 0.2.2 (Jun 20, 2016) -- OHBM we are coming! New feature and bugfix release - greately improved documentation - publish command API RFing allows for custom options to annex, and uses --to REMOTE for consistent with annex invocation - variety of fixes and enhancements throughout ### 0.2.1 (Jun 10, 2016) - variety of fixes and enhancements throughout ## 0.2 (May 20, 2016) Major RFing to switch from relying on rdf to git native submodules etc ## 0.1 (Oct 14, 2015) Release primarily focusing on interface functionality including initial publishing [Kaggle]: https://www.kaggle.com [BALSA]: http://balsa.wustl.edu [NDA]: http://data-archive.nimh.nih.gov [NITRC]: https://www.nitrc.org [CRCNS]: http://crcns.org [FCON1000]: http://fcon_1000.projects.nitrc.org [OpenfMRI]: http://openfmri.org [git-annex]: http://git-annex.branchable.com/ [meta-data support and management]: http://docs.datalad.org/en/latest/cmdline.html#meta-data-handling [meta-data]: http://docs.datalad.org/en/latest/cmdline.html#meta-data-handling [install]: http://datalad.readthedocs.io/en/latest/generated/man/datalad-install.html [ls]: http://datalad.readthedocs.io/en/latest/generated/man/datalad-ls.html [save]: http://datalad.readthedocs.io/en/latest/generated/man/datalad-save.html [get]: http://datalad.readthedocs.io/en/latest/generated/man/datalad-get.html [create-sibling]: http://datalad.readthedocs.io/en/latest/generated/man/datalad-create-sibling.html [create-sibling-github]: http://datalad.readthedocs.io/en/latest/generated/man/datalad-create-sibling-github.html [search]: http://datalad.readthedocs.io/en/latest/generated/man/datalad-search.html [export]: http://datalad.readthedocs.io/en/latest/generated/man/datalad-export.html [Configuration documentation]: http://docs.datalad.org/config.html [rfc822-compliant metadata]: http://docs.datalad.org/en/latest/metadata.html#rfc822-compliant-meta-datadatalad-0.4.1/CONTRIBUTING.md000066400000000000000000000355031301115270200153350ustar00rootroot00000000000000Contributing to DataLad ======================= [gh-datalad]: http://github.com/datalad/datalad Files organization ------------------ - `datalad/` is the main Python module where major development is happening, with major submodules being: - `cmdline/` - helpers for accessing `interface/` functionality from command line - `crawler/` - functionality for crawling (online) resources and creating or updating datasets and collections based on the scraped/downloaded data - `nodes/` processing elements which are used in the pipeline - `pipelines/` pipelines generators, to produce pipelines to be ran - `pipeline.py` pipeline runner - `customremotes/` - custom special remotes for annex provided by datalad - `downloaders/` - support for accessing data from various sources (e.g. http, S3, XNAT) via a unified interface. - `configs/` - specifications for known data providers and associated credentials - `interface/` - high level interface functions which get exposed via command line (`cmdline/`) or Python (`datalad.api`). - `tests/` - some unit- and regression- tests (more could be found under `tests/` of corresponding submodules) - `utils.py` provides convenience helpers used by unit-tests such as `@with_tree`, `@serve_path_via_http` and other decorators - `ui/` - user-level interactions, such as messages about errors, warnings, progress reports, AND when supported by available frontend -- interactive dialogs - `support/` - various support modules, e.g. for git/git-annex interfaces, constraints for the `interface/`, etc - `docs/` - yet to be heavily populated documentation - `bash-completions` - bash and zsh completion setup for datalad (just `source` it) - `fixtures/` currently not under git, contains generated by vcr fixtures - `tools/` contains helper utilities used during development, testing, and benchmarking of DataLad. Implemented in any most appropriate language (Python, bash, etc.) How to contribute ----------------- The preferred way to contribute to the DataLad code base is to fork the [main repository][gh-datalad] on GitHub. Here we outline the workflow used by the developers: 0. Have a clone of our main [project repository][gh-datalad] as `origin` remote in your git: git clone git://github.com/datalad/datalad 1. Fork the [project repository][gh-datalad]: click on the 'Fork' button near the top of the page. This creates a copy of the code base under your account on the GitHub server. 2. Add your forked clone as a remote to the local clone you already have on your local disk: git remote add gh-YourLogin git@github.com:YourLogin/datalad.git git fetch gh-YourLogin To ease addition of other github repositories as remotes, here is a little bash function/script to add to your `~/.bashrc`: ghremote () { url="$1" proj=${url##*/} url_=${url%/*} login=${url_##*/} git remote add gh-$login $url git fetch gh-$login } thus you could simply run: ghremote git@github.com:YourLogin/datalad.git to add the above `gh-YourLogin` remote. Additional handy aliases such as `ghpr` (to fetch existing pr from someone's remote) and `ghsendpr` could be found at [yarikoptic's bash config file](http://git.onerussian.com/?p=etc/bash.git;a=blob;f=.bash/bashrc/30_aliases_sh;hb=HEAD#l865) 3. Create a branch (generally off the `origin/master`) to hold your changes: git checkout -b nf-my-feature and start making changes. Ideally, use a prefix signaling the purpose of the branch - `nf-` for new features - `bf-` for bug fixes - `rf-` for refactoring - `doc-` for documentation contributions (including in the code docstrings). We recommend to not work in the ``master`` branch! 4. Work on this copy on your computer using Git to do the version control. When you're done editing, do: git add modified_files git commit to record your changes in Git. Ideally, prefix your commit messages with the `NF`, `BF`, `RF`, `DOC` similar to the branch name prefixes, but you could also use `TST` for commits concerned solely with tests, and `BK` to signal that the commit causes a breakage (e.g. of tests) at that point. Multiple entries could be listed joined with a `+` (e.g. `rf+doc-`). See `git log` for examples. If a commit closes an existing DataLad issue, then add to the end of the message `(Closes #ISSUE_NUMER)` 5. Push to GitHub with: git push -u gh-YourLogin nf-my-feature Finally, go to the web page of your fork of the DataLad repo, and click 'Pull request' (PR) to send your changes to the maintainers for review. This will send an email to the committers. You can commit new changes to this branch and keep pushing to your remote -- github automagically adds them to your previously opened PR. (If any of the above seems like magic to you, then look up the [Git documentation](http://git-scm.com/documentation) on the web.) Development environment ----------------------- Although we now support Python 3 (>= 3.3), primarily we still use Python 2.7 and thus instructions below are for python 2.7 deployments. Replace `python-{` with `python{,3}-{` to also install dependencies for Python 3 (e.g., if you would like to develop and test through tox). See [README.md:Dependencies](README.md#Dependencies) for basic information about installation of datalad itself. On Debian-based systems we recommend to enable [NeuroDebian](http://neuro.debian.net) since we use it to provide backports of recent fixed external modules we depend upon: ```sh apt-get install -y -q git git-annex-standalone apt-get install -y -q patool python-scrapy python-{appdirs,argcomplete,git,humanize,keyring,lxml,msgpack,mock,progressbar,requests,setuptools,six} ``` and additionally, for development we suggest to use tox and new versions of dependencies from pypy: ```sh apt-get install -y -q python-{dev,httpretty,nose,pip,vcr,virtualenv} python-tox # Some libraries which might be needed for installing via pip apt-get install -y -q lib{ffi,ssl,curl4-openssl,xml2,xslt1}-dev ``` some of which you could also install from PyPi using pip (prior installation of those libraries listed above might be necessary) ```sh pip install -r requirements-devel.txt ``` and you will need to install recent git-annex using appropriate for your OS means (for Debian/Ubuntu, once again, just use NeuroDebian). Documentation ------------- ### Docstrings We use [NumPy standard] for the description of parameters docstrings. If you are using PyCharm, set your project settings (`Tools` -> `Python integrated tools` -> `Docstring format`). [NumPy standard]: https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt#docstring-standard In addition, we follow the guidelines of [Restructured Text] with the additional features and treatments provided by [Sphinx]. [Restructured Text]: http://docutils.sourceforge.net/docs/user/rst/quickstart.html [Sphinx]: http://www.sphinx-doc.org/en/stable/ Additional Hints ---------------- 1. For merge commits to have more informative description, add to your `.git/config` or `~/.gitconfig` following section: [merge] summary = true log = true and if conflicts occur, provide short summary on how they were resolved in "Conflicts" listing within the merge commit (see [example](https://github.com/datalad/datalad/commit/eb062a8009d160ae51929998771964738636dcc2)). Quality Assurance ----------------- It is recommended to check that your contribution complies with the following rules before submitting a pull request: - All public methods should have informative docstrings with sample usage presented as doctests when appropriate. - All other tests pass when everything is rebuilt from scratch. - New code should be accompanied by tests. ### Tests `datalad/tests` contains tests for the core portion of the project, and more tests are provided under corresponding submodules in `tests/` subdirectories to simplify re-running the tests concerning that portion of the codebase. To execute many tests, the codebase first needs to be "installed" in order to generate scripts for the entry points. For that, the recommended course of action is to use `virtualenv`, e.g. ```sh virtualenv --system-site-packages venv-tests source venv-tests/bin/activate pip install -r requirements.txt python setup.py develop ``` and then use that virtual environment to run the tests, via ```sh python -m nose -s -v datalad ``` or similarly, ```sh nosetests -s -v datalad ``` then to later deactivate the virtualenv just simply enter ```sh deactivate ``` Alternatively, or complimentary to that, you can use `tox` -- there is a `tox.ini` file which sets up a few virtual environments for testing locally, which you can later reuse like any other regular virtualenv for troubleshooting. Additionally, [tools/testing/test_README_in_docker](tools/testing/test_README_in_docker) script can be used to establish a clean docker environment (based on any NeuroDebian-supported release of Debian or Ubuntu) with all dependencies listed in README.md pre-installed. ### Coverage You can also check for common programming errors with the following tools: - Code with good unittest coverage (at least 80%), check with: pip install nose coverage nosetests --with-coverage path/to/tests_for_package - We rely on https://codecov.io to provide convenient view of code coverage. Installation of the codecov extension for Firefox/Iceweasel or Chromium is strongly advised, since it provides coverage annotation of pull requests. ### Linting We are not (yet) fully PEP8 compliant, so please use these tools as guidelines for your contributions, but not to PEP8 entire code base. [beyond-pep8]: https://www.youtube.com/watch?v=wf-BqAjZb8M *Sidenote*: watch [Raymond Hettinger - Beyond PEP 8][beyond-pep8] - No pyflakes warnings, check with: pip install pyflakes pyflakes path/to/module.py - No PEP8 warnings, check with: pip install pep8 pep8 path/to/module.py - AutoPEP8 can help you fix some of the easy redundant errors: pip install autopep8 autopep8 path/to/pep8.py Also, some team developers use [PyCharm community edition](https://www.jetbrains.com/pycharm) which provides built-in PEP8 checker and handy tools such as smart splits/joins making it easier to maintain code following the PEP8 recommendations. NeuroDebian provides `pycharm-community-sloppy` package to ease pycharm installation even further. Easy Issues ----------- A great way to start contributing to DataLad is to pick an item from the list of [Easy issues](https://github.com/datalad/datalad/labels/easy) in the issue tracker. Resolving these issues allows you to start contributing to the project without much prior knowledge. Your assistance in this area will be greatly appreciated by the more experienced developers as it helps free up their time to concentrate on other issues. Various hints for developers ---------------------------- ### Useful tools - While performing IO/net heavy operations use [dstat](http://dag.wieers.com/home-made/dstat) for quick logging of various health stats in a separate terminal window: dstat -c --top-cpu -d --top-bio --top-latency --net - To monitor speed of any data pipelining [pv](http://www.ivarch.com/programs/pv.shtml) is really handy, just plug it in the middle of your pipe. - For remote debugging epdb could be used (avail in pip) by using `import epdb; epdb.serve()` in Python code and then connecting to it with `python -c "import epdb; epdb.connect()".` - We are using codecov which has extensions for the popular browsers (Firefox, Chrome) which annotates pull requests on github regarding changed coverage. ### Useful Environment Variables Refer datalad/config.py for information on how to add these environment variables to the config file and their naming convention - *DATALAD_LOG_LEVEL*: Used for control the verbosity of logs printed to stdout while running datalad commands/debugging - *DATALAD_LOG_OUTPUTS*: Used to control either both stdout and stderr of external commands execution are logged in detail (at DEBUG level) - *DATALAD_LOG_TIMESTAMP*: Used to add timestamp to datalad logs - *DATALAD_LOG_TRACEBACK*: Runs TraceBack function with collide set to True, if this flag is set to 'collide'. This replaces any common prefix between current traceback log and previous invocation with "..." - *DATALAD_EXC_STR_TBLIMIT*: This flag is used by the datalad extract_tb function which extracts and formats stack-traces. It caps the number of lines to DATALAD_EXC_STR_TBLIMIT of pre-processed entries from traceback. - *DATALAD_TESTS_TEMP_KEEP*: Function rmtemp will not remove temporary file/directory created for testing if this flag is set - *DATALAD_TESTS_TEMP_DIR*: Create a temporary directory at location specified by this flag. It is used by tests to create a temporary git directory while testing git annex archives etc - *DATALAD_TESTS_NONETWORK*: Skips network tests completely if this flag is set Examples include test for s3, git_repositories, openfmri etc - *DATALAD_TESTS_SSH*: Skips SSH tests if this flag is **not** set - *DATALAD_TESTS_NOTEARDOWN*: Does not execute teardown_package which cleans up temp files and directories created by tests if this flag is set - *DATALAD_TESTS_USECASSETTE*: Specifies the location of the file to record network transactions by the VCR module. Currently used by when testing custom special remotes - *DATALAD_TESTS_PROTOCOLREMOTE*: Binary flag to specify whether to test protocol interactions of custom remote with annex - *DATALAD_TESTS_RUNCMDLINE*: Binary flag to specify if shell testing using shunit2 to be carried out - *DATALAD_TESTS_TEMP_FS*: Specify the temporary file system to use as loop device for testing DATALAD_TESTS_TEMP_DIR creation - *DATALAD_TESTS_TEMP_FSSIZE*: Specify the size of temporary file system to use as loop device for testing DATALAD_TESTS_TEMP_DIR creation - *DATALAD_TESTS_NONLO*: Specifies network interfaces to bring down/up for testing. Currently used by travis. - *DATALAD_API_ALWAYSRENDER*: Would make api functions always use a version with cmdline output renderer (i.e. the one with `_` suffix) - *DATALAD_CMD_PROTOCOL*: Specifies the protocol number used by the Runner to note shell command or python function call times and allows for dry runs. 'externals-time' for ExecutionTimeExternalsProtocol, 'time' for ExecutionTimeProtocol and 'null' for NullProtocol. Any new DATALAD_CMD_PROTOCOL has to implement datalad.support.protocol.ProtocolInterface - *DATALAD_CMD_PROTOCOL_PREFIX*: Sets a prefix to add before the command call times are noted by DATALAD_CMD_PROTOCOL. datalad-0.4.1/CONTRIBUTORS000066400000000000000000000002621301115270200147560ustar00rootroot00000000000000The following people have contributed to DataLad: Alex Waite Benjamin Poldrack Debanjum Singh Solanky Gergana Alteva Horea Christian Jason Gors Michael Hanke Yaroslav Halchenko datalad-0.4.1/COPYING000066400000000000000000000037201301115270200141330ustar00rootroot00000000000000# Main Copyright/License DataLad, including all examples, code snippets and attached documentation is covered by the MIT license. The MIT License Copyright (c) 2013- Yaroslav Halchenko 2015- DataLad Team Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. See CONTRIBUTORS file for a full list of contributors. # 3rd Party Code Some code distributed within DataLad was not developed by the DataLad team, hence you should adhere to the copyright and license terms of respective authors if you are to use corresponding parts. ## `datalad/resources/website` - [JQuery](https://code.jquery.com) - MIT License - Copyright (c) 2016- - [Datatables](http://datatables.net) - MIT License - Copyright (c) 2016- - [BlueImp-MD5](https://github.com/blueimp/JavaScript-MD5) - MIT License - Copyright (c) 2016- - [QUnit](https://qunitjs.com/) - MIT License - Copyright (c) - [Sinon-QUnit Plugin](http://sinonjs.org/qunit/) - BSD License - Copyright (c) 2010-2011datalad-0.4.1/Gruntfile.js000066400000000000000000000004471301115270200154000ustar00rootroot00000000000000module.exports = function(grunt) { // Project configuration. grunt.initConfig({ qunit: { files: ['datalad/resources/website/tests/test.html'] } }); // Load plugin grunt.loadNpmTasks('grunt-contrib-qunit'); // Task to run tests grunt.registerTask('test', 'qunit'); }; datalad-0.4.1/Makefile000066400000000000000000000014151301115270200145370ustar00rootroot00000000000000# simple makefile to simplify repetetive build env management tasks under posix # Ideas borrowed from scikit-learn's and PyMVPA Makefiles -- thanks! PYTHON ?= python NOSETESTS ?= nosetests MODULE ?= datalad all: clean test clean: $(PYTHON) setup.py clean rm -rf dist build bin bin: mkdir -p $@ PYTHONPATH=bin:$(PYTHONPATH) python setup.py develop --install-dir $@ test-code: bin PATH=bin:$(PATH) PYTHONPATH=bin:$(PYTHONPATH) $(NOSETESTS) -s -v $(MODULE) test-coverage: rm -rf coverage .coverage $(NOSETESTS) -s -v --with-coverage $(MODULE) test: test-code trailing-spaces: find $(MODULE) -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \; code-analysis: flake8 $(MODULE) | grep -v __init__ | grep -v external pylint -E -i y $(MODULE)/ # -d E1103,E0611,E1101 datalad-0.4.1/README.md000066400000000000000000000136221301115270200143610ustar00rootroot00000000000000 ____ _ _ _ | _ \ __ _ | |_ __ _ | | __ _ __| | | | | | / _` | | __| / _` | | | / _` | / _` | | |_| | | (_| | | |_ | (_| | | |___ | (_| | | (_| | |____/ \__,_| \__| \__,_| |_____| \__,_| \__,_| Read me [![Travis tests status](https://secure.travis-ci.org/datalad/datalad.png?branch=master)](https://travis-ci.org/datalad/datalad) [![codecov.io](https://codecov.io/github/datalad/datalad/coverage.svg?branch=master)](https://codecov.io/github/datalad/datalad?branch=master) [![Documentation](https://readthedocs.org/projects/datalad/badge/?version=latest)](http://datalad.rtfd.org) # 1000ft overview DataLad aims to make data management and data distribution more accessible. To do that it stands on the shoulders of [Git] and [Git-annex] to deliver a decentralized system for data exchange. This includes automated ingestion of data from online portals, and exposing it in readily usable form as Git(-annex) repositories, so-called datasets. The actual data storage and permission management, however, remains with the original data providers. # Status DataLad is under rapid development. While the code base is still growing, the focus is increasingly shifting towards robust and safe operation with a sensible API. Organization and configuration are still subject of considerable reorganization and standardization. However, DataLad is, in fact, usable today and user feedback is always welcome. # DataLad 101 A growing number of datasets is made available from http://datasets.datalad.org . Those datasets are just regular git/git-annex repositories organized into a hierarchy using git submodules mechanism. So you can use regular git/git-annex commands to work with them, but might need `datalad` to be installed to provide additional functionality (e.g., fetching from portals requiring authentication such as CRCNS, HCP; or accessing data originally distributed in tarballs). But datalad aims to provide higher level interface on top of git/git-annex to simplify consumption and sharing of new or derived datasets. To that end, you can install **all** of those datasets using datalad install -r /// which will `git clone` all of those datasets under `datasets.datalad.org` sub-directory. This command will not fetch any large data files, but will merely recreate full hierarchy of all of those datasets locally, which also takes a good chunk of your filesystem meta-data storage. Instead of fetching all datasets at once you could either specify specific dataset to be installed, e.g. datalad install ///openfmri/ds000113 or install top level dataset by omitting `-r` option and then calling `datalad install` for specific sub-datasets you want to have installed, possibly with `-r` to install their sub-datasets as well, e.g. datalad install /// cd datasets.datalad.org datalad install -r openfmri/ds000001 indi/fcon1000 You can navigate datasets you have installed in your terminal or browser, while fetching necessary files or installing new sub-datasets using the `datalad get [FILE|DIR]` command. DataLad will take care about downloading, extracting, and possibly authenticating (would ask you for credentials) in a uniform fashion regardless of the original data location or distribution serialization (e.g., a tarball). Since it is using git and git-annex underneath, you can be assured that you are getting **exact** correct version of the data. Use-cases DataLad covers are not limited to "consumption" of data. DataLad aims also to help publishing original or derived data, thus facilitating more efficient data management when collaborating or simply sharing your data. You can find more documentation at http://docs.datalad.org . # Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) if you are interested in internals or contributing to the project. # Installation ## Debian-based systems On Debian-based systems we recommend to enable [NeuroDebian] from which we provide recent releases of DataLad. datalad package recommends some relatively heavy packages (e.g. scrapy) which are useful only if you are interested in using `crawl` functionality. If you need just the base functionality of the datalad, install without recommended packages (e.g., `apt-get install --no-install-recommends datalad`) ## Other Linux'es, OSX (Windows yet TODO) via pip By default, installation via pip installs core functionality of datalad allowing for managing datasets etc. Additional installation schemes are available, so you could provide enhanced installation via `pip install datalad[SCHEME]` where `SCHEME` could be - `crawl` to also install `scrapy` which is used in some crawling constructs - `tests` to also install dependencies used by unit-tests battery of the datalad - `full` to install all dependencies. For installation through `pip` you would need some external dependencies not shipped from it (e.g. `git-annex`, etc.) for which please refer to the next section. ## Dependencies Our [setup.py] and accompanying packaging describe all necessary dependencies. On Debian-based systems we recommend to enable [NeuroDebian] since we use it to provide backports of recent fixed external modules we depend upon, and up-to-date [Git-annex] is necessary for proper operation of DataLad packaged (install `git-annex-standalone` from NeuroDebian repository). Additionally, if you would like to develop and run our tests battery see [CONTRIBUTING.md](CONTRIBUTING.md) regarding additional dependencies. Later we will provide bundled installations of DataLad across popular platforms. # License MIT/Expat # Disclaimer It is in a alpha stage -- **nothing** is set in stone yet -- but already usable in a limited scope. [Git]: https://git-scm.com [Git-annex]: http://git-annex.branchable.com [setup.py]: https://github.com/datalad/datalad/blob/master/setup.py [NeuroDebian]: http://neuro.debian.net datalad-0.4.1/cfgs/000077500000000000000000000000001301115270200140205ustar00rootroot00000000000000datalad-0.4.1/cfgs/allen-human.cfg000066400000000000000000000014521301115270200167040ustar00rootroot00000000000000[DEFAULT] # Now organization specific settings organization = allen # Now project specific -- that is the level where things would be placed # according to the default scheme of things project = human description = "Allen institute human MRI/genetic data" [genetic] url = file:///home/yoh/proj/datalad/sample_htmls/microarray.html # First fetch archived ones and place/tag/commit accordingly include_href = well_known_file_download/177 # and only then enable fresh ones #include_href = well_known_file_download/178 # regardless of the extract filename_e = "%%(href_a)s.zip" %% locals() # how do we figure out either it changed or not [docs] url = file:///home/yoh/proj/datalad/sample_htmls/Documentation #url = http://help.brain-map.org/display/humanbrain/Documentation include_href = download/attachments datalad-0.4.1/cfgs/arjlover-fast.cfg000066400000000000000000000017231301115270200172630ustar00rootroot00000000000000[DEFAULT] # Now organization specific settings incoming = repos/arjlover-fast public = %(incoming)s description = "film.arjlover.net: Collection of Soviet-era cartoons/films/audiobooks for kids" # in general we are after videos include_href = \.(avi|mpeg|mpg|mp3|mp4|ogg)$ # TODO: --relaxed mode = fast [multiki] url = http://multiki.arjlover.net/multiki/ #url = file:///home/yoh/proj/datalad/sample_htmls/multiki.html #include_href = szopie.avi$ # "INTERESTING" one -- just plain hierarchy where we need to get inside [audiobooks] url = http://audio.arjlover.net/audio/ include_href=.* recurse=/$ [filmy] url = http://film.arjlover.net/film/ #include_href = 01\.99\.avi$ #url = file:///home/yoh/proj/datalad/sample_htmls/films_short.html ## [filmiki] url = http://filmiki.arjlover.net/filmiki/ #include_href = 01\.99\.avi$ [radioteatr] url = http://radioteatr.arjlover.net #url = file:///home/yoh/proj/datalad/sample_htmls/radio-theater_short.html #include_href = \.mp3$ datalad-0.4.1/cfgs/arjlover.cfg000066400000000000000000000001321301115270200163210ustar00rootroot00000000000000[INCLUDES] before = arjlover-fast.cfg [DEFAULT] incoming = repos/arjlover mode = relaxed datalad-0.4.1/cfgs/common.cfg000066400000000000000000000037371301115270200160030ustar00rootroot00000000000000# There should be a config file per repository # identifying all pieces to be downloaded and placed in annexes: # # incoming -- annex where we "download things" before either # linking or extracting into a public annex # public -- annex where we extract and possibly complement # with manual additions/downloads. But whatever gets # added automatically should be maintained in aggrement # with .meta_info.json stored in incoming # By default incoming == public, but then we would track that things # are not extracted?? TODO # TODO: should we allow hierarchies? i.e. page pointing to files under # different subdirectories and we would like to preserve that # structure... [DEFAULT] # modes of operation. # download # fast -- git annex addurl --fast, where no download will be # carried out, in case of encountering archives -- crash. # Intended for quickly creating git-annex repositories # for website directories such as arjlover. In fast mode # it would not even query for remote filenames mode = download # BIG defaults which might need to be shared among ALL # organizations and projects keep_orig = True # store "timestamp" information per each target file meta_info = True directory = %(__name__)s # default public = %(incoming)s incoming_top = repos/incoming public_top = repos/public # Generic arrangement incoming = %(incoming_top)s/%(organization)s/%(project)s public = %(public_top)s/%(organization)s/%(project)s # These are just common among sections exclude_href = # based on the link text include_href_a = exclude_href_a = # specify which files should be extracted. How to extract would be # decided internally # By defaulit would just extract all known (internally) archives types #archives_re = (\.(tar\.gz|tar\.bz))$ # It could also be used to add a suffix (e.g. %(filename)s.zip) corresponding # to the archive whenever url/filename doesn't carry any filename = %(filename)s datalad-0.4.1/cfgs/example-feat.cfg000066400000000000000000000007031301115270200170510ustar00rootroot00000000000000[DEFAULT] # Now organization specific settings incoming = repos/1r/classes public = %(incoming)s description = "TEST TWO" # in general we are after videos #[cope7] #url = http://www.onerussian.com/Sci/experiments/YHOB/feat-III-2nd-S1-19+22-30_z2.3_t0.05_gen_scan_sg.gfeat/cope7.feat/report.html url = http://www.onerussian.com/classes/cis780/ [cis780/docs] include_href = \.pdf$ [cis780/scripts] include_href = \.tgz$ #filename = %(filename)s__/ datalad-0.4.1/cfgs/nih-videos.cfg000066400000000000000000000021761301115270200165540ustar00rootroot00000000000000[DEFAULT] # Now organization specific settings incoming = repos/nih/videocast public = %(incoming)s description = "NIH VideoCasting and Podcasting archived videos" # in general we are after videos but can't afford to download them exec = from nihhelpers import get_video_filename mode = fast [recent] url = http://videocast.nih.gov/pastevents.asp include_href = .*File=\d*&download=1$ filename_e = get_video_filename(link, filename) # NOT IMPLEMENTED! recurse_a = "Next Page" # Initial run -- all the pages recurse = .*pastevents\.asp\?c=0&s=\d+ #recurse = .*pastevents\.asp\?c=0&s=11$ # for subsequent runs not really needed to check the whole history # and first few pages should be enough #recurse = .*pastevents\.asp\?c=0&s=[123]$ # TODO: interesting use-case -- may be those should be added as alternative urls #exclude_href_a = mirror # TODO... They should appear on http://videocast.nih.gov/PastEvents.asp # but seems to be not yet there # [videos] # heavy -- so not for the laptop # [materials] # include_href = \.(zip|tar\.gz) # # TODO: interesting use-case -- may be those should be added as alternative urls # exclude_href_a = mirror datalad-0.4.1/cfgs/nihhelpers.py000066400000000000000000000045511301115270200165400ustar00rootroot00000000000000#emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- #ex: set sts=4 ts=4 sw=4 noet: """ COPYRIGHT: Yaroslav Halchenko 2013 LICENSE: MIT Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ __author__ = 'Yaroslav Halchenko' __copyright__ = 'Copyright (c) 2013 Yaroslav Halchenko' __license__ = 'MIT' import re import os import bs4 import time def slugify(value): """Normalizes the string: removes non-alpha characters. """ import unicodedata value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') value = unicode(re.sub('[^\w\s-]', '', value).strip()) return value def get_video_filename(link, filename): video_entry = list(link.parents)[3] #
assert(isinstance(video_entry, bs4.element.Tag)) bold_entries = video_entry.find_all('b') assert(len(bold_entries) == 2) # that is what we know atm title = bold_entries[0].text date_ = bold_entries[1].text # Parse/convert the date date = time.strptime(date_, '%A, %B %d, %Y') date_str = time.strftime('%Y/%m-%d', date) # try to find extension in the filename ext = os.path.splitext(filename)[1] if not ext or len(ext) > 5: # For now just hope that it is a video extension... # TODO -- check with mime ... etc? ext = '.avi' return "%s - %s%s" % (date_str, slugify(title), ext) datalad-0.4.1/cfgs/openfmri-ds000001.cfg000066400000000000000000000022331301115270200173650ustar00rootroot00000000000000[INCLUDES] before = openfmri.cfg [DEFAULT] dataset = ds000001 #url = https://openfmri.org/dataset/ds000001 #url = file:///home/yoh/proj/datalad/sample_htmls/openfmri/ds00001/ds000001 experiment = Balloon Analog Risk-taking Task [data] include_href = .*_raw_.*\.tgz filename = raw/ check_url_limit = 1 # TODO? and how? #checksums_url = https://openfmri.org/system/files/ds001_raw_checksums.txt checksums_url = .*_raw_checksums\.txt #changelog_url = https://openfmri.org/system/files/release_history_6.txt license_url = http://opendatacommons.org/licenses/pddl/1.0/ #TODO: check where they 'extract' now into? may be we are doomed now #to introduce tracking within the archives if they extract under the #same directory [data/processed] include_href = .*_models.tgz # TODO -- we would need to strip the .tgz! filename = "models/" check_url_limit = 1 [addurl:aws-http] # url to check versions etc: http://openfmri.aws.amazon.com.s3.amazonaws.com/?versions # url to download -- the same? backend = s3 s3_prefix = ds001/ s3_versioning = True # either all files found on the remote should be present locally absent_locally = .*\.bak # or the other way around absent_remotely = .* datalad-0.4.1/cfgs/openfmri.cfg000066400000000000000000000006711301115270200163240ustar00rootroot00000000000000[DEFAULT] organization = openfmri project = %(dataset)s incoming = repos/incoming/%(organization)s/%(project)s public = repos/public/%(organization)s/%(project)s url = https://openfmri.org/dataset/%(dataset)s description = "OpenFMRI: %(experiment)s" archives_re = (\.(tar\.gz|tar\.bz|tgz|zip))$ [changelogs] directory = include_href = .*release_history_.*\.txt filename = "changelog.txt" check_url_limit = 1 # do not annex anything annex = datalad-0.4.1/datalad/000077500000000000000000000000001301115270200144705ustar00rootroot00000000000000datalad-0.4.1/datalad/__init__.py000066400000000000000000000101551301115270200166030ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """DataLad aims to expose (scientific) data available online as a unified data distribution with the convenience of git-annex repositories as a backend.""" # Other imports are interspersed with lgr.debug to ease troubleshooting startup # delays etc. from .config import ConfigManager cfg = ConfigManager() from .log import lgr lgr.log(5, "Instantiating ssh manager") from .support.sshconnector import SSHManager ssh_manager = SSHManager() import atexit atexit.register(ssh_manager.close, allow_fail=False) atexit.register(lgr.log, 5, "Exiting") from .version import __version__ def test(package='datalad', **kwargs): """A helper to run datalad's tests. Requires numpy and nose See numpy.testing.Tester -- **kwargs are passed into the Tester().test call """ try: from numpy.testing import Tester Tester(package=package).test(**kwargs) # we don't have any benchmarks atm # bench = Tester().bench except ImportError: raise RuntimeError('Need numpy >= 1.2 for datalad.tests(). Nothing is done') test.__test__ = False # Following fixtures are necessary at the top level __init__ for fixtures which # would cover all **/tests and not just datalad/tests/ # To store settings which setup_package changes and teardown_package should return _test_states = { 'loglevel': None, 'DATALAD_LOG_LEVEL': None, } def setup_package(): import os # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: os.environ['HOME'] = os.environ['GIT_HOME'] # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent _test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL os.environ['DATALAD_LOG_LEVEL'] = '100' else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend ui.set_backend('tests-noninteractive') def teardown_package(): import os if os.environ.get('DATALAD_TESTS_NOTEARDOWN'): return from datalad.ui import ui ui.set_backend(_test_states['ui_backend']) if _test_states['loglevel'] is not None: lgr.setLevel(_test_states['loglevel']) if _test_states['DATALAD_LOG_LEVEL'] is None: os.environ.pop('DATALAD_LOG_LEVEL') else: os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL'] from datalad.tests import _TEMP_PATHS_GENERATED from datalad.tests.utils import rmtemp if len(_TEMP_PATHS_GENERATED): msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED)) else: msg = "Nothing to remove" lgr.debug("Teardown tests. " + msg) for path in _TEMP_PATHS_GENERATED: rmtemp(path, ignore_errors=True) lgr.log(5, "Done importing main __init__") datalad-0.4.1/datalad/__main__.py000066400000000000000000000060741301115270200165710ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Helper to use datalad as a "runnable" module with -m datalad""" import sys from . import __version__ from .auto import AutomagicIO from .log import lgr def usage(outfile, executable=sys.argv[0]): if '__main__.py' in executable: # That was -m datalad way to launch executable = "%s -m datalad" % sys.executable outfile.write("""Usage: %s [OPTIONS] [ARGS] Purpose: To provide FUSE-like operation whenever necessary files (as accessed by open, h5py.File) are requested, they get fetched. Meta-options: --help Display this help then exit. --version Output version information then exit. """ % executable) def runctx(cmd, globals=None, locals=None): if globals is None: globals = {} if locals is None: locals = {} try: exec(cmd, globals, locals) finally: # good opportunity to avoid atexit I guess. pass for now pass def main(argv=None): import os import getopt if argv is None: argv = sys.argv try: opts, prog_argv = getopt.getopt(argv[1:], "", ["help", "version"]) # TODO: support options for whatever we would support ;) # probably needs to hook in somehow into commands/options available # under cmdline/ except getopt.error as msg: sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) sys.stderr.write("Try `%s --help' for more information\n" % sys.argv[0]) sys.exit(1) # and now we need to execute target script "manually" # Borrowing up on from trace.py for opt, val in opts: if opt == "--help": usage(sys.stdout, executable=argv[0]) sys.exit(0) if opt == "--version": sys.stdout.write("datalad %s\n" % __version__) sys.exit(0) sys.argv = prog_argv progname = prog_argv[0] sys.path[0] = os.path.split(progname)[0] try: with open(progname) as fp: code = compile(fp.read(), progname, 'exec') # try to emulate __main__ namespace as much as possible globs = { '__file__': progname, '__name__': '__main__', '__package__': None, '__cached__': None, } # Since used explicitly -- activate the beast aio = AutomagicIO(activate=True) lgr.info("Running code of %s", progname) runctx(code, globs, globs) # TODO: see if we could hide our presence from the final tracebacks if execution fails except IOError as err: lgr.error("Cannot run file %r because: %s" % (sys.argv[0], err)) sys.exit(1) except SystemExit: pass if __name__ == '__main__': main() datalad-0.4.1/datalad/api.py000066400000000000000000000123411301115270200156140ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Python DataLad API exposing user-oriented commands (also available via CLI)""" # Should have no spurious imports/definitions at the module leve from .distribution.dataset import Dataset def _generate_func_api(): """Auto detect all available interfaces and generate a function-based API from them """ from importlib import import_module from inspect import isgenerator from collections import namedtuple from collections import OrderedDict from functools import wraps from datalad import cfg from .interface.base import update_docstring_with_parameters from .interface.base import get_interface_groups from .interface.base import get_api_name from .interface.base import alter_interface_docs_for_api def _kwargs_to_namespace(call, args, kwargs): """ Given a __call__, args and kwargs passed, prepare a cmdlineargs-like thing """ from inspect import getargspec argspec = getargspec(call) defaults = argspec.defaults nargs = len(argspec.args) assert (nargs >= len(defaults)) # map any args to their name argmap = list(zip(argspec.args[:len(args)], args)) # map defaults of kwargs to their names (update below) argmap += list(zip(argspec.args[-len(defaults):], defaults)) kwargs_ = OrderedDict(argmap) # update with provided kwarg args kwargs_.update(kwargs) assert (nargs == len(kwargs_)) # Get all arguments removing those possible ones used internally and # which shouldn't be exposed outside anyways [kwargs_.pop(k) for k in kwargs_ if k.startswith('_')] namespace = namedtuple("smth", kwargs_.keys())(**kwargs_) return namespace def call_gen(call, renderer): """Helper to generate a call_ for call, to use provided renderer""" @wraps(call) def call_(*args, **kwargs): ret1 = ret = call(*args, **kwargs) if isgenerator(ret): # At first I thought we might just rerun it for output # at the end, but that wouldn't work if command actually # has a side-effect, i.e. actually doing something # so we actually need to memoize all generated output and output # it instead from datalad.utils import saved_generator ret, ret1 = saved_generator(ret) renderer(ret, _kwargs_to_namespace(call, args, kwargs)) return ret1 # TODO: see if we could proxy the "signature" of function # call from the original one call_.__doc__ += \ "\nNote\n----\n\n" \ "This version of a function uses cmdline results renderer before " \ "returning the result" return call_ always_render = cfg.obtain('datalad.api.alwaysrender') for grp_name, grp_descr, interfaces in get_interface_groups(): for intfspec in interfaces: # turn the interface spec into an instance mod = import_module(intfspec[0], package='datalad') intf = getattr(mod, intfspec[1]) spec = getattr(intf, '_params_', dict()) # FIXME no longer using an interface class instance # convert the parameter SPEC into a docstring for the function update_docstring_with_parameters( intf.__call__, spec, prefix=alter_interface_docs_for_api( intf.__doc__), suffix=alter_interface_docs_for_api( intf.__call__.__doc__) ) globals()[get_api_name(intfspec)] = intf.__call__ # And the one with '_' suffix which would use cmdline results # renderer if hasattr(intf, 'result_renderer_cmdline'): intf__ = call_gen(intf.__call__, intf.result_renderer_cmdline) globals()[get_api_name(intfspec) + '_'] = intf__ if always_render: globals()[get_api_name(intfspec)] = intf__ def _fix_datasetmethod_docs(): """Fix up dataset methods docstrings which didn't get proper docs """ from six import PY2 for attr in dir(Dataset): try: func = getattr(Dataset, attr) orig_func = getattr(func, '__orig_func__') except AttributeError: continue if PY2: func = func.__func__ orig__doc__ = func.__doc__ if orig__doc__ and orig__doc__.strip(): # pragma: no cover raise RuntimeError( "No meaningful docstring should have been assigned before now. Got %r" % orig__doc__ ) func.__doc__ = orig_func.__doc__ # Invoke above helpers _generate_func_api() _fix_datasetmethod_docs() # Be nice and clean up the namespace properly del _generate_func_api del _fix_datasetmethod_docs datalad-0.4.1/datalad/auto.py000066400000000000000000000170741301115270200160230ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Proxy basic file operations (e.g. open) to auto-obtain files upon I/O """ import sys # OPT delay import for expensive mock until used #from mock import patch from six import PY2 import six.moves.builtins as __builtin__ builtins_name = '__builtin__' if PY2 else 'builtins' try: import h5py except ImportError: h5py = None import logging import os from os.path import dirname, lexists, realpath from os.path import exists from git.exc import InvalidGitRepositoryError from .dochelpers import exc_str from .support.annexrepo import AnnexRepo from .cmdline.helpers import get_repo_instance lgr = logging.getLogger("datalad.auto") class _EarlyExit(Exception): """Helper to early escape try/except logic in wrappde open""" pass class AutomagicIO(object): """Class to proxy commonly used API for accessing files so they get automatically fetched Currently supports builtin open() and h5py.File when those are read """ def __init__(self, autoget=True, activate=False): self._active = False self._builtin_open = __builtin__.open self._builtin_exists = os.path.exists if h5py: self._h5py_File = h5py.File else: self._h5py_File = None self._autoget = autoget self._in_open = False self._log_online = True from mock import patch self._patch = patch if activate: self.activate() def __enter__(self): self.activate() return self def __exit__(self, exc_type, exc_value, traceback): self.deactivate() @property def autoget(self): return self._autoget @property def active(self): return self._active def _proxy_open_name_mode(self, origname, origfunc, *args, **kwargs): """Proxy for various "open" which have first argument name and 2nd - mode """ # wrap it all for resilience to errors -- proxying must do no harm! try: if self._in_open: raise _EarlyExit self._in_open = True # just in case someone kept alias/assignment # return stock open for the duration of handling so that # logging etc could workout correctly with self._patch(origname, origfunc): lgr.log(2, "Proxying open with %r %r", args, kwargs) # had to go with *args since in PY2 it is name, in PY3 file # deduce arguments if len(args) > 0: # name/file was provided file = args[0] else: filearg = "name" if PY2 else "file" if filearg not in kwargs: # so the name was missing etc, just proxy into original open call and let it puke lgr.debug("No name/file was given, avoiding proxying") raise _EarlyExit file = kwargs.get(filearg) mode = 'r' if len(args) > 1: mode = args[1] elif 'mode' in kwargs: mode = kwargs['mode'] if 'r' in mode: self._dataset_auto_get(file) else: lgr.debug("Skipping operation on %s since mode=%r", file, mode) except _EarlyExit: pass except Exception as e: # If anything goes wrong -- we should complain and proceed with self._patch(origname, origfunc): lgr.warning("Failed proxying open with %r, %r: %s", args, kwargs, exc_str(e)) finally: self._in_open = False # finally give it back to stock open return origfunc(*args, **kwargs) def _proxy_open(self, *args, **kwargs): return self._proxy_open_name_mode(builtins_name + '.open', self._builtin_open, *args, **kwargs) def _proxy_h5py_File(self, *args, **kwargs): return self._proxy_open_name_mode('h5py.File', self._h5py_File, *args, **kwargs) def _proxy_exists(self, path): # TODO: decide either it should may be retrieved right away. # For now, as long as it is a symlink pointing to under .git/annex if exists(path): return True return lexists(path) and 'annex/objects' in realpath(path) def _dataset_auto_get(self, filepath): """Verify that filepath is under annex, and if so and not present - get it""" if not self._autoget: return # if filepath is not there at all (program just "checked" if it could access it if not lexists(filepath): lgr.log(2, "Not testing/getting file %s since it is not there", filepath) return # deduce directory for filepath filedir = dirname(filepath) try: # TODO: verify logic for create -- we shouldn't 'annexify' non-annexified # see https://github.com/datalad/datalad/issues/204 annex = get_repo_instance(filedir) except (RuntimeError, InvalidGitRepositoryError) as e: # must be not under annex etc return if not isinstance(annex, AnnexRepo): # not an annex -- can do nothing return # "quick" check first if under annex at all try: # might fail. TODO: troubleshoot when it does e.g. # datalad/tests/test_auto.py:test_proxying_open_testrepobased under_annex = annex.is_under_annex(filepath, batch=True) except: # MIH: really? what if MemoryError under_annex = None # either it has content if (under_annex or under_annex is None) and not annex.file_has_content(filepath): lgr.info("File %s has no content -- retrieving", filepath) annex.get(filepath) def activate(self): lgr.info("Activating DataLad's AutoMagicIO") # Some beasts (e.g. tornado used by IPython) override outputs, and # provide fileno which throws exception. In such cases we should not log online self._log_online = hasattr(sys.stdout, 'fileno') and hasattr(sys.stderr, 'fileno') try: if self._log_online: sys.stdout.fileno() sys.stderr.fileno() except: # MIH: IOError? self._log_online = False if self.active: lgr.warning("%s already active. No action taken" % self) return # overloads __builtin__.open = self._proxy_open os.path.exists = self._proxy_exists if h5py: h5py.File = self._proxy_h5py_File self._active = True def deactivate(self): if not self.active: lgr.warning("%s is not active, can't deactivate" % self) return __builtin__.open = self._builtin_open if h5py: h5py.File = self._h5py_File os.path.exists = self._builtin_exists self._active = False def __del__(self): try: if self._active: self.deactivate() except: # MIH: IOError? pass try: super(self.__class__, self).__del__() except: pass datalad-0.4.1/datalad/cmd.py000066400000000000000000000445471301115270200156230ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """ Wrapper for command and function calls, allowing for dry runs and output handling """ import subprocess import sys import logging import os import shutil import shlex import atexit import functools from six import PY3, PY2 from six import string_types, binary_type from os.path import abspath, isabs from .dochelpers import exc_str from .support.exceptions import CommandError from .support.protocol import NullProtocol, DryRunProtocol, \ ExecutionTimeProtocol, ExecutionTimeExternalsProtocol from .utils import on_windows lgr = logging.getLogger('datalad.cmd') _TEMP_std = sys.stdout, sys.stderr if PY2: # TODO apparently there is a recommended substitution for Python2 # which is a backported implementation of python3 subprocess # https://pypi.python.org/pypi/subprocess32/ pass class Runner(object): """Provides a wrapper for calling functions and commands. An object of this class provides a methods that calls shell commands or python functions, allowing for protocolling the calls and output handling. Outputs (stdout and stderr) can be either logged or streamed to system's stdout/stderr during execution. This can be enabled or disabled for both of them independently. Additionally, a protocol object can be a used with the Runner. Such a protocol has to implement datalad.support.protocol.ProtocolInterface, is able to record calls and allows for dry runs. """ __slots__ = ['commands', 'dry', 'cwd', 'env', 'protocol', '_log_outputs'] def __init__(self, cwd=None, env=None, protocol=None, log_outputs=None): """ Parameters ---------- cwd: string, optional Base current working directory for commands. Could be overridden per run call via cwd option env: dict, optional Custom environment to use for calls. Could be overridden per run call via env option protocol: ProtocolInterface Protocol object to write to. log_outputs : bool, optional Switch to instruct either outputs should be logged or not. If not set (default), config 'datalad.log outputs' would be consulted """ self.cwd = cwd self.env = env if protocol is None: # TODO: config cmd.protocol = null protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null') protocol = { 'externals-time': ExecutionTimeExternalsProtocol, 'time': ExecutionTimeProtocol, 'null': NullProtocol }[protocol_str]() if protocol_str != 'null': # we need to dump it into a file at the end # TODO: config cmd.protocol_prefix = protocol filename = '%s-%s.log' % ( os.environ.get('DATALAD_CMD_PROTOCOL_PREFIX', 'protocol'), id(self) ) atexit.register(functools.partial(protocol.write_to_file, filename)) self.protocol = protocol self._log_outputs = log_outputs # we don't know yet either we need ot log every output or not def __call__(self, cmd, *args, **kwargs): """Convenience method This will call run() or call() depending on the kind of `cmd`. If `cmd` is a string it is interpreted as the to be executed command. Otherwise it is expected to be a callable. Any other argument is passed to the respective method. Parameters ---------- cmd: str or callable command string to be executed via shell or callable to be called. `*args`: `**kwargs`: see Runner.run() and Runner.call() for available arguments. Raises ------ TypeError if cmd is neither a string nor a callable. """ if isinstance(cmd, string_types) or isinstance(cmd, list): return self.run(cmd, *args, **kwargs) elif callable(cmd): return self.call(cmd, *args, **kwargs) else: raise TypeError("Argument 'command' is neither a string, " "nor a list nor a callable.") @property def log_outputs(self): if self._log_outputs is None: try: from . import cfg self._log_outputs = \ cfg.getbool('datalad.log', 'outputs', default=False) except ImportError: # could be too early, then DON'T log! return True return self._log_outputs # Two helpers to encapsulate formatting/output def _log_out(self, line): if line and self.log_outputs: self.log("stdout| " + line.rstrip('\n')) def _log_err(self, line, expected=False): if line and self.log_outputs: self.log("stderr| " + line.rstrip('\n'), level={True: logging.DEBUG, False: logging.ERROR}[expected]) def _get_output_online(self, proc, log_stdout, log_stderr, expect_stderr=False, expect_fail=False): """ If log_stdout or log_stderr are callables, they will be given a read line to be processed, and return processed result. So if they need to 'swallow' the line from being logged, should just return None Parameters ---------- proc log_stdout: bool or callable or 'online' or 'offline' log_stderr: : bool or callable or 'online' or 'offline' If any of those 'offline', we would call proc.communicate at the end to grab possibly outstanding output from it expect_stderr expect_fail Returns ------- """ stdout, stderr = binary_type(), binary_type() def decide_to_log(v): """Hacky workaround for now so we could specify per each which to log online and which to the log""" if isinstance(v, bool) or callable(v): return v elif v in {'online'}: return True elif v in {'offline'}: return False else: raise ValueError("can be bool, callable, 'online' or 'offline'") log_stdout_ = decide_to_log(log_stdout) log_stderr_ = decide_to_log(log_stderr) while proc.poll() is None: if log_stdout_: lgr.log(3, "Reading line from stdout") line = proc.stdout.readline() if line and callable(log_stdout_): # Let it be processed line = log_stdout_(line) if line: stdout += line self._log_out(line.decode()) # TODO: what level to log at? was: level=5 # Changes on that should be properly adapted in # test.cmd.test_runner_log_stdout() else: pass if log_stderr_: # see for a possibly useful approach to processing output # in another thread http://codereview.stackexchange.com/a/17959 # current problem is that if there is no output on stderr # it stalls lgr.log(3, "Reading line from stderr") line = proc.stderr.readline() if line and callable(log_stderr_): # Let it be processed line = log_stderr_(line) if line: stderr += line self._log_err(line.decode() if PY3 else line, expect_stderr or expect_fail) # TODO: what's the proper log level here? # Changes on that should be properly adapted in # test.cmd.test_runner_log_stderr() else: pass if log_stdout in {'offline'} or log_stderr in {'offline'}: lgr.log(4, "Issuing proc.communicate() since one of the targets " "is 'offline'") stdout_, stderr_ = proc.communicate() stdout += stdout_ stderr += stderr_ return stdout, stderr def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False, expect_stderr=False, expect_fail=False, cwd=None, env=None, shell=None): """Runs the command `cmd` using shell. In case of dry-mode `cmd` is just added to `commands` and it is actually executed otherwise. Allows for separately logging stdout and stderr or streaming it to system's stdout or stderr respectively. Note: Using a string as `cmd` and shell=True allows for piping, multiple commands, etc., but that implies shlex.split() is not used. This is considered to be a security hazard. So be careful with input. Parameters ---------- cmd : str, list String (or list) defining the command call. No shell is used if cmd is specified as a list log_stdout: bool, optional If True, stdout is logged. Goes to sys.stdout otherwise. log_stderr: bool, optional If True, stderr is logged. Goes to sys.stderr otherwise. log_online: bool, optional Either to log as output comes in. Setting to True is preferable for running user-invoked actions to provide timely output expect_stderr: bool, optional Normally, having stderr output is a signal of a problem and thus it gets logged at ERROR level. But some utilities, e.g. wget, use stderr for their progress output. Whenever such output is expected, set it to True and output will be logged at DEBUG level unless exit status is non-0 (in non-online mode only, in online -- would log at DEBUG) expect_fail: bool, optional Normally, if command exits with non-0 status, it is considered an ERROR and logged accordingly. But if the call intended for checking routine, such alarming message should not be logged as ERROR, thus it will be logged at DEBUG level. cwd : string, optional Directory under which run the command (passed to Popen) env : string, optional Custom environment to pass shell: bool, optional Run command in a shell. If not specified, then it runs in a shell only if command is specified as a string (not a list) Returns ------- (stdout, stderr) Raises ------ CommandError if command's exitcode wasn't 0 or None. exitcode is passed to CommandError's `code`-field. Command's stdout and stderr are stored in CommandError's `stdout` and `stderr` fields respectively. """ # TODO: having two PIPEs is dangerous, and leads to lock downs so we # would need either threaded solution as in .communicate or just allow # only one to be monitored and another one just being dumped into a file outputstream = subprocess.PIPE if log_stdout else sys.stdout errstream = subprocess.PIPE if log_stderr else sys.stderr self.log("Running: %s" % (cmd,)) if self.protocol.do_execute_ext_commands: if shell is None: shell = isinstance(cmd, string_types) if self.protocol.records_ext_commands: prot_exc = None prot_id = self.protocol.start_section( shlex.split(cmd, posix=not on_windows) if isinstance(cmd, string_types) else cmd) try: proc = subprocess.Popen(cmd, stdout=outputstream, stderr=errstream, shell=shell, cwd=cwd or self.cwd, env=env or self.env) except Exception as e: prot_exc = e lgr.error("Failed to start %r%r: %s" % (cmd, " under %r" % cwd if cwd else '', exc_str(e))) raise finally: if self.protocol.records_ext_commands: self.protocol.end_section(prot_id, prot_exc) if log_online: out = self._get_output_online(proc, log_stdout, log_stderr, expect_stderr=expect_stderr, expect_fail=expect_fail) else: out = proc.communicate() if PY3: # Decoding was delayed to this point def decode_if_not_None(x): return "" if x is None else binary_type.decode(x) # TODO: check if we can avoid PY3 specific here out = tuple(map(decode_if_not_None, out)) status = proc.poll() # needs to be done after we know status if not log_online: self._log_out(out[0]) if status not in [0, None]: self._log_err(out[1], expected=expect_fail) else: # as directed self._log_err(out[1], expected=expect_stderr) if status not in [0, None]: msg = "Failed to run %r%s. Exit code=%d. out=%s err=%s" \ % (cmd, " under %r" % (cwd or self.cwd), status, out[0], out[1]) (lgr.debug if expect_fail else lgr.error)(msg) raise CommandError(str(cmd), msg, status, out[0], out[1]) else: self.log("Finished running %r with status %s" % (cmd, status), level=8) else: if self.protocol.records_ext_commands: self.protocol.add_section(shlex.split(cmd, posix=not on_windows) if isinstance(cmd, string_types) else cmd, None) out = ("DRY", "DRY") return out def call(self, f, *args, **kwargs): """Helper to unify collection of logging all "dry" actions. Calls `f` if `Runner`-object is not in dry-mode. Adds `f` along with its arguments to `commands` otherwise. f : callable `*args`, `**kwargs`: Callable arguments """ if self.protocol.do_execute_callables: if self.protocol.records_callables: prot_exc = None prot_id = self.protocol.start_section( [str(f), "args=%s" % str(args), "kwargs=%s" % str(kwargs)]) try: return f(*args, **kwargs) except Exception as e: prot_exc = e raise finally: if self.protocol.records_callables: self.protocol.end_section(prot_id, prot_exc) else: if self.protocol.records_callables: self.protocol.add_section( [str(f), "args=%s" % str(args), "kwargs=%s" % str(kwargs)], None) def log(self, msg, level=logging.DEBUG): """log helper Logs at DEBUG-level by default and adds "Protocol:"-prefix in order to log the used protocol. """ if isinstance(self.protocol, NullProtocol): lgr.log(level, msg) else: lgr.log(level, "{%s} %s" % (self.protocol.__class__.__name__, msg)) class GitRunner(Runner): """ Runner to be used to run git and git annex commands Overloads the runner class to check & update GIT_DIR and GIT_WORK_TREE environment variables set to the absolute path if is defined and is relative path """ @staticmethod def get_git_environ_adjusted(env=None): """ Replaces GIT_DIR and GIT_WORK_TREE with absolute paths if relative path and defined """ git_env = env.copy() if env else os.environ.copy() # if env set copy else get os environment for varstring in ['GIT_DIR', 'GIT_WORK_TREE']: var = git_env.get(varstring) if var: # if env variable set if not isabs(var): # and it's a relative path git_env[varstring] = abspath(var) # convert it to absolute path lgr.debug("Updated %s to %s" % (varstring, git_env[varstring])) return git_env def run(self, cmd, env=None, *args, **kwargs): return super(GitRunner, self).run( cmd, env=self.get_git_environ_adjusted(), *args, **kwargs) # #### # Preserve from previous version # TODO: document intention # #### # this one might get under Runner for better output/control def link_file_load(src, dst, dry_run=False): """Just a little helper to hardlink files's load """ dst_dir = os.path.dirname(dst) if not os.path.exists(dst_dir): os.makedirs(dst_dir) if os.path.lexists(dst): lgr.debug("Destination file %(dst)s exists. Removing it first" % locals()) # TODO: how would it interact with git/git-annex os.unlink(dst) lgr.debug("Hardlinking %(src)s under %(dst)s" % locals()) src_realpath = os.path.realpath(src) try: os.link(src_realpath, dst) except AttributeError as e: lgr.warn("Linking of %s failed (%s), copying file" % (src, e)) shutil.copyfile(src_realpath, dst) shutil.copystat(src_realpath, dst) else: lgr.log(2, "Hardlinking finished") def get_runner(*args, **kwargs): # needs local import, because the ConfigManager itself needs the runner from . import cfg # TODO: this is all crawl specific -- should be moved away if cfg.obtain('datalad.crawl.dryrun', default=False): kwargs = kwargs.copy() kwargs['protocol'] = DryRunProtocol() return Runner(*args, **kwargs) datalad-0.4.1/datalad/cmdline/000077500000000000000000000000001301115270200161035ustar00rootroot00000000000000datalad-0.4.1/datalad/cmdline/__init__.py000066400000000000000000000006611301115270200202170ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """ """ __docformat__ = 'restructuredtext' datalad-0.4.1/datalad/cmdline/common_args.py000066400000000000000000000032511301115270200207620ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """ """ __docformat__ = 'restructuredtext' # argument spec template # = ( # , # {} #) from ..cmdline.helpers import HelpAction, LogLevelAction help = ( 'help', ('-h', '--help', '--help-np'), dict(nargs=0, action=HelpAction, help="""show this help message. --help-np forcefully disables the use of a pager for displaying the help message""") ) version = ( 'version', ('--version',), dict(action='version', help="show the program's version and license information") ) _log_level_names = ['critical', 'error', 'warning', 'info', 'debug'] log_level = ( 'log-level', ('-l', '--log-level'), dict(action=LogLevelAction, choices=_log_level_names + [str(x) for x in range(1, 10)], metavar="LEVEL", default='warning', help="""set logging verbosity level. Choose among %s. Also you can specify an integer <10 to provide even more debugging information""" % ', '.join(_log_level_names)) ) pbs_runner = ( 'pbs-runner', ('-p', '--pbs-runner'), dict(choices=['condor'], default=None, help="""execute command by scheduling it via available PBS. For settings, config file will be consulted""") ) datalad-0.4.1/datalad/cmdline/helpers.py000066400000000000000000000216511301115270200201240ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """ """ __docformat__ = 'restructuredtext' import argparse import os import re import sys import gzip from tempfile import NamedTemporaryFile from ..cmd import Runner from ..log import is_interactive from ..utils import getpwd from ..version import __version__ from ..dochelpers import exc_str from logging import getLogger lgr = getLogger('datalad.cmdline') class HelpAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): if is_interactive() and option_string == '--help': # lets use the manpage on mature systems ... try: import subprocess # get the datalad manpage to use manfile = os.environ.get('MANPATH', '/usr/share/man') \ + '/man1/{0}.1.gz'.format(parser.prog.replace(' ', '-')) # extract version field from the manpage if not os.path.exists(manfile): raise IOError("manfile is not found") with gzip.open(manfile) as f: man_th = [line for line in f if line.startswith(b".TH")][0] man_version = man_th.split(b' ')[5].strip(b" '\"\t\n").decode('utf-8') # don't show manpage if man_version not equal to current datalad_version if __version__ != man_version: raise ValueError subprocess.check_call( 'man %s 2> /dev/null' % manfile, shell=True) sys.exit(0) except (subprocess.CalledProcessError, IOError, OSError, IndexError, ValueError) as e: lgr.debug("Did not use manpage since %s", exc_str(e)) if option_string == '-h': helpstr = "%s\n%s" % ( parser.format_usage(), "Use '--help' to get more comprehensive information.") else: helpstr = parser.format_help() # better for help2man # for main command -- should be different sections. And since we are in # heavy output messaging mode... if "commands for dataset operations" in helpstr.lower(): opt_args_str = '*Global options*' pos_args_str = '*Commands*' # tune up usage -- default one is way too heavy helpstr = re.sub('^[uU]sage: .*?\n\s*\n', 'Usage: datalad [global-opts] command [command-opts]\n\n', helpstr, flags=re.MULTILINE | re.DOTALL) # and altogether remove sections with long list of commands helpstr = re.sub(r'positional arguments:\s*\n\s*{.*}\n', '', helpstr) else: opt_args_str = "*Options*" pos_args_str = "*Arguments*" helpstr = re.sub(r'optional arguments:', opt_args_str, helpstr) helpstr = re.sub(r'positional arguments:', pos_args_str, helpstr) # convert all headings to have the first character uppercase headpat = re.compile(r'^([a-z])(.*):$', re.MULTILINE) helpstr = re.subn( headpat, lambda match: r'{0}{1}:'.format(match.group(1).upper(), match.group(2)), helpstr)[0] # usage is on the same line helpstr = re.sub(r'^usage:', 'Usage:', helpstr) print(helpstr) sys.exit(0) class LogLevelAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): from ..log import LoggerHelper LoggerHelper().set_level(level=values) # MIH: Disabled. Non-functional, untested. #class PBSAction(argparse.Action): # """Action to schedule actual command execution via PBS (e.g. Condor)""" # def __call__(self, parser, namespace, values, option_string=None): # pbs = values[0] # import pdb; pdb.set_trace() # i = 1 def parser_add_common_opt(parser, opt, names=None, **kwargs): from . import common_args opt_tmpl = getattr(common_args, opt) opt_kwargs = opt_tmpl[2].copy() opt_kwargs.update(kwargs) if names is None: parser.add_argument(*opt_tmpl[1], **opt_kwargs) else: parser.add_argument(*names, **opt_kwargs) def strip_arg_from_argv(args, value, opt_names): """Strip an originally listed option (with its value) from the list cmdline args """ # Yarik doesn't know better args = args or sys.argv # remove present pbs-runner option args_clean = [] skip = 0 for i, arg in enumerate(args): if skip: # we skip only one as instructed skip -= 1 continue if not (arg in opt_names and i < len(args) - 1 and args[i + 1] == value): args_clean.append(arg) else: # we need to skip this one and next one skip = 1 return args_clean def run_via_pbs(args, pbs): assert(pbs in ('condor',)) # for now # TODO: RF to support multiple backends, parameters, etc, for now -- just condor, no options f = NamedTemporaryFile('w', prefix='datalad-%s-' % pbs, suffix='.submit', delete=False) try: pwd = getpwd() logs = f.name.replace('.submit', '.log') exe = args[0] # TODO: we might need better way to join them, escaping spaces etc. There must be a stock helper #exe_args = ' '.join(map(repr, args[1:])) if len(args) > 1 else '' exe_args = ' '.join(args[1:]) if len(args) > 1 else '' f.write("""\ Executable = %(exe)s Initialdir = %(pwd)s Output = %(logs)s Error = %(logs)s getenv = True arguments = %(exe_args)s queue """ % locals()) f.close() Runner().run(['condor_submit', f.name]) lgr.info("Scheduled execution via %s. Logs will be stored under %s" % (pbs, logs)) finally: os.unlink(f.name) class RegexpType(object): """Factory for creating regular expression types for argparse DEPRECATED AFAIK -- now things are in the config file, but we might provide a mode where we operate solely from cmdline """ def __call__(self, string): if string: return re.compile(string) else: return None # TODO: useful also outside of cmdline, move to support/ from os import curdir def get_repo_instance(path=curdir, class_=None): """Returns an instance of appropriate datalad repository for path. Check whether a certain path is inside a known type of repository and returns an instance representing it. May also check for a certain type instead of detecting the type of repository. Parameters ---------- path: str path to check; default: current working directory class_: class if given, check whether path is inside a repository, that can be represented as an instance of the passed class. Raises ------ RuntimeError, in case cwd is not inside a known repository. """ from os.path import ismount, exists, normpath, isabs from git.exc import InvalidGitRepositoryError from ..utils import expandpath from ..support.gitrepo import GitRepo from ..support.annexrepo import AnnexRepo dir_ = expandpath(path) abspath_ = path if isabs(path) else dir_ if class_ is not None: if class_ == AnnexRepo: type_ = "annex" elif class_ == GitRepo: type_ = "git" else: raise RuntimeError("Unknown class %s." % str(class_)) while not ismount(dir_): # TODO: always correct termination? if exists(opj(dir_, '.git')): # found git dir if class_ is None: # detect repo type: try: return AnnexRepo(dir_, create=False) except RuntimeError as e: pass try: return GitRepo(dir_, create=False) except InvalidGitRepositoryError as e: raise RuntimeError("No datalad repository found in %s" % abspath_) else: try: return class_(dir_, create=False) except (RuntimeError, InvalidGitRepositoryError) as e: raise RuntimeError("No %s repository found in %s." % (type_, abspath_)) else: dir_ = normpath(opj(dir_, "..")) if class_ is not None: raise RuntimeError("No %s repository found in %s" % (type_, abspath_)) else: raise RuntimeError("No datalad repository found in %s" % abspath_) from appdirs import AppDirs from os.path import join as opj dirs = AppDirs("datalad", "datalad.org") datalad-0.4.1/datalad/cmdline/main.py000066400000000000000000000320751301115270200174100ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """""" __docformat__ = 'restructuredtext' import logging lgr = logging.getLogger('datalad.cmdline') lgr.log(5, "Importing cmdline.main") import argparse import sys import textwrap from importlib import import_module import datalad from datalad.cmdline import helpers from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.exceptions import IncompleteResultsError from ..utils import setup_exceptionhook, chpwd from ..dochelpers import exc_str def _license_info(): return """\ Copyright (c) 2013-2016 DataLad developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ # TODO: OPT look into making setup_parser smarter to become faster # Now it seems to take up to 200ms to do all the parser setup # even though it might not be necessary to know about all the commands etc. # I wondered if it could somehow decide on what commands to worry about etc # by going through sys.args first def setup_parser( formatter_class=argparse.RawDescriptionHelpFormatter, return_subparsers=False): lgr.log(5, "Starting to setup_parser") # delay since it can be a heavy import from ..interface.base import dedent_docstring, get_interface_groups, \ get_cmdline_command_name, alter_interface_docs_for_cmdline # setup cmdline args parser parts = {} # main parser parser = argparse.ArgumentParser( # cannot use '@' because we need to input JSON-LD properties (which might come wit @ prefix) # MH: question, do we need this at all? fromfile_prefix_chars=':', # usage="%(prog)s ...", description=dedent_docstring("""\ DataLad provides a unified data distribution with the convenience of git-annex repositories as a backend. DataLad command line tools allow to manipulate (obtain, create, update, publish, etc.) datasets and their collections."""), epilog='"Control Your Data"', formatter_class=formatter_class, add_help=False) # common options helpers.parser_add_common_opt(parser, 'help') helpers.parser_add_common_opt(parser, 'log_level') helpers.parser_add_common_opt(parser, 'pbs_runner') helpers.parser_add_common_opt( parser, 'version', version='datalad %s\n\n%s' % (datalad.__version__, _license_info())) if __debug__: parser.add_argument( '--dbg', action='store_true', dest='common_debug', help="enter Python debugger when uncaught exception happens") parser.add_argument( '--idbg', action='store_true', dest='common_idebug', help="enter IPython debugger when uncaught exception happens") parser.add_argument( '-C', action='append', dest='change_path', metavar='PATH', help="""run as if datalad was started in instead of the current working directory. When multiple -C options are given, each subsequent non-absolute -C is interpreted relative to the preceding -C . This option affects the interpretations of the path names in that they are made relative to the working directory caused by the -C option""") # yoh: atm we only dump to console. Might adopt the same separation later on # and for consistency will call it --verbose-level as well for now # log-level is set via common_opts ATM # parser.add_argument('--log-level', # choices=('critical', 'error', 'warning', 'info', 'debug'), # dest='common_log_level', # help="""level of verbosity in log files. By default # everything, including debug messages is logged.""") #parser.add_argument('-l', '--verbose-level', # choices=('critical', 'error', 'warning', 'info', 'debug'), # dest='common_verbose_level', # help="""level of verbosity of console output. By default # only warnings and errors are printed.""") # subparsers subparsers = parser.add_subparsers() # auto detect all available interfaces and generate a function-based # API from them grp_short_descriptions = [] interface_groups = get_interface_groups() for grp_name, grp_descr, _interfaces in interface_groups: # for all subcommand modules it can find cmd_short_descriptions = [] for _intfspec in _interfaces: # turn the interface spec into an instance lgr.log(5, "Importing module %s " % _intfspec[0]) _mod = import_module(_intfspec[0], package='datalad') _intf = getattr(_mod, _intfspec[1]) cmd_name = get_cmdline_command_name(_intfspec) # deal with optional parser args if hasattr(_intf, 'parser_args'): parser_args = _intf.parser_args else: parser_args = dict(formatter_class=formatter_class) # use class description, if no explicit description is available parser_args['description'] = alter_interface_docs_for_cmdline( _intf.__doc__.strip()) # create subparser, use module suffix as cmd name subparser = subparsers.add_parser(cmd_name, add_help=False, **parser_args) # all subparser can report the version helpers.parser_add_common_opt( subparser, 'version', version='datalad %s %s\n\n%s' % (cmd_name, datalad.__version__, _license_info())) # our own custom help for all commands helpers.parser_add_common_opt(subparser, 'help') helpers.parser_add_common_opt(subparser, 'log_level') helpers.parser_add_common_opt(subparser, 'pbs_runner') # let module configure the parser _intf.setup_parser(subparser) # logger for command # configure 'run' function for this command plumbing_args = dict( func=_intf.call_from_parser, logger=logging.getLogger(_intf.__module__), subparser=subparser) if hasattr(_intf, 'result_renderer_cmdline'): plumbing_args['result_renderer'] = _intf.result_renderer_cmdline subparser.set_defaults(**plumbing_args) # store short description for later sdescr = getattr(_intf, 'short_description', parser_args['description'].split('\n')[0]) cmd_short_descriptions.append((cmd_name, sdescr)) parts[cmd_name] = subparser grp_short_descriptions.append(cmd_short_descriptions) # create command summary cmd_summary = [] for i, grp in enumerate(interface_groups): grp_descr = grp[1] grp_cmds = grp_short_descriptions[i] cmd_summary.append('\n*%s*\n' % (grp_descr,)) for cd in grp_cmds: cmd_summary.append(' - %-20s %s' % ((cd[0] + ':', textwrap.fill( cd[1].rstrip(' .'), 75, #initial_indent=' ' * 4, subsequent_indent=' ' * 8)))) # we need one last formal section to not have the trailed be # confused with the last command group cmd_summary.append('\n*General information*\n') parser.description = '%s\n%s\n\n%s' \ % (parser.description, '\n'.join(cmd_summary), textwrap.fill(dedent_docstring("""\ Detailed usage information for individual commands is available via command-specific --help, i.e.: datalad --help"""), 75, initial_indent='', subsequent_indent='')) parts['datalad'] = parser lgr.log(5, "Finished setup_parser") if return_subparsers: return parts else: return parser # yoh: arn't used # def generate_api_call(cmdlineargs=None): # parser = setup_parser() # # parse cmd args # cmdlineargs = parser.parse_args(cmdlineargs) # # convert cmdline args into API call spec # functor, args, kwargs = cmdlineargs.func(cmdlineargs) # return cmdlineargs, functor, args, kwargs def main(args=None): lgr.log(5, "Starting main(%r)", args) # PYTHON_ARGCOMPLETE_OK parser = setup_parser() try: import argcomplete argcomplete.autocomplete(parser) except ImportError: pass # parse cmd args cmdlineargs, unparsed_args = parser.parse_known_args(args) has_func = hasattr(cmdlineargs, 'func') and cmdlineargs.func is not None if unparsed_args: if has_func and cmdlineargs.func.__self__.__name__ != 'Export': lgr.error('unknown argument{}: {}'.format( 's' if len(unparsed_args) > 1 else '', unparsed_args if len(unparsed_args) > 1 else unparsed_args[0])) cmdlineargs.subparser.print_usage() sys.exit(1) else: # store all unparsed arguments cmdlineargs.datalad_unparsed_args = unparsed_args if cmdlineargs.change_path is not None: for path in cmdlineargs.change_path: chpwd(path) ret = None if cmdlineargs.pbs_runner: from .helpers import run_via_pbs from .helpers import strip_arg_from_argv from .common_args import pbs_runner as pbs_runner_opt args_ = strip_arg_from_argv(args or sys.argv, cmdlineargs.pbs_runner, pbs_runner_opt[1]) # run the function associated with the selected command run_via_pbs(args_, cmdlineargs.pbs_runner) elif has_func: if cmdlineargs.common_debug or cmdlineargs.common_idebug: # so we could see/stop clearly at the point of failure setup_exceptionhook(ipython=cmdlineargs.common_idebug) ret = cmdlineargs.func(cmdlineargs) else: # otherwise - guard and only log the summary. Postmortem is not # as convenient if being caught in this ultimate except try: ret = cmdlineargs.func(cmdlineargs) except InsufficientArgumentsError as exc: # if the func reports inappropriate usage, give help output lgr.error('%s (%s)' % (exc_str(exc), exc.__class__.__name__)) cmdlineargs.subparser.print_usage(sys.stderr) sys.exit(2) except IncompleteResultsError as exc: # we didn't get everything we wanted: still try to present what # we got as usual, but exit with an error try: if hasattr(cmdlineargs, 'result_renderer'): cmdlineargs.result_renderer(exc.results, cmdlineargs) except Exception as exc2: # could be anything really, but we do not want to blow # loud in exception handler lgr.warning("Failed to render partial results: %s", exc_str(exc2)) lgr.error('could not perform all requested actions: %s', exc_str(exc)) sys.exit(1) except Exception as exc: lgr.error('%s (%s)' % (exc_str(exc), exc.__class__.__name__)) sys.exit(1) else: # just let argparser spit out its error, since there is smth wrong parser.parse_args(args) # if that one didn't puke -- we should parser.print_usage() lgr.error("Please specify the command") sys.exit(2) try: if hasattr(cmdlineargs, 'result_renderer'): cmdlineargs.result_renderer(ret, cmdlineargs) except Exception as exc: lgr.error("Failed to render results due to %s", exc_str(exc)) sys.exit(1) lgr.log(5, "Done importing cmdline.main") datalad-0.4.1/datalad/cmdline/tests/000077500000000000000000000000001301115270200172455ustar00rootroot00000000000000datalad-0.4.1/datalad/cmdline/tests/__init__.py000066400000000000000000000007001301115270200213530ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """cmdline tests """ __docformat__ = 'restructuredtext' datalad-0.4.1/datalad/cmdline/tests/test_formatters.py000066400000000000000000000055161301115270200230530ustar00rootroot00000000000000# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """""" from six.moves import StringIO as SIO from os.path import exists from nose import SkipTest try: import formatters as fmt except ImportError: # pragma: no cover # must be running from installed version where formatters is not present # These tests can be ran only with formatters, which is outside of the # datalad module space in the root of the sourcebase if not exists('formatters.py'): raise SkipTest from ..main import setup_parser from datalad.tests.utils import ok_, assert_in, ok_startswith demo_example = """ #!/bin/sh set -e set -u # BOILERPLATE HOME=IS_MY_CASTLE #% EXAMPLE START # # A simple start (on the command line) # ==================================== # # Lorem ipsum #% datalad install http://the.world.com #% # Epilog -- with multiline rubish sdoifpwjefw # vsdokvpsokdvpsdkv spdokvpskdvpoksd # pfdsvja329u0fjpdsv sdpf9p93qk #% datalad imagine --too \\ --much \\ --too say \\ yes=no #% # The result is not comprehensible. #% #% EXAMPLE END # define shunit test cases below, or just anything desired """ def test_cmdline_example_to_rst(): # don't puke on nothing out = fmt.cmdline_example_to_rst(SIO('')) out.seek(0) ok_startswith(out.read(), '.. AUTO-GENERATED') out = fmt.cmdline_example_to_rst(SIO(''), ref='dummy') out.seek(0) assert_in('.. dummy:', out.read()) # full scale test out = fmt.cmdline_example_to_rst( SIO(demo_example), ref='mydemo') out.seek(0) assert_in('.. code-block:: sh', out.read()) def test_parser_access(): parsers = setup_parser(return_subparsers=True) # we have a bunch ok_(len(parsers) > 3) assert_in('install', parsers.keys()) def test_manpage_formatter(): addonsections = {'mytest': "uniquedummystring"} parsers = setup_parser(return_subparsers=True) for p in parsers: mp = fmt.ManPageFormatter( p, ext_sections=addonsections).format_man_page(parsers[p]) for section in ('SYNOPSIS', 'DESCRIPTION', 'OPTIONS', 'MYTEST'): assert_in('.SH {0}'.format(section), mp) assert_in('uniquedummystring', mp) def test_rstmanpage_formatter(): parsers = setup_parser(return_subparsers=True) for p in parsers: mp = fmt.RSTManPageFormatter(p).format_man_page(parsers[p]) for section in ('Synopsis', 'Description', 'Options'): assert_in('\n{0}'.format(section), mp) assert_in('{0}\n{1}'.format(p, '=' * len(p)), mp) datalad-0.4.1/datalad/cmdline/tests/test_helpers.py000066400000000000000000000052701301115270200223240ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for cmdline.helpers""" __docformat__ = 'restructuredtext' from mock import patch from nose.tools import assert_is_instance from os import mkdir from os.path import join as opj, exists, realpath from ..helpers import get_repo_instance from ..helpers import strip_arg_from_argv from ...tests.utils import ok_, eq_, assert_cwd_unchanged, ok_clean_git, \ with_tempfile, SkipTest, with_testrepos from ...support.annexrepo import AnnexRepo from ...support.gitrepo import GitRepo from ...utils import chpwd, getpwd @assert_cwd_unchanged @with_testrepos('^basic_git$', flavors=['clone']) def test_get_repo_instance_git(path): # get instance from path repo = get_repo_instance(path, GitRepo) assert_is_instance(repo, GitRepo) eq_(realpath(repo.path), realpath(path)) old_pwd = getpwd() # get instance from current dir chpwd(path) repo = get_repo_instance() assert_is_instance(repo, GitRepo) eq_(realpath(repo.path), realpath(path)) # get instance from current subdir new_subdir = opj(path, "subdir") mkdir(new_subdir) chpwd(new_subdir) eq_(new_subdir, getpwd()) repo = get_repo_instance() assert_is_instance(repo, GitRepo) eq_(realpath(repo.path), realpath(path)) chpwd(old_pwd) @assert_cwd_unchanged @with_testrepos('.*annex.*', flavors=['clone']) def test_get_repo_instance_annex(path): # get instance from path repo = get_repo_instance(path, AnnexRepo) assert_is_instance(repo, AnnexRepo) eq_(realpath(repo.path), realpath(path)) old_pwd = getpwd() # get instance from current dir chpwd(path) repo = get_repo_instance() assert_is_instance(repo, AnnexRepo) eq_(realpath(repo.path), realpath(path)) # get instance from current subdir new_subdir = opj(path, "subdir") mkdir(new_subdir) chpwd(new_subdir) eq_(new_subdir, getpwd()) repo = get_repo_instance() assert_is_instance(repo, AnnexRepo) eq_(realpath(repo.path), realpath(path)) chpwd(old_pwd) def test_strip_arg_from_argv(): eq_(strip_arg_from_argv(['-s', 'value'], 'value', ('-s',)), []) eq_(strip_arg_from_argv(['-s', 'value'], 'value', ('-s', '--long-s')), []) eq_(strip_arg_from_argv( ['cmd', '-s', 'value', '--more'], 'value', ('-s', '--long-s')), ['cmd', '--more']) datalad-0.4.1/datalad/cmdline/tests/test_main.py000066400000000000000000000107741301115270200216130ustar00rootroot00000000000000# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test functioning of the datalad main cmdline utility """ import re import sys from six.moves import StringIO from mock import patch import datalad from ..main import main from datalad.tests.utils import assert_equal, assert_raises, in_, ok_startswith from datalad.tests.utils import assert_in from datalad.tests.utils import assert_re_in def run_main(args, exit_code=0, expect_stderr=False): """Run main() of the datalad, do basic checks and provide outputs Parameters ---------- args : list List of string cmdline arguments to pass exit_code : int Expected exit code. Would raise AssertionError if differs expect_stderr : bool or string Either to expect stderr output. If string -- match Returns ------- stdout, stderr strings Output produced """ with patch('sys.stderr', new_callable=StringIO) as cmerr: with patch('sys.stdout', new_callable=StringIO) as cmout: with assert_raises(SystemExit) as cm: main(args) assert_equal(cm.exception.code, exit_code) # exit code must be 0 stdout = cmout.getvalue() stderr = cmerr.getvalue() if expect_stderr is False: assert_equal(stderr, "") elif expect_stderr is True: # do nothing -- just return pass else: # must be a string assert_equal(stderr, expect_stderr) return stdout, stderr # TODO: switch to stdout for --version output def test_version(): stdout, stderr = run_main(['--version'], expect_stderr=True) # and output should contain our version, copyright, license # https://hg.python.org/cpython/file/default/Doc/whatsnew/3.4.rst#l1952 out = stdout if sys.version_info >= (3, 4) else stderr ok_startswith(out, 'datalad %s\n' % datalad.__version__) in_("Copyright", out) in_("Permission is hereby granted", out) def test_help_np(): stdout, stderr = run_main(['--help-np']) # Let's extract section titles: # enough of bin/datalad and .tox/py27/bin/datalad -- guarantee consistency! ;) ok_startswith(stdout, 'Usage: datalad') # Sections start/end with * if ran under DATALAD_HELP2MAN mode sections = [l[1:-1] for l in filter(re.compile('^\*.*\*$').match, stdout.split('\n'))] # but order is still not guaranteed (dict somewhere)! TODO # see https://travis-ci.org/datalad/datalad/jobs/80519004 # thus testing sets assert_equal(set(sections), {'Commands for dataset operations', 'Commands for meta data handling', 'Miscellaneous commands', 'General information', 'Global options'}) # none of the lines must be longer than 80 chars # TODO: decide on create-sibling and possibly # rewrite-urls long_lines = ["%d %s" % (len(l), l) for l in stdout.split('\n') if len(l) > 80 and '{' not in l # on nd70 summary line is unsplit ] if long_lines: raise AssertionError( "Following lines in --help output were longer than 80 chars:\n%s" % '\n'.join(long_lines) ) def test_usage_on_insufficient_args(): stdout, stderr = run_main(['install'], exit_code=2, expect_stderr=True) ok_startswith(stderr, 'usage:') def test_subcmd_usage_on_unknown_args(): stdout, stderr = run_main(['get', '--murks'], exit_code=1) in_('get', stdout) def check_incorrect_option(opts, err_str): stdout, stderr = run_main((sys.argv[0],) + opts, expect_stderr=True, exit_code=2) out = stdout + stderr assert_in("usage: ", out) assert_re_in(err_str, out, match=False) def test_incorrect_options(): # apparently a bit different if following a good one so let's do both err_invalid = "error: (invalid|too few arguments)" yield check_incorrect_option, ('--buga',), err_invalid yield check_incorrect_option, ('--dbg', '--buga'), err_invalid err_insufficient = err_invalid # "specify" yield check_incorrect_option, ('--dbg',), err_insufficient yield check_incorrect_option, tuple(), err_insufficient datalad-0.4.1/datalad/config.py000066400000000000000000000461711301115270200163200ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """ """ from datalad.cmd import Runner from datalad.dochelpers import exc_str import re import os from os.path import join as opj, exists cfg_kv_regex = re.compile(r'(^.*)\n(.*)$', flags=re.MULTILINE) cfg_section_regex = re.compile(r'(.*)\.[^.]+') cfg_sectionoption_regex = re.compile(r'(.*)\.([^.]+)') _where_reload_doc = """ where : {'dataset', 'local', 'global'}, optional Indicator which configuration file to modify. 'dataset' indicates the persistent configuration in .datalad/config of a dataset; 'local' the configuration of a dataset's Git repository in .git/config; 'global' refers to the general configuration that is not specific to a single repository (usually in $USER/.gitconfig). reload : bool Flag whether to reload the configuration from file(s) after modification. This can be disable to make multiple sequential modifications slightly more efficient.""".lstrip() def _where_reload(obj): """Helper decorator to simplify providing repetitive docstring""" obj.__doc__ = obj.__doc__ % _where_reload_doc return obj def _parse_gitconfig_dump(dump, store, replace): if replace: # if we want to replace existing values in the store # collect into a new dict and `update` the store at the # end. This way we get the desired behavior of multi-value # keys, but only for the current source dct = {} else: # if we don't want to replace value, perform the multi-value # preserving addition on the existing store right away dct = store for line in dump.split('\0'): if not line: continue k, v = cfg_kv_regex.match(line).groups() present_v = dct.get(k, None) if present_v is None: dct[k] = v else: if isinstance(present_v, tuple): dct[k] = present_v + (v,) else: dct[k] = (present_v, v) if replace: store.update(dct) return store def _parse_env(store): dct = {} for k in os.environ: if not k.startswith('DATALAD_'): continue dct[k.replace('_', '.').lower()] = os.environ[k] store.update(dct) return store def anything2bool(val): if hasattr(val, 'lower'): val = val.lower() if val in {"off", "no", "false", "0"} or not bool(val): return False elif val in {"on", "yes", "true", True} \ or (hasattr(val, 'isdigit') and val.isdigit() and int(val)) \ or isinstance(val, int) and val: return True else: raise TypeError( "Got value %s which could not be interpreted as a boolean" % repr(val)) class ConfigManager(object): """Thin wrapper around `git-config` with support for a dataset configuration. The general idea is to have an object that is primarily used to read/query configuration option. Upon creation, current configuration is read via one (or max two, in the case of the presence of dataset-specific configuration) calls to `git config`. If this class is initialized with a Dataset instance, it supports reading and writing configuration from ``.datalad/config`` inside a dataset too. This file is committed to Git and hence useful to ship certain configuration items with a dataset. The API aims to provide the most significant read-access API of a dictionary, the Python ConfigParser, and GitPython's config parser implementations. This class is presently not capable of efficiently writing multiple configurations items at once. Instead, each modification results in a dedicated call to `git config`. This author thinks this is OK, as he cannot think of a situation where a large number of items need to be written during normal operation. If such need arises, various solutions are possible (via GitPython, or an independent writer). Any DATALAD_* environment variable is also presented as a configuration item. Settings read from environment variables are not stored in any of the configuration file, but are read dynamically from the environment at each `reload()` call. Their values take precedence over any specification in configuration files. Parameters ---------- dataset : Dataset, optional If provided, all `git config` calls are executed in this dataset's directory. Moreover, any modifications are, by default, directed to this dataset's configuration file (which will be created on demand) dataset_only : bool If True, configuration items are only read from a datasets persistent configuration file, if any present (the one in ``.datalad/config``, not ``.git/config``). """ def __init__(self, dataset=None, dataset_only=False): # store in a simple dict # no subclassing, because we want to be largely read-only, and implement # config writing separately self._store = {} self._dataset = dataset self._dataset_only = dataset_only # Since configs could contain sensitive information, to prevent # any "facilitated" leakage -- just disable loging of outputs for # this runner run_kwargs = dict(log_outputs=False) if dataset is not None: # make sure we run the git config calls in the dataset # to pick up the right config files run_kwargs['cwd'] = dataset.path self._runner = Runner(**run_kwargs) self.reload() def reload(self): """Reload all configuration items from the configured sources""" self._store = {} # 2-step strategy: # - load datalad dataset config from dataset # - load git config from all supported by git sources # in doing so we always stay compatible with where Git gets its # config from, but also allow to override persistent information # from dataset locally or globally if self._dataset: # now any dataset config dscfg_fname = opj(self._dataset.path, '.datalad', 'config') if exists(dscfg_fname): stdout, stderr = self._run(['-z', '-l', '--file', dscfg_fname], log_stderr=True) # overwrite existing value, do not amend to get multi-line # values self._store = _parse_gitconfig_dump( stdout, self._store, replace=False) if not self._dataset_only: stdout, stderr = self._run(['-z', '-l'], log_stderr=True) self._store = _parse_gitconfig_dump( stdout, self._store, replace=True) # override with environment variables self._store = _parse_env(self._store) @_where_reload def obtain(self, var, default=None, dialog_type=None, valtype=None, store=False, where=None, reload=True, **kwargs): """ Convenience method to obtain settings interactively, if needed A UI will be used to ask for user input in interactive sessions. Questions to ask, and additional explanations can be passed directly as arguments, or retrieved from a list of pre-configured items. Additionally, this method allows for type conversion and storage of obtained settings. Both aspects can also be pre-configured. Parameters ---------- var : str Variable name including any section like `git config` expects them, e.g. 'core.editor' default : any type In interactive sessions and if `store` is True, this default value will be presented to the user for confirmation (or modification). In all other cases, this value will be silently assigned unless there is an existing configuration setting. dialog_type : {'question', 'yesno', None} Which dialog type to use in interactive sessions. If `None`, pre-configured UI options are used. store : bool Whether to store the obtained value (or default) %s `**kwargs` Additional arguments for the UI function call, such as a question `text`. """ # do local import, as this module is import prominently and the # could theroetically import all kind of weired things for type # conversion from datalad.interface.common_cfg import definitions as cfg_defs # fetch what we know about this variable cdef = cfg_defs.get(var, {}) # type conversion setup if valtype is None and 'type' in cdef: valtype = cdef['type'] if valtype is None: valtype = lambda x: x # any default? if default is None and 'default' in cdef: default = cdef['default'] _value = None if var in self: # nothing needs to be obtained, it is all here already _value = self[var] elif store is False and default is not None: # nothing will be stored, and we have a default -> no user confirmation # we cannot use logging, because we want to use the config to confiugre # the logging #lgr.debug('using default {} for config setting {}'.format(default, var)) _value = default if _value is not None: # we got everything we need and can exit early try: return valtype(_value) except Exception as e: raise ValueError( "value '{}' of existing configuration for '{}' cannot be " "converted to the desired type '{}' ({})".format( _value, var, valtype, exc_str(e))) # now we need to try to obtain something from the user from datalad.ui import ui # configure UI dialog_opts = kwargs if dialog_type is None: # no override # check for common knowledge on how to obtain a value if 'ui' in cdef: dialog_type = cdef['ui'][0] # pull standard dialog settings dialog_opts = cdef['ui'][1] # update with input dialog_opts.update(kwargs) if (not ui.is_interactive or dialog_type is None) and default is None: raise RuntimeError( "cannot obtain value for configuration item '{}', " "not preconfigured, no default, no UI available".format(var)) if not hasattr(ui, dialog_type): raise ValueError("UI '{}' does not support dialog type '{}'".format( ui, dialog_type)) # configure storage destination, if needed if store: if where is None and 'destination' in cdef: where = cdef['destination'] if where is None: raise ValueError( "request to store configuration item '{}', but no " "storage destination specified".format(var)) # obtain via UI dialog = getattr(ui, dialog_type) _value = dialog(default=default, **dialog_opts) if _value is None: # we got nothing if default is None: raise RuntimeError( "could not obtain value for configuration item '{}', " "not preconfigured, no default".format(var)) # XXX maybe we should return default here, even it was returned # from the UI -- if that is even possible # execute type conversion before storing to check that we got # something that looks like what we want try: value = valtype(_value) except Exception as e: raise ValueError( "cannot convert user input `{}` to desired type ({})".format( _value, exc_str(e))) # XXX we could consider "looping" until we have a value of proper # type in case of a user typo... if store: # store value as it was before any conversion, needs to be str # anyway # needs string conversion nevertheless, because default could come # in as something else self.add(var, '{}'.format(_value), where=where, reload=reload) return value # # Compatibility with dict API # def __len__(self): return len(self._store) def __getitem__(self, key): return self._store.__getitem__(key) def __contains__(self, key): return self._store.__contains__(key) def keys(self): """Returns list of configuration item names""" return self._store.keys() def get(self, key, default=None): """D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.""" return self._store.get(key, default) # # Compatibility with ConfigParser API # def sections(self): """Returns a list of the sections available""" return list(set([cfg_section_regex.match(k).group(1) for k in self._store])) def options(self, section): """Returns a list of options available in the specified section.""" opts = [] for k in self._store: sec, opt = cfg_sectionoption_regex.match(k).groups() if sec == section: opts.append(opt) return opts def has_section(self, section): """Indicates whether a section is present in the configuration""" for k in self._store: if k.startswith(section): return True return False def has_option(self, section, option): """If the given section exists, and contains the given option""" for k in self._store: sec, opt = cfg_sectionoption_regex.match(k).groups() if sec == section and opt == option: return True return False def getint(self, section, option): """A convenience method which coerces the option value to an integer""" return int(self.get_value(section, option)) def getbool(self, section, option, default=None): """A convenience method which coerces the option value to a bool Values "on", "yes", "true" and any int!=0 are considered True Values which evaluate to bool False, "off", "no", "false" are considered False TypeError is raised for other values. """ val = self.get_value(section, option, default=default) return anything2bool(val) def getfloat(self, section, option): """A convenience method which coerces the option value to a float""" return float(self.get_value(section, option)) # this is a hybrid of ConfigParser and dict API def items(self, section=None): """Return a list of (name, value) pairs for each option Optionally limited to a given section. """ if section is None: return self._store.items() return [(k, v) for k, v in self._store.items() if cfg_section_regex.match(k).group(1) == section] # # Compatibility with GitPython's ConfigParser # def get_value(self, section, option, default=None): """Like `get()`, but with an optional default value If the default is not None, the given default value will be returned in case the option did not exist. This behavior immitates GitPython's config parser. """ try: return self['.'.join((section, option))] except KeyError as e: # this strange dance is needed because gitpython does it this way if default is not None: return default else: raise e # # Modify configuration (proxy respective git-config call) # @_where_reload def _run(self, args, where=None, reload=False, **kwargs): """Centralized helper to run "git config" calls Parameters ---------- args : list Arguments to pass for git config %s **kwargs Keywords arguments for Runner's call """ if where: args = self._get_location_args(where) + args out = self._runner.run(['git', 'config'] + args, **kwargs) if reload: self.reload() return out def _get_location_args(self, where, args=None): if args is None: args = [] cfg_labels = ('dataset', 'local', 'global') if where not in cfg_labels: raise ValueError( "unknown configuration label '{}' (not in {})".format( where, cfg_labels)) if where == 'dataset': if not self._dataset: raise ValueError( 'ConfigManager cannot store to configuration to dataset, none specified') # create an empty config file if none exists, `git config` will # fail otherwise dscfg_dirname = opj(self._dataset.path, '.datalad') dscfg_fname = opj(dscfg_dirname, 'config') if not exists(dscfg_dirname): os.makedirs(dscfg_dirname) if not exists(dscfg_fname): open(dscfg_fname, 'w').close() args.extend(['--file', opj(self._dataset.path, '.datalad', 'config')]) elif where == 'global': args.append('--global') elif where == 'local': args.append('--local') return args @_where_reload def add(self, var, value, where='dataset', reload=True): """Add a configuration variable and value Parameters ---------- var : str Variable name including any section like `git config` expects them, e.g. 'core.editor' value : str Variable value %s""" self._run(['--add', var, value], where=where, reload=reload, log_stderr=True) @_where_reload def rename_section(self, old, new, where='dataset', reload=True): """Rename a configuration section Parameters ---------- old : str Name of the section to rename. new : str Name of the section to rename to. %s""" self._run(['--rename-section', old, new], where=where, reload=reload) @_where_reload def remove_section(self, sec, where='dataset', reload=True): """Rename a configuration section Parameters ---------- sec : str Name of the section to remove. %s""" self._run(['--remove-section', sec], where=where, reload=reload) @_where_reload def unset(self, var, where='dataset', reload=True): """Remove all occurrences of a variable Parameters ---------- var : str Name of the variable to remove %s""" # use unset all as it is simpler for now self._run(['--unset-all', var], where=where, reload=reload) datalad-0.4.1/datalad/consts.py000066400000000000000000000030071301115270200163530ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """constants for datalad """ from os.path import join from os.path import expanduser # directory containing prepared metadata of a dataset repository: HANDLE_META_DIR = ".datalad" CRAWLER_META_DIR = join(HANDLE_META_DIR, 'crawl') CRAWLER_META_CONFIG_FILENAME = 'crawl.cfg' CRAWLER_META_CONFIG_PATH = join(CRAWLER_META_DIR, CRAWLER_META_CONFIG_FILENAME) CRAWLER_META_VERSIONS_DIR = join(CRAWLER_META_DIR, 'versions') # TODO: RENAME THIS UGLINESS? CRAWLER_META_STATUSES_DIR = join(CRAWLER_META_DIR, 'statuses') # Make use of those in datalad.metadata METADATA_DIR = join(HANDLE_META_DIR, 'meta') METADATA_FILENAME = 'meta.json' ARCHIVES_SPECIAL_REMOTE = 'datalad-archives' DATALAD_SPECIAL_REMOTE = 'datalad' DATALAD_GIT_DIR = join('.git', 'datalad') ARCHIVES_TEMP_DIR = join(DATALAD_GIT_DIR, 'tmp', 'archives') DATASETS_TOPURL = "http://datasets.datalad.org/" # Centralized deployment LOCAL_CENTRAL_PATH = join(expanduser('~'), 'datalad') WEB_META_LOG = join(DATALAD_GIT_DIR, 'logs') WEB_META_DIR = join(DATALAD_GIT_DIR, 'metadata') WEB_HTML_DIR = join(DATALAD_GIT_DIR, 'web') # Format to use for time stamps TIMESTAMP_FMT = "%Y-%m-%dT%H:%M:%S%z" datalad-0.4.1/datalad/crawler/000077500000000000000000000000001301115270200161275ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/__init__.py000066400000000000000000000007701301115270200202440ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Crawling of external resources (e.g. websites) to get/update datasets """ __docformat__ = 'restructuredtext' datalad-0.4.1/datalad/crawler/base.py000066400000000000000000000007701301115270200174170ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Crawling of external resources (e.g. websites) to get/update datasets """ __docformat__ = 'restructuredtext' datalad-0.4.1/datalad/crawler/db.py000066400000000000000000000122541301115270200170720ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Module to provide DB for storing crawling/available information""" from abc import ABCMeta, abstractmethod, abstractproperty from six import itervalues import json class URLDB(object): """Database collating urls for the content across all datasets Schema: TODO, but needs for sure - URL (only "public" or internal as for content from archives, or that separate table?) - common checksums which we might use/rely upon (MD5, SHA1, SHA256, SHA512) - last_checked (if online) - last_verified (when verified to contain the content according to the checksums Allow to query by any known checksum """ __metaclass__ = ABCMeta @abstractmethod def __contains__(self, url): """Return True if DB knows about this URL""" pass @abstractmethod def __getitem__(self, url): """Given URL, return file names where it was downloaded""" pass class JsonURLDB(URLDB): """Mimic original dict-based urldb which was dumped to a json file, but which also had "public_incoming" mapping to map from incoming file names to "public". Following changes would be done programmatically now and we will track only "incoming" So internally it is just a dictionary of "fpath: url_info" where url_info is a dict containing mtime, size, and URL """ __db_version__ = '0.1' def __init__(self, data={}): self._data = data.copy() self._urls = set() # set of known urls self._referenced_files = set() def urls(self): if not self._urls: self._urls = set(x['url'] for x in itervalues(self._data)) return self._urls def __contains__(self, fpath): return fpath in self._data def get(self, fpath, *args, **kwargs): return self._data.get(fpath, *args, **kwargs) def __getitem__(self, fpath): self._referenced_files.add(fpath) return self.get(fpath) def __setitem__(self, fpath, v): url = v.get('url') # must have URL self._urls.add(url) self._referenced_files.add(fpath) self._data[fpath] = v def get_abandoned_files(self): """Return files which were not referenced (set or get) in the life-time of this DB """ return set(self._data).difference(self._referenced_files) def prune(self, fpaths=None): """Prune provided or abandoned (if fpaths is None) files entries """ if fpaths is None: fpaths = self.get_abandoned_files() for fpath in fpaths: if fpath in self._data: del self._data[fpath] # reset _urls self._urls = None return self # for easier chaining def save(self, fpath): """Save DB as a JSON file """ db = { 'version': self.__db_version__, 'data': self._data } with open(fpath, 'w') as f: json.dump(db, f, indent=2, sort_keys=True, separators=(',', ': ')) return self # for easier chaining def load(self, fpath): with open(fpath) as f: db = json.load(f) if db.get('version') != self.__db_version__: raise ValueError("Loaded db from %s is of unsupported version %s. " "Currently supported: %s" % (fpath, db.get('version'), self.__db_version__)) self.data = db.get('data') return self # for easier chaining from os.path import lexists from ..utils import updated from ..support.network import SimpleURLStamper # TODO: formatlize above DB into API so we could have various implementations to give to DBNode class DBNode(object): def __init__(self, db, url_stamper=None): self.db = db if url_stamper is None: # TODO: sucks since we would need to inform about mode -- full/fast/relaxed url_stamper = SimpleURLStamper() self._url_stamper = url_stamper def skip_known_url(self, data): if data['url'] in self.db.urls: return yield data def skip_existing_file(self, data): # TODO: Hm... we could have file known AND existing or not.... filename = data['filename'] # TODO: filepath probably?? if filename in self.db and lexists(filename): return yield data def instruct_to_remove_abandoned(self, data): # data is not used for filename in self.db.get_abandoned_files(): yield updated(data, {'filename': filename, 'fileaction': 'remove' }) def check_url(self, data): """Check URL for being modified, etc""" url = data['url'] filename = data['filename'] # get information about that url url_stamp = self._url_stamper(url) old_url_stamp = self.db.get(filename) yield data datalad-0.4.1/datalad/crawler/dbs/000077500000000000000000000000001301115270200166775ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/dbs/__init__.py000066400000000000000000000007151301115270200210130ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """DB support for the crawler """ __docformat__ = 'restructuredtext' datalad-0.4.1/datalad/crawler/dbs/base.py000066400000000000000000000157301301115270200201710ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Base classes for various DBs """ import os import json from abc import ABCMeta, abstractmethod, abstractproperty from os.path import join as opj, exists, lexists, realpath, basename, dirname from os.path import normpath from os.path import isabs from ...utils import auto_repr from ...utils import find_files from ...consts import HANDLE_META_DIR from ...support.annexrepo import AnnexRepo from ...support.gitrepo import GitRepo import logging lgr = logging.getLogger('datalad.crawler.dbs') __docformat__ = 'restructuredtext' @auto_repr class JsonBaseDB(object): """ Base class for DBs which would store to JSON files """ __metaclass__ = ABCMeta def __init__(self, repo, name=None): #super(JsonBaseDB, self).__init__() self.repo = repo self.name = name self._filepath = None self._loaded = None self.__db = None def _assure_loaded(self): """Make it lazy loading/creation so we get actual active branch where it is used """ if self._filepath is not None: return self._filepath = opj(realpath(self.repo.path), self.__class__.__crawler_subdir__, (self.name or self.repo.get_active_branch()) + '.json') if lexists(self._filepath): self.load() self._loaded = True else: self.__db = self.get_empty_db() self._loaded = False # so we don't bother saving an empty one if was not loaded @property def _db(self): self._assure_loaded() return self.__db def load(self): self._assure_loaded() with open(self._filepath) as f: json_db = json.load(f) # return f.read().strip() self.__db = self._get_loaded_db(json_db) def save(self): if self._filepath is None: # nothing to do return db = self._get_db_to_save() if (not self._loaded) and (db == self.get_empty_db()): lgr.debug("DB %s which we defaulted to found to be empty, not saving" % self) return d = dirname(self._filepath) if not exists(d): os.makedirs(d) lgr.debug("Writing %s to %s" % (self.__class__.__name__, self._filepath)) with open(self._filepath, 'w') as f: json.dump(db, f, indent=2, sort_keys=True, separators=(',', ': ')) # stage to be committed: self.repo.add(self._filepath, git=True) @property def db_version(self): return self._db['db_version'] def get_empty_db(self): """Return default empty DB. Relies on subclasses specific""" db = self._get_empty_db() db['db_version'] = self.__class__.__version__ return db @abstractmethod def _get_empty_db(self): pass @abstractmethod def _get_loaded_db(self, db): """Given a DB loaded from a file, prepare it for being used """ pass @abstractmethod def _get_db_to_save(self): """Return DB to be saved as JSON file """ pass @auto_repr class FileStatusesBaseDB(object): """Base class for DBs to monitor status of the files """ def __init__(self, annex, track_queried=True): """ Parameters ---------- annex : AnnexRepo Annex repository which will be consulted on the size and full path track_queried : bool, optional To track what file paths were queried """ # with all the multiple inheritance smth is not working out as it should # super(FileStatusesBaseDB, self).__init__() self.annex = annex # which file paths were referred self._track_queried = track_queried self._queried_filepaths = set() @property def track_queried(self): return self._track_queried @property def queried_filepaths(self): return self._queried_filepaths def _get_filepath(self, fpath): if isabs(fpath): return normpath(fpath) else: return normpath(opj(self.annex.path, fpath)) # TODO: think if default should be provided def get(self, fpath): """Given a file (under annex) relative path, return its status record annex information about size, etc. might be used if load is not available Parameters ---------- fpath: str Path (relative to the top of the repo) of the file of which to get stats """ filepath = self._get_filepath(fpath) if self._track_queried: self._queried_filepaths.add(filepath) return self._get(filepath) def _get(self, filepath): raise NotImplementedError("must be defined in subclasses") def set(self, fpath, status=None): """Set a new status for the fpath. If status is None, get from available file """ filepath = self._get_filepath(fpath) if self._track_queried: self._queried_filepaths.add(filepath) self._set(filepath, status) def _set(self, filepath, status): raise NotImplementedError("must be defined in subclasses") def remove(self, fpath): self._remove(self._get_filepath(fpath)) def _remove(self, filepath): raise NotImplementedError("must be defined in subclasses") def is_different(self, fpath, status, url=None): """Return True if file pointed by fpath is newer in status """ # TODO: make use of URL -- we should validate that url is among those associated # with the file old_status = self.get(fpath) if status.filename and not old_status.filename: old_status.filename = basename(fpath) return old_status != status def get_obsolete(self): """Returns full paths for files which weren't queried, thus must have been deleted Note that it doesn't track across branches, etc. """ if not self._track_queried: raise RuntimeError("Cannot determine which files were removed since track_queried was set to False") obsolete = [] # those aren't tracked by annexificator datalad_path = opj(self.annex.path, HANDLE_META_DIR) for fpath in find_files('.*', topdir=self.annex.path): filepath = self._get_filepath(fpath) if filepath.startswith(datalad_path): continue if fpath not in self._queried_filepaths: obsolete.append(filepath) return obsolete def reset(self): """Reset internal state, e.g. about known queried filed paths""" self._queried_filepaths = set() datalad-0.4.1/datalad/crawler/dbs/files.py000066400000000000000000000121351301115270200203550ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """File-based "DB" which uses file modification times to deduce if new version is available """ import os from os.path import join as opj, exists, lexists, islink, realpath, sep from ...dochelpers import exc_str from ...support.status import FileStatus from ...support.exceptions import CommandError from ...utils import auto_repr from ...utils import swallow_logs from ...consts import CRAWLER_META_STATUSES_DIR from .base import JsonBaseDB, FileStatusesBaseDB import logging lgr = logging.getLogger('datalad.crawler.dbs') __docformat__ = 'restructuredtext' __all__ = ['PhysicalFileStatusesDB'] # # Concrete implementations # @auto_repr class PhysicalFileStatusesDB(FileStatusesBaseDB): """Non-persistent DB based on file attributes It uses file modification times and size as known to annex to deduce if a new version is available In general, this should not be used since neither git nor annex stores any files meta-information (besides mode and size), so mtime would get lost while switching the branches and dropping the load """ def _get(self, filepath): if not lexists(filepath): return None # I wish I could just test using filesystem stats but that would not # be reliable, and also file might not even be here. # File might be under git, not annex so then we would need to assess size filestat = os.lstat(filepath) try: with swallow_logs(): info = self.annex.info(filepath, batch=True) size = info['size'] except (CommandError, TypeError) as exc: # must be under git or a plain file lgr.debug("File %s must be not under annex, since info failed: %s" % (filepath, exc_str(exc))) size = filestat.st_size # deduce mtime from the file or a content which it points to. Take the oldest (I wonder # if it would bite ;) XXX) mtime = filestat.st_mtime if islink(filepath): filepath_ = realpath(filepath) # symlinked to if exists(filepath_): mtime_ = os.stat(filepath_).st_mtime mtime = min(mtime_, mtime) return FileStatus( size=size, mtime=mtime ) def _set(self, filepath, status): # this DB doesn't implement much of it, besides marking internally that we do care about this file pass def save(self): # nothing for us to do but JsonFileStatusesDB has it so let's keep it common pass def _remove(self, filepath): pass @auto_repr class JsonFileStatusesDB(JsonBaseDB, PhysicalFileStatusesDB): """Persistent DB to store information about files' size/mtime/filename in a JSON file """ __version__ = 1 __crawler_subdir__ = CRAWLER_META_STATUSES_DIR def __init__(self, annex, track_queried=True, name=None): PhysicalFileStatusesDB.__init__(self, annex, track_queried=track_queried) JsonBaseDB.__init__(self, annex, name=name) # # Defining abstract methods implementations # def _get_empty_db(self): return {'files': {}} def _get_loaded_db(self, db): """Given a DB loaded from a file, prepare it for being used """ assert (set(db.keys()) == {'db_version', 'files'}) return db def _get_db_to_save(self): """Return DB to be saved as JSON file """ return self._db def _get_fpath(self, filepath): assert (filepath.startswith(self.annex.path)) fpath = filepath[len(self.annex.path.rstrip(sep)) + 1:] return fpath def _get_fileattributes_status(self, fpath): filepath = self._get_filepath(fpath) return PhysicalFileStatusesDB._get(self, filepath) def _get(self, filepath): # TODO: may be avoid this all fpath -> filepath -> fpath? fpath = self._get_fpath(filepath) files = self._db['files'] if fpath not in files: return None return FileStatus(**files[fpath]) # TODO: get URL all the way here? def _set(self, filepath, status): fpath = self._get_fpath(filepath) if status is None: status_dict = {} # get it from the locally available file # status = PhysicalFileStatusesDB._get(self, filepath) # NOPE since then generated files would keep changing else: status_dict = {f: getattr(status, f) for f in ('size', 'mtime', 'filename') if getattr(status, f) is not None} self._db['files'][fpath] = status_dict def _remove(self, filepath): fpath = self._get_fpath(filepath) if fpath in self._db['files']: self._db['files'].pop(fpath) datalad-0.4.1/datalad/crawler/dbs/tests/000077500000000000000000000000001301115270200200415ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/dbs/tests/__init__.py000066400000000000000000000006051301115270200221530ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## datalad-0.4.1/datalad/crawler/dbs/tests/test_files.py000066400000000000000000000100601301115270200225510ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import os from os.path import join as opj, curdir, sep from ..files import PhysicalFileStatusesDB, JsonFileStatusesDB from ....tests.utils import with_tree from ....tests.utils import assert_equal from ....tests.utils import assert_false from ....tests.utils import assert_true from ....tests.utils import chpwd from ....support.annexrepo import AnnexRepo @with_tree( tree={'file1.txt': 'load1', '2git': 'load', 'd': { 'file2.txt': 'load2' }}) def _test_AnnexDB(cls, path): filepath1 = opj(path, 'file1.txt') filep2 = opj('d', 'file2.txt') filepath2 = opj(path, filep2) annex = AnnexRepo(path, create=True) # PhysicalFileStatusesDB relies on information in annex so files # must be committed first annex.add('file1.txt') annex.commit("initial commit") db = cls(annex=annex) def set_db_status_from_file(fpath): """To test JsonFileStatusesDB, we need to keep updating the status stored""" if cls is JsonFileStatusesDB: # we need first to set the status db.set(fpath, db._get_fileattributes_status(fpath)) set_db_status_from_file('file1.txt') status1 = db.get('file1.txt') assert(status1.size) status1_ = db.get('file1.txt') assert_equal(status1, status1_) assert_false(db.is_different('file1.txt', status1)) assert_false(db.is_different('file1.txt', status1_)) # even if we add a filename specification status1_.filename = 'file1.txt' assert_false(db.is_different('file1.txt', status1_)) status1_.filename = 'different.txt' assert_false(db.is_different('file1.txt', status1_)) os.unlink(filepath1) # under annex- - we don't have unlock yet and thus can't inplace augment with open(filepath1, 'a') as f: f.write('+') set_db_status_from_file(filepath1) assert(db.is_different('file1.txt', status1)) # we should be able to get status of files out and inside of git set_db_status_from_file('2git') status_git1 = db.get('2git') annex.add('2git', git=True) annex.commit("added 2git") assert_equal(db.get('2git'), status_git1) # we should be able to get status of files with relative path to top dir and abs path set_db_status_from_file(filep2) status2 = db.get(filep2) status2_full = db.get(filepath2) assert_equal(status2, status2_full) # TODO? what about relative to curdir?? #with chpwd(opj(path, 'd')): # status2_dir = db.get('./file2.txt') # assert_equal(status2, status2_dir) # since we asked about each file we added to DB/annex -- none should be # known as "deleted" assert_equal(db.get_obsolete(), []) # Possibly save its state for persistent storage #import pdb; pdb.set_trace() db.save() # but, if we create another DB which wasn't queried yet db2 = cls(annex=annex) # all files should be returned assert_equal( set(db2.get_obsolete()), {opj(path, p) for p in ['file1.txt', filep2, '2git']}) # and if we query one, it shouldn't be listed as deleted any more status2_ = db2.get(filep2) assert_equal(status2, status2_) assert_equal( set(db2.get_obsolete()), {opj(path, p) for p in ['file1.txt', '2git']}) # and if we queried with ./ prefix, should still work db2.get(curdir + sep + 'file1.txt') assert_equal( set(db2.get_obsolete()), {opj(path, p) for p in ['2git']}) # and if we queried with a full path, should still work db2.get(opj(path, '2git')) assert_equal(db2.get_obsolete(), []) def test_AnnexDBs(): for cls in (PhysicalFileStatusesDB, JsonFileStatusesDB,): yield _test_AnnexDB, clsdatalad-0.4.1/datalad/crawler/dbs/versions.py000066400000000000000000000061051301115270200211230ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Simple helper to store/retrieve information about the version of the last scraped """ from collections import OrderedDict from six import iteritems from ...utils import auto_repr from ...consts import CRAWLER_META_VERSIONS_DIR from .base import JsonBaseDB import logging lgr = logging.getLogger('datalad.crawler.dbs') __docformat__ = 'restructuredtext' @auto_repr class SingleVersionDB(JsonBaseDB): """ Simple helper to store/retrieve information about the last scraped version Since we do not expect many changes being done to this DB, it also saves its state into the file upon any change """ __version__ = 1 __crawler_subdir__ = CRAWLER_META_VERSIONS_DIR # # Defining abstract methods implementations # def _get_empty_db(self): return {'version': None, 'versions': OrderedDict()} def _get_loaded_db(self, db): """Given a DB loaded from a file, prepare it for being used """ assert (set(db.keys()) == {'db_version', 'version', 'versions'}) # no compatibility layers for now assert (db['db_version'] == self.__class__.__version__) db['versions'] = OrderedDict(db['versions']) return db def _get_db_to_save(self): """Return DB to be saved as JSON file """ db = self._db.copy() # since we have it ordered, let's store it as a list of items db['versions'] = list(db['versions'].items()) return db # # Custom properties and methods # @property def version(self): return self._db['version'] @version.setter def version(self, v): self._db['version'] = v self.save() @property def versions(self): return self._db['versions'] # @versions.setter # def versions(self, v): # self._db['versions'] = v # self.save() def update_versions(self, new_versions): """Update known versions with new information """ versions = self._db['versions'] for new_version, new_fpaths in iteritems(new_versions): if new_version not in versions: # TODO: check that it is newer!? versions[new_version] = {} fpaths = versions[new_version] for new_fpath, entry in iteritems(new_fpaths): if new_fpath not in fpaths: # new new_fpath fpaths[new_fpath] = {} if entry in fpaths[new_fpath] and entry != fpaths[new_fpath]: raise NotImplementedError("conflict resolutions for when new item added for the same entry") else: fpaths[new_fpath] = entry self.save() datalad-0.4.1/datalad/crawler/nodes/000077500000000000000000000000001301115270200172375ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/nodes/__init__.py000066400000000000000000000015761301115270200213610ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Nodes for the pipelines Each node must be a callable accepting data and must yield (thus be a generator) "derived" data. Many nodes, despite being classes, named in lower case for uniform appearance within pipeline definitions. Some (e.g. Sink) uses typical agreement on camel casing class names -- such nodes are typically instantiated outside of the pipeline definition so they could be retrospected later on """ # XXX Many nodes feel like class is a bit too heavy for them since by relying # on datalad-0.4.1/datalad/crawler/nodes/annex.py000066400000000000000000002006071301115270200207270ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Nodes to interact with annex -- initiate a new dataset or operate with existing one via Annexificator class, which could be used to add files, checkout branches, etc """ import os import re import time from os import listdir from os.path import expanduser, join as opj, exists, isabs, lexists, curdir, realpath from os.path import split as ops from os.path import isdir, islink from os.path import relpath from os import unlink, makedirs from collections import OrderedDict from humanize import naturalsize from six import iteritems from six import string_types from distutils.version import LooseVersion from functools import partial from git import Repo from ...version import __version__ from ...api import add_archive_content from ...api import clean from ...consts import CRAWLER_META_DIR, CRAWLER_META_CONFIG_FILENAME from ...utils import rmtree, updated from ...utils import lmtime from ...utils import find_files from ...utils import auto_repr from ...utils import getpwd from ...tests.utils import put_file_under_git from ...downloaders.providers import Providers from ...distribution.dataset import Dataset from ...api import create from ...support.configparserinc import SafeConfigParserWithIncludes from ...support.gitrepo import GitRepo, _normalize_path from ...support.annexrepo import AnnexRepo from ...support.stats import ActivityStats from ...support.versions import get_versions from ...support.exceptions import AnnexBatchCommandError from ...support.network import get_url_straight_filename, get_url_disposition_filename from ... import cfg from ...cmd import get_runner from ..pipeline import CRAWLER_PIPELINE_SECTION from ..pipeline import initiate_pipeline_config from ..dbs.files import PhysicalFileStatusesDB, JsonFileStatusesDB from ..dbs.versions import SingleVersionDB from datalad.dochelpers import exc_str from logging import getLogger lgr = getLogger('datalad.crawl.annex') _runner = get_runner() _call = _runner.call _run = _runner.run # TODO: make use of datalad_stats @auto_repr class initiate_dataset(object): """Action to initiate a dataset following one of the known templates """ def __init__(self, template, dataset_name=None, path=None, branch=None, backend=None, template_func=None, template_kwargs=None, add_to_super='auto', data_fields=[], add_fields={}, existing=None): """ Parameters ---------- template : str Which template (probably matching the superdataset name) to use. TODO: refer to specs of template that it might understand some arguments encoded, such as #func=custom_pipeline template_func : str, optional Explicitly specify the function name within template module template_kwargs: dict, optional Keyword arguments to pass into the `template_func`. dataset_name : str, optional Name of the dataset. If None, reacts on 'dataset_name' in data path : str, optional Path were to initiate the dataset. If not specified, would use default path for all new datasets (DATALAD_CRAWL_COLLECTIONSPATH) branch : str, optional Which branch to initialize backend : str, optional Supported by git-annex backend. By default (if None specified), it is MD5E backend to improve compatibility with filesystems having a relatively small limit for a maximum path size add_to_super : bool or 'auto', optional Add to super-dataset data_fields : list or tuple of str, optional Additional fields from data to store into configuration for the dataset crawling options -- would be passed into the corresponding crawler template add_fields : dict, optional Dictionary of additional fields to store in the crawler configuration to be passed into the template existing : ('skip', 'raise', 'adjust', 'replace', 'crawl'), optional Behavior if encountering existing dataset """ # TODO: add_fields might not be flexible enough for storing more elaborate # configurations for e.g. "basic" template self.template = template self.template_func = template_func self.template_kwargs = template_kwargs self.dataset_name = dataset_name self.data_fields = data_fields self.add_fields = add_fields self.existing = existing self.path = path self.branch = branch # TODO: backend -> backends (https://github.com/datalad/datalad/issues/358) self.backend = backend self.add_to_super = add_to_super def _initiate_dataset(self, path, name): lgr.info("Initiating dataset %s" % name) if self.branch is not None: raise NotImplementedError("Disabled for now") # because all the 'create' magic is stuffed into the constructor ATM # we need first to initiate a git repository git_repo = GitRepo(path, create=True) # since we are initiating, that branch shouldn't exist yet, thus --orphan git_repo.checkout(self.branch, options=["--orphan"]) # TODO: RF whenevever create becomes a dedicated factory/method # and/or branch becomes an option for the "creator" backend = self.backend or cfg.obtain('datalad.crawl.default_backend', default='MD5E') direct = cfg.obtain('datalad.crawl.init_direct', default=False) if direct: raise NotImplementedError("Disabled for now to init direct mode ones") ds = create( path=path, force=False, # no_annex=False, # TODO: add as an arg # Passing save arg based on backend was that we need to save only if # custom backend was specified, but now with dataset id -- should always save # save=not bool(backend), # annex_version=None, annex_backend=backend, #git_opts=None, #annex_opts=None, #annex_init_opts=None ) if self.add_to_super: # place hack from 'add-to-super' times here sds = ds.get_superdataset() if sds is not None: from datalad.distribution.utils import _install_subds_inplace subdsrelpath = relpath(realpath(ds.path), realpath(sds.path)) # realpath OK lgr.debug("Adding %s as a subdataset to %s", subdsrelpath, sds) _install_subds_inplace(ds=sds, path=ds.path, relativepath=subdsrelpath) # this leaves the subdataset staged in the parent elif str(self.add_to_super) != 'auto': raise ValueError( "Was instructed to add to super dataset but no super dataset " "was found for %s" % ds ) # create/AnnexRepo specification of backend does it non-persistently in .git/config if backend: put_file_under_git(path, '.gitattributes', '* annex.backend=%s' % backend, annexed=False) return ds def _save_crawl_config(self, dataset_path, data): kwargs = self.template_kwargs or {} # update with those from data kwargs.update({f: data[f] for f in self.data_fields}) # additional options given as a dictionary kwargs.update(self.add_fields) return initiate_pipeline_config( template=self.template, template_func=self.template_func, template_kwargs=kwargs, path=dataset_path, commit=True ) def __call__(self, data={}): # figure out directory where create such a dataset dataset_name = self.dataset_name or data.get('dataset_name', None) dataset_path = opj(os.curdir, dataset_name) \ if self.path is None \ else self.path data_updated = updated(data, {'dataset_path': dataset_path, 'dataset_name': dataset_name}) lgr.debug("Request to initialize a dataset %s at %s", dataset_name, dataset_path) init = True if exists(dataset_path): # TODO: config crawl.subdataset.existing = skip|raise|replace|crawl|adjust # TODO: config crawl.subdataset.crawl_new = false|true existing = self.existing or 'skip' if existing == 'skip': lgr.info("Skipping dataset %s since already exists" % dataset_name) yield data_updated return elif existing == 'raise': raise RuntimeError("%s already exists" % dataset_path) elif existing == 'replace': _call(rmtree, dataset_path) elif existing == 'adjust': # E.g. just regenerate configs/meta init = False else: # TODO: 'crawl' ;) raise ValueError(self.existing) if init: _call(self._initiate_dataset, dataset_path, dataset_name) _call(self._save_crawl_config, dataset_path, data) yield data_updated class Annexificator(object): """A helper which would encapsulate the operation of adding new content to git/annex repo If 'filename' field was not found in the data, filename from the URL gets taken. 'path' field of data (if present) is used to define path within the subdirectory. Should be relative. If absolute found -- ValueError is raised """ def __init__(self, path=None, no_annex=False, mode='full', options=None, special_remotes=[], allow_dirty=False, yield_non_updated=False, auto_finalize=True, statusdb=None, skip_problematic=False, **kwargs): """ Note that always_commit=False for the used AnnexRepo to minimize number of unnecessary commits Parameters ---------- mode : str of {'full', 'fast', 'relaxed'} What mode of download to use for the content. In "full" content gets downloaded and checksummed (according to the backend), 'fast' and 'relaxed' are just original annex modes where no actual download is performed and the files' keys are their URLs no_annex : bool Assume/create a simple Git repository, without git-annex special_remotes : list, optional List of custom special remotes to initialize and enable by default yield_non_updated : bool, optional Either to yield original data (with filepath) if load was not updated in annex auto_finalize : bool, optional In some cases, if e.g. adding a file in place of an existing directory or placing a file under a directory for which there is a file atm, we would 'finalize' before carrying out the operation statusdb : {'json', 'fileattr'}, optional DB of file statuses which will be used to figure out if remote load has changed. If None, no statusdb will be used so Annexificator will process every given URL as if it leads to new content. 'json' -- JsonFileStatusesDB will be used which will store information about each provided file/url into a JSON file. 'fileattr' -- PhysicalFileStatusesDB will be used to decide based on information in annex and file(s) mtime on the disk. Note that statusdb "lives" within the branch, so switch_branch would drop existing DB (which should get committed within the branch) and would create a new one if DB is requested again. skip_problematic: bool, optional If True, it would not raise an exception if e.g. url is 404 or forbidden -- then just nothing is yielded, and effectively that entry is skipped **kwargs : dict, optional to be passed into AnnexRepo """ if path is None: path = realpath(curdir) # TODO: commented out to ease developing for now # self.repo = _call(AnnexRepo, path, **kwargs) # TODO: backend -- should be fetched from the config I guess... or should we # give that duty to the dataset initialization routine to change default backend? # Well -- different annexifiers might have different ideas for the backend, but # then those could be overriden via options if exists(path): if not exists(opj(path, '.git')): if (len(listdir(path))) and (not allow_dirty): raise RuntimeError("Directory %s is not empty." % path) self.repo = (GitRepo if no_annex else AnnexRepo)(path, always_commit=False, **kwargs) git_remotes = self.repo.get_remotes() if special_remotes: if no_annex: # isinstance(self.repo, GitRepo): raise ValueError("Cannot have special remotes in a simple git repo") # TODO: move under AnnexRepo with proper testing etc repo_info_repos = [v for k, v in self.repo.repo_info().items() if k.endswith(' repositories')] annex_remotes = {r['description']: r for r in sum(repo_info_repos, [])} for remote in special_remotes: if remote not in git_remotes: if remote in annex_remotes: # Already known - needs only enabling lgr.info("Enabling existing special remote %s" % remote) self.repo.enable_remote(remote) else: lgr.info("Initiating special remote %s" % remote) self.repo.init_remote( remote, ['encryption=none', 'type=external', 'autoenable=true', 'externaltype=%s' % remote]) self.mode = mode self.options = options or [] self.auto_finalize = auto_finalize self._states = set() # TODO: may be should be a lazy centralized instance? self._providers = Providers.from_config_files() self.yield_non_updated = yield_non_updated if (not allow_dirty) and self.repo.dirty: raise RuntimeError("Repository %s is dirty. Finalize your changes before running this pipeline" % path) self.statusdb = statusdb self._statusdb = None # actual DB to be instantiated later self.skip_problematic = skip_problematic # def add(self, filename, url=None): # # TODO: modes # self.repo.add_url_to_file(filename, url, batch=True #, TODO backend # ) # raise NotImplementedError() # # def addurl(self, url, filename): # raise NotImplementedError() # # TODO: register url within "The DB" after it was added # self.register_url_in_db(url, filename) # def register_url_in_db(self, url, filename): # might need to go outside -- since has nothing to do with self raise NotImplementedError() def reset(self): if self._statusdb: self._statusdb.reset() @staticmethod def _get_filename_from_url(url): if url is None: return None filename = get_url_straight_filename(url) # if still no filename if not filename: filename = get_url_disposition_filename(url) if not filename: raise ValueError("No filename was figured out for %s. " "Please adjust pipeline to provide one" % url) return filename def _get_url_status(self, data, url): """A helper to return url_status if url_status is present in data otherwise request a new one """ if 'url_status' in data: return data['url_status'] else: downloader = self._providers.get_provider(url).get_downloader(url) return downloader.get_status(url) def __call__(self, data): # filename=None, get_disposition_filename=False): # some checks assert (self.mode is not None) stats = data.get('datalad_stats', ActivityStats()) url = data.get('url') if url: stats.urls += 1 fpath = self._get_fpath(data, stats, return_None=True) url_status = None if url: try: url_status = self._get_url_status(data, url) except Exception: if self.skip_problematic: stats.skipped += 1 lgr.debug("Failed to obtain status for url %s" % url) return raise if fpath is None: # pick it from url_status and give for "reprocessing" fpath = self._get_fpath(data, stats, filename=url_status.filename) if not fpath: if self.skip_problematic: stats.skipped += 1 lgr.debug("Failed to figure out filename for url %s" % url) return raise ValueError("No filename was provided") filepath = opj(self.repo.path, fpath) lgr.debug("Request to annex %(url)s to %(fpath)s", locals()) # since filename could have come from url -- let's update with it updated_data = updated(data, {'filename': ops(fpath)[1], # TODO? 'filepath': filepath }) if self.statusdb is not None and self._statusdb is None: if isinstance(self.statusdb, string_types): # initiate the DB self._statusdb = { 'json': JsonFileStatusesDB, 'fileattr': PhysicalFileStatusesDB}[self.statusdb](annex=self.repo) else: # use provided persistent instance self._statusdb = self.statusdb statusdb = self._statusdb if url: if lexists(filepath): # check if URL provides us updated content. If not -- we should do nothing # APP1: in this one it would depend on local_status being asked first BUT # may be for S3 support where statusdb would only record most recent # modification time # local_status = self.statusdb.get(fpath) # remote_status = downloader.get_status(url, old_status=local_status) # if remote_status == local_status: # WIP TODO not is_status_different(local_status, remote_status): # APP2: no explicit local_status # if self.mode != 'full' and fpath == '1-copy.dat': # import pdb; pdb.set_trace() # TODO: what if the file came from another url bearing the same mtime and size???? # unlikely but possible. We would need to provide URL for comparison(s) if self.mode == 'relaxed' or ( statusdb is not None and not statusdb.is_different(fpath, url_status, url)): # TODO: check if remote_status < local_status, i.e. mtime jumped back # and if so -- warning or may be according to some config setting -- skip # e.g. config.get('crawl', 'new_older_files') == 'skip' lgr.debug("Skipping download. URL %s doesn't provide new content for %s. New status: %s", url, filepath, url_status) stats.skipped += 1 if self.yield_non_updated: yield updated_data # there might be more to it! return else: # just to mark file as still of interest to us so it doesn't get wiped out later # as it should have happened if we removed creation/tracking of that file intentionally if statusdb: statusdb.get(fpath) if not url: lgr.debug("Adding %s to annex without url being provided" % filepath) # so we have only filename assert fpath # just add into git directly for now # TODO: tune add so we could use its json output, and may be even batch it out_json = _call(self.repo.add, fpath, options=self.options) # elif self.mode == 'full': # # since addurl ignores annex.largefiles we need first to download that file and then # # annex add it # # see http://git-annex.branchable.com/todo/make_addurl_respect_annex.largefiles_option # lgr.debug("Downloading %s into %s and adding to annex", url, filepath) # def _download_and_git_annex_add(url, fpath): # # Just to feed into _call for dry-run # filepath_downloaded = downloader.download(url, filepath, overwrite=True, stats=stats) # assert filepath_downloaded == filepath # self.repo.add(fpath, options=self.options) # # and if the file ended up under annex, and not directly under git -- addurl # # TODO: better function which explicitly checks if file is under annex or either under git # if self.repo.file_has_content(fpath): # stats.add_annex += 1 # self.repo.add_url_to_file(fpath, url, batch=True) # else: # stats.add_git += 1 # _call(_download_and_git_annex_add, url, fpath) else: # !!!! If file shouldn't get under annex due to largefile setting -- we must download it!!! # TODO: http://git-annex.branchable.com/todo/make_addurl_respect_annex.largefiles_option/#comment-b43ef555564cc78c6dee2092f7eb9bac # we should make use of matchexpression command, but that might reincarnated # above code so just left it commented out for now annex_options = self.options if self.mode == 'full': lgr.debug("Downloading %s into %s and adding to annex" % (url, filepath)) else: annex_options = annex_options + ["--%s" % self.mode] lgr.debug("Pointing %s to %s within annex in %s mode" % (url, filepath, self.mode)) if lexists(filepath): lgr.debug("Removing %s since it exists before fetching a new copy" % filepath) if isdir(filepath): # if directory - tricky, since we would want then to check if no # staged changes under _call(self._check_no_staged_changes_under_dir, filepath, stats=stats) _call(rmtree, filepath) else: _call(unlink, filepath) _call(stats.increment, 'overwritten') else: _call(self._check_non_existing_filepath, filepath, stats=stats) # TODO: We need to implement our special remote here since no downloaders used if self.mode == 'full' and url_status and url_status.size: # > 1024**2: lgr.info("Need to download %s from %s. No progress indication will be reported" % (naturalsize(url_status.size), url)) try: out_json = _call(self.repo.add_url_to_file, fpath, url, options=annex_options, batch=True) except AnnexBatchCommandError as exc: if self.skip_problematic: lgr.warning("Skipping %s due to %s", url, exc_str(exc)) return else: raise added_to_annex = 'key' in out_json if self.mode == 'full' or not added_to_annex: # we need to adjust our download stats since addurl doesn't do that and we do # not use our downloaders here _call(stats.increment, 'downloaded') _call(stats.increment, 'downloaded_size', _call(lambda: os.stat(filepath).st_size)) # file might have been added but really not changed anything (e.g. the same README was generated) # TODO: # if out_json: # if not try -- should be here! # File might have been not modified at all, so let's check its status first changed = set().union(*self._get_status(args=[fpath])) if fpath in changed: _call(stats.increment, 'add_annex' if ('key' in out_json and out_json['key'] is not None) else 'add_git' ) else: _call(stats.increment, 'skipped') # TODO!!: sanity check that no large files are added to git directly! # so we have downloaded the beast # since annex doesn't care to set mtime for the symlink itself, we better set it ourselves if lexists(filepath): # and islink(filepath): if url_status: # set mtime of the symlink or git-added file itself # utime dereferences! # _call(os.utime, filepath, (time.time(), remote_status.mtime)) # *nix only! TODO if url_status.mtime: _call(lmtime, filepath, url_status.mtime) if statusdb: _call(statusdb.set, filepath, url_status) else: # we still need to inform DB about this file so later it would signal to remove it # if we no longer care about it if statusdb: _call(statusdb.set, filepath) self._states.add("Updated git/annex from a remote location") # WiP: commented out to do testing before merge # db_filename = self.db.get_filename(url) # if filename is not None and filename != db_filename: # # need to download new # self.repo.add_urls # # remove old # self.repo.remove([db_filename]) # self.db.set_filename(url, filename) # # figure out if we need to download it # #if self.mode in ('relaxed', 'fast'): # git annex addurl --pathdepth=-1 --backend=SHA256E '-c' 'annex.alwayscommit=false' URL # with subsequent "drop" leaves no record that it ever was here yield updated_data # There might be more to it! def _check_no_staged_changes_under_dir(self, dirpath, stats=None): """Helper to verify that we can "safely" remove a directory """ dirty_files = self._get_status() dirty_files = sum(dirty_files, []) dirpath_normalized = _normalize_path(self.repo.path, dirpath) for dirty_file in dirty_files: if stats: _call(stats.increment, 'removed') if dirty_file.startswith(dirpath_normalized): if self.auto_finalize: self.finalize()({'datalad_stats': stats}) return else: raise RuntimeError( "We need to save some file instead of directory %(dirpath)s " "but there are uncommitted changes (%(dirty_file)s) under " "that directory. Please commit them first" % locals()) def _check_non_existing_filepath(self, filepath, stats=None): """Helper to verify that we can safely save into the target path For instance, we can't save into a file d/file if d is a file, not a directory """ # if file doesn't exist we need to verify that there are no conflicts dirpath, name = ops(filepath) if dirpath: # we need to assure that either the directory exists or directories # on the way to it exist and are not a file by some chance while dirpath: if lexists(dirpath): if not isdir(dirpath): # we have got a problem # HANDLE THE SITUATION # check if given file is not staged for a commit or dirty dirty_files = self._get_status() # it was a tuple of 3 dirty_files = sum(dirty_files, []) dirpath_normalized = _normalize_path(self.repo.path, dirpath) if dirpath_normalized in dirty_files: if self.auto_finalize: self.finalize()({'datalad_stats': stats}) else: raise RuntimeError( "We need to annex file %(filepath)s but there is a file " "%(dirpath)s in its path which destiny wasn't yet decided " "within git. Please commit or remove it before trying " "to annex this new file" % locals()) lgr.debug("Removing %s as it is in the path of %s" % (dirpath, filepath)) _call(os.unlink, dirpath) if stats: _call(stats.increment, 'overwritten') break # in any case -- we are done! dirpath, _ = ops(dirpath) if not dirpath.startswith(self.repo.path): # shouldn't happen! raise RuntimeError("We escaped the border of the repository itself. " "path: %s repo: %s" % (dirpath, self.repo.path)) def _get_fpath(self, data, stats, filename=None, return_None=False): """Return relative path (fpath) to the file based on information in data Raises ------ ValueError if no filename field was provided """ # figure out the filename. If disposition one was needed, pipeline should # have had it explicitly if filename is None: filename = data['filename'] if 'filename' in data else None stats.files += 1 fpath = filename if filename is None: if return_None: return None stats.skipped += 1 raise ValueError("No filename were provided") elif isabs(filename): stats.skipped += 1 raise ValueError("Got absolute filename %r" % filename) path_ = data.get('path', None) if path_: # TODO: test all this handling of provided paths if isabs(path_): stats.skipped += 1 raise ValueError("Absolute path %s was provided" % path_) fpath = opj(path_, fpath) return fpath def switch_branch(self, branch, parent=None, must_exist=None, allow_remote=True): """Node generator to switch branches, returns actual node Parameters ---------- branch : str Name of the branch parent : str or None, optional If parent is provided, it will serve as a parent of the branch. If None, detached new branch will be created must_exist : bool or None, optional If None, doesn't matter. If True, would fail if branch does not exist. If False, would fail if branch already exists allow_remote : bool, optional If not exists locally, will try to find one among remote ones """ def switch_branch(data): """Switches to the branch %s""" % branch # if self.repo.dirty list(self.finalize()(data)) # statusdb is valid only within the same branch self._statusdb = None existing_branches = self.repo.get_branches() if must_exist is not None: assert must_exist == (branch in existing_branches) # TODO: this should be a part of the gitrepo logic if branch not in existing_branches and allow_remote: remote_branches = self.repo.get_remote_branches() remotes = sorted(set([b.split('/', 1)[0] for b in remote_branches])) for r in ['origin'] + remotes: # ok if origin tested twice remote_branch = "%s/%s" % (r, branch) if remote_branch in remote_branches: lgr.info("Did not find branch %r locally. Checking out remote one %r" % (branch, remote_branch)) self.repo.checkout(remote_branch, options=['--track']) # refresh the list -- same check will come again existing_branches = self.repo.get_branches() break if branch not in existing_branches: if parent is None: # new detached branch lgr.info("Checking out a new detached branch %s" % (branch)) self.repo.checkout(branch, options=["--orphan"]) if self.repo.dirty: self.repo.remove('.', r=True, f=True) # TODO: might be insufficient if directories etc TEST/fix else: if parent not in existing_branches: raise RuntimeError("Parent branch %s does not exist" % parent) lgr.info("Checking out %s into a new branch %s" % (parent, branch)) self.repo.checkout(parent, options=["-b", branch]) else: lgr.info("Checking out an existing branch %s" % (branch)) self.repo.checkout(branch) yield updated(data, {"git_branch": branch}) return switch_branch def merge_branch(self, branch, target_branch=None, strategy=None, commit=True, one_commit_at_a_time=False, skip_no_changes=None, **merge_kwargs): """Merge a branch into the current branch Parameters ---------- branch: str Branch to be merged target_branch: str, optional Into which branch to merge. If not None, it will be checked out first. At the end we will return into original branch strategy: None or 'theirs', optional With 'theirs' strategy remote branch content is used 100% as is. 'theirs' with commit=False can be used to prepare data from that branch for processing by further steps in the pipeline commit: bool, optional Either to commit when merge is complete or not one_commit_at_at_time: bool, optional Either to generate or not skip_no_changes: None or bool, optional Either to not perform any action if there are no changes from previous merge point. If None, config TODO will be consulted with default of being True (i.e. skip if no changes) """ # TODO: support merge of multiple branches at once assert (strategy in (None, 'theirs')) def merge_branch(data): if target_branch is not None: orig_branch = self.repo.get_active_branch() target_branch_ = target_branch list(self.switch_branch(target_branch_)(data)) else: orig_branch = None target_branch_ = self.repo.get_active_branch() if self.repo.dirty: raise RuntimeError("Requested to merge another branch while current state is dirty") last_merged_checksum = self.repo.get_merge_base([target_branch_, branch]) skip_no_changes_ = skip_no_changes if skip_no_changes is None: # TODO: skip_no_changes = config.getboolean('crawl', 'skip_merge_if_no_changes', default=True) skip_no_changes_ = True if last_merged_checksum == self.repo.get_hexsha(branch): lgr.debug("Branch %s doesn't provide any new commits for current HEAD" % branch) if skip_no_changes_: lgr.debug("Skipping the merge") return if one_commit_at_a_time: all_to_merge = list( self.repo.get_branch_commits( branch, limit='left-only', stop=last_merged_checksum, value='hexsha'))[::-1] else: all_to_merge = [branch] nmerges = len(all_to_merge) # There were no merges, but we were instructed to not skip if not nmerges and skip_no_changes_ is False: # so we will try to merge it nevertheless lgr.info("There was nothing to merge but we were instructed to merge due to skip_no_changes=False") all_to_merge = [branch] nmerges = 1 plmerges = "s" if nmerges > 1 else "" lgr.info("Initiating %(nmerges)d merge%(plmerges)s of %(branch)s using strategy %(strategy)s", locals()) options = ['--no-commit'] if not commit else [] for to_merge in all_to_merge: # we might have switched away to orig_branch if self.repo.get_active_branch() != target_branch_: self.repo.checkout(target_branch_) if strategy is None: self.repo.merge(to_merge, options=options, **merge_kwargs) elif strategy == 'theirs': self.repo.merge(to_merge, options=["-s", "ours", "--no-commit"], expect_stderr=True, **merge_kwargs) self.repo._git_custom_command([], "git read-tree -m -u %s" % to_merge) self.repo.add('.', options=self.options) # so everything is staged to be committed else: raise NotImplementedError(strategy) if commit: if strategy is not None: msg = branch if (nmerges == 1) else ("%s (%s)" % (branch, to_merge)) self._commit("Merged %s using strategy %s" % (msg, strategy), options=["-a"]) else: # record into our activity stats stats = data.get('datalad_stats', None) if stats: stats.merges.append([branch, target_branch_]) if orig_branch is not None: self.repo.checkout(orig_branch) yield data return merge_branch def _precommit(self): self.repo.precommit() # so that all batched annexes stop if self._statusdb: self._statusdb.save() # there is something to commit and backends was set but no .gitattributes yet path = self.repo.path if self.repo.dirty and not exists(opj(path, '.gitattributes')): backends = self.repo.default_backends if backends: # then record default backend into the .gitattributes put_file_under_git(path, '.gitattributes', '* annex.backend=%s' % backends[0], annexed=False) # at least use repo._git_custom_command def _commit(self, msg=None, options=[]): # we need a custom commit due to "fancy" merges and GitPython # not supporting that ATM # https://github.com/gitpython-developers/GitPython/issues/361 # and apparently not actively developed msg = str(msg).strip() if not msg: # we need to provide some commit msg, could may be deduced from current status # TODO msg = "a commit" msg = GitRepo._get_prefixed_commit_msg(msg) if msg is not None: options = options + ["-m", msg] self._precommit() # so that all batched annexes stop self.repo._git_custom_command([], ["git", "commit"] + options) # self.repo.commit(msg) # self.repo.repo.git.commit(options) def _unstage(self, fpaths): # self.repo.cmd_call_wrapper.run(["git", "reset"] + fpaths) self.repo._git_custom_command(fpaths, ["git", "reset"]) def _stage(self, fpaths): self.repo.add(fpaths, git=True) # self.repo.cmd_call_wrapper.run(["git", "add"] + fpaths) def _get_status(self, args=[]): """Custom check of status to see what files were staged, untracked etc until https://github.com/gitpython-developers/GitPython/issues/379#issuecomment-180101921 is resolved """ # out, err = self.repo.cmd_call_wrapper.run(["git", "status", "--porcelain"]) cmd_args = ["git", "status", "--porcelain"] + args staged, notstaged, untracked, deleted = [], [], [], [] statuses = { '??': untracked, 'A ': staged, 'M ': staged, ' M': notstaged, ' D': deleted } if isinstance(self.repo, AnnexRepo) and self.repo.is_direct_mode(): statuses['AD'] = staged out, err = self.repo.proxy(cmd_args) else: out, err = self.repo._git_custom_command([], cmd_args) assert not err for l in out.split('\n'): if not l: continue act = l[:2] # first two characters is what is happening to the file fname = l[3:] try: statuses[act].append(fname) # for the purpose of this use, we don't even want MM or anything else except KeyError: raise RuntimeError("git status %r not yet supported. TODO" % act) return staged, notstaged, untracked, deleted def commit_versions(self, regex, dirs=True, # either match directory names rename=False, **kwargs): """Generate multiple commits if multiple versions were staged Parameters ---------- TODO **kwargs: dict, optional Passed to get_versions """ def _commit_versions(data): self._precommit() # so that all batched annexes stop # figure out versions for all files (so we could dataset conflicts with existing # non versioned) # TODO: we need to care only about staged (and unstaged?) files ATM! # So let's do it. And use separate/new Git repo since we are doing manual commits through # calls to git. TODO: RF to avoid this # Not usable for us ATM due to # https://github.com/gitpython-developers/GitPython/issues/379 # repo = Repo(self.repo.path) # # def process_diff(diff): # """returns full paths for files in the diff""" # out = [] # for obj in diff: # assert(not obj.renamed) # not handling atm # assert(not obj.deleted_file) # not handling atm # assert(obj.a_path == obj.b_path) # not handling atm # out.append(opj(self.repo.path, obj.a_path)) # return out # # staged = process_diff(repo.index.diff('HEAD'))#repo.head.commit)) # notstaged = process_diff(repo.index.diff(None)) staged, notstaged, untracked, deleted = self._get_status() # verify that everything is under control! assert (not notstaged) # not handling atm, although should be safe I guess just needs logic # to not unstage them assert (not untracked) # not handling atm assert (not deleted) # not handling atm if not staged: return # nothing to be done -- so we wash our hands off entirely if not dirs: raise NotImplementedError("ATM matching will happen to dirnames as well") versions = get_versions(staged, regex, **kwargs) if not versions: # no versioned files were added, nothing to do really for d in self.finalize()(data): yield d return # we don't really care about unversioned ones... overlay and all that ;) if None in versions: versions.pop(None) # take only new versions to deal with versions_db = SingleVersionDB(self.repo) prev_version = versions_db.version if prev_version is None: new_versions = versions # consider all! else: version_keys = list(versions.keys()) if prev_version not in versions_db.versions: # shouldn't happen raise RuntimeError( "previous version %s not found among known to DB: %s" % (prev_version, versions_db.versions.keys())) # all new versions must be greater than the previous version # since otherwise it would mean that we are complementing previous version and it might be # a sign of a problem # Well -- so far in the single use-case with openfmri it was that they added # derivatives for the same version, so I guess we will allow for that, thus allowing = assert (all((LooseVersion(prev_version) <= LooseVersion(v)) for v in versions)) # old implementation when we didn't have entire versions db stored # new_versions = OrderedDict(versions.items()[version_keys.index(prev_version) + 1:]) new_versions = versions # if we have "new_versions" smallest one smaller than previous -- we got a problem! # TODO: how to dataset ==? which could be legit if more stuff was added for the same # version? but then if we already tagged with that -- we would need special handling if new_versions: smallest_new_version = next(iter(new_versions)) if prev_version: if LooseVersion(smallest_new_version) < LooseVersion(prev_version): raise ValueError("Smallest new version %s is < prev_version %s" % (smallest_new_version, prev_version)) versions_db.update_versions(versions) # store all new known versions # early return if no special treatment is needed nnew_versions = len(new_versions) if nnew_versions <= 1: # if a single new version -- no special treatment is needed, but we need to # inform db about this new version if nnew_versions == 1: _call(setattr, versions_db, 'version', smallest_new_version) # we can't return a generator here for d in self.finalize()(data): yield d return # unstage all versioned files from the index nunstaged = 0 for version, fpaths in iteritems(versions): nfpaths = len(fpaths) lgr.debug("Unstaging %d files for version %s", nfpaths, version) nunstaged += nfpaths _call(self._unstage, list(fpaths.values())) stats = data.get('datalad_stats', None) stats_str = ('\n\n' + stats.as_str(mode='full')) if stats else '' for iversion, (version, fpaths) in enumerate(iteritems(new_versions)): # for all versions past previous # stage/add files of that version to index if rename: # we need to rename and create a new vfpaths vfpaths = [] for fpath, vfpath in iteritems(fpaths): # ATM we do not allow unversioned -- should have failed earlier, if not HERE! # assert(not lexists(fpath)) # nope! it must be there from previous commit of a versioned file! # so rely on logic before lgr.debug("Renaming %s into %s" % (vfpath, fpath)) os.rename(vfpath, fpath) vfpaths.append(fpath) else: # so far we didn't bother about status, so just values would be sufficient vfpaths = list(fpaths.values()) nfpaths = len(vfpaths) lgr.debug("Staging %d files for version %s", nfpaths, version) nunstaged -= nfpaths assert (nfpaths >= 0) assert (nunstaged >= 0) _call(self._stage, vfpaths) # RF: with .finalize() to avoid code duplication etc # ??? what to do about stats and states? reset them or somehow tune/figure it out? vmsg = "Multi-version commit #%d/%d: %s. Remaining unstaged: %d" % ( iversion + 1, nnew_versions, version, nunstaged) if stats: _call(stats.reset) if version: _call(setattr, versions_db, 'version', version) _call(self._commit, "%s (%s)%s" % (', '.join(self._states), vmsg, stats_str), options=[]) # unless we update data, no need to yield multiple times I guess # but shouldn't hurt yield data assert (nunstaged == 0) # we at the end committed all of them! return _commit_versions def remove_other_versions(self, name=None, db=None, overlay=None, remove_unversioned=False, fpath_subs=None, exclude=None): """Remove other (non-current) versions of the files Pretty much to be used in tandem with commit_versions Parameters ---------- name : str, optional Name of the SingleVersionDB to consult (e.g., original name of the branch) db : SingleVersionDB, optional If provided, `name` must be None. overlay : int or callable, optional Overlay files of the next version to only replace files from the previous version. If specified as `int`, value will determine how many leading levels of .-separated (e.g., of major.minor.patch) semantic version format will be used to identify "unique" non-overlayable version. E.g. overlay=2, would overlay all .patch levels, while starting without overlay for any new major.minor version. If a callable, it would be used to augment versions before identifying non-overlayable version component. So in other words `overlay=2` should be identical to `overlay=lambda v: '.'.join(v.split('.')[:2])` fpath_subs : list of (from, to), optional Regex substitutions to apply to (versioned but with version part removed) filenames before considering. To be used whenever file names at some point were changed (e.g., `ds001_R1.0.1.tgz` one lucky day became `ds000001_R1.0.2.zip`) remove_unversioned: bool, optional If there is a version defined now, remove those files which are unversioned i.e. not listed associated with any version exclude : basestring, optional Regexp to search to exclude files from considering to remove them if `remove_unversioned`. Passed to `find_files`. E.g. `README.*` which could have been generated in `incoming` branch """ if overlay is None: overlay_version_func = lambda x: x elif isinstance(overlay, int) and not isinstance(overlay, bool): overlay_version_func = lambda v: '.'.join(v.split('.')[:overlay]) elif hasattr(overlay, '__call__'): overlay_version_func = overlay else: raise TypeError("overlay must be an int or a callable. Got %s" % repr(overlay)) if db is not None and name is not None: raise ValueError( "Must have specified either name or version_db, not both" ) def _remove_other_versions(data): stats = data.get('datalad_stats', None) versions_db = SingleVersionDB(self.repo, name=name) \ if db is None \ else db current_version = versions_db.version if not current_version: lgr.info("No version information was found, skipping remove_other_versions") yield data return current_overlay_version = overlay_version_func(current_version) prev_version = None tracked_files = {} # track files in case of overlaying for version, fpaths in iteritems(versions_db.versions): # sanity check since we now will have assumption that versions # are sorted if prev_version is not None: assert(prev_version < LooseVersion(version)) prev_version = LooseVersion(version) overlay_version = overlay_version_func(version) # we do not care about non-versioned or current "overlay" version #import pdb; pdb.set_trace() if version is None: continue files_to_remove = [] if current_overlay_version == overlay_version and \ LooseVersion(version) <= LooseVersion(current_version): # the same overlay but before current version # we need to track the last known within overlay and if # current updates, remove older version fpaths_considered = {} for fpath, vfpath in fpaths.items(): if fpath_subs: fpath_orig = fpath for from_, to_ in fpath_subs: fpath = re.sub(from_, to_, fpath) if fpath in fpaths_considered: # May be it is not that severe, but for now we will # crash if there is a collision raise ValueError( "Multiple files (%s, %s) collided into the same name %s", fpaths_considered[fpath], fpath_orig, fpath ) fpaths_considered[fpath] = fpath_orig if fpath in tracked_files: files_to_remove.append(tracked_files[fpath]) tracked_files[fpath] = vfpath # replace with current one if version == current_version: # just clean out those tracked_files with the most recent versions # and remove nothing tracked_files = {} else: files_to_remove = fpaths.values() for vfpath in files_to_remove: vfpathfull = opj(self.repo.path, vfpath) if os.path.lexists(vfpathfull): lgr.debug( "Removing %s of version %s (overlay %s). " "Current one %s (overlay %s)", vfpathfull, version, overlay_version, current_version, current_overlay_version ) os.unlink(vfpathfull) assert not tracked_files, "we must not end up having tracked files" if remove_unversioned: # it might be that we haven't 'recorded' unversioned ones at all # and now got an explicit version, so we would just need to remove them all # For that we need to get all files which left, and remove them unless they # were a versioned file (considered above) for any version all_versioned_files = set() for versioned_files_ in versions_db.versions.values(): all_versioned_files.update(versioned_files_.values()) for fpath in find_files( '.*', topdir=self.repo.path, exclude=exclude, exclude_datalad=True, exclude_vcs=True ): fpath = relpath(fpath, self.repo.path) # ./bla -> bla if fpath in all_versioned_files: lgr.log( 5, "Not removing %s file since it was versioned", fpath) continue lgr.log(5, "Removing unversioned %s file", fpath) os.unlink(fpath) elif exclude: lgr.warning("`exclude=%r` was specified whenever remove_unversioned is False", exclude) if stats: stats.versions.append(current_version) yield data return _remove_other_versions # TODO: @borrow_kwargs from api_add_... def add_archive_content(self, commit=False, **aac_kwargs): """ Parameters ---------- aac_kwargs: dict, optional Options to pass into api.add_archive_content """ def _add_archive_content(data): # if no stats -- they will be brand new each time :-/ stats = data.get('datalad_stats', ActivityStats()) archive = self._get_fpath(data, stats) # TODO: may be adjust annex_options annex = add_archive_content( archive, annex=self.repo, key=False, commit=commit, allow_dirty=True, annex_options=self.options, stats=stats, **aac_kwargs ) self._states.add("Added files from extracted archives") assert (annex is self.repo) # must be the same annex, and no new one created # to propagate statistics from this call into commit msg since we commit=False here # we update data with stats which gets a new instance if wasn't present yield updated(data, {'datalad_stats': stats}) return _add_archive_content # TODO: either separate out commit or allow to pass a custom commit msg? def finalize(self, tag=False, existing_tag=None, cleanup=False, aggregate=False): """Finalize operations -- commit uncommited, prune abandoned? etc Parameters ---------- tag: bool or str, optional If set, information in datalad_stats and data can be used to tag release if versions is non-empty. If True, simply the last version to be used. If str, it is .format'ed using datalad_stats, so something like "r{stats.versions[0]}" can be used. Also `last_version` is provided as the last one from stats.versions (None if empty) existing_tag: None or '+suffix', optional What to do if tag already exists, if None, warning is issued. If `+suffix`, +0, +1, +2 ... are tried until available one is found. cleanup: bool, optional Either to perform cleanup operations, such as 'git gc' and 'datalad clean' aggregate: bool, optional Aggregate meta-data (ATM no recursion, guessing the type) """ def _finalize(data): self._precommit() stats = data.get('datalad_stats', None) if self.repo.dirty: # or self.tracker.dirty # for dry run lgr.info("Repository found dirty -- adding and committing") _call(self.repo.add, '.', options=self.options) # so everything is committed stats_str = ('\n\n' + stats.as_str(mode='full')) if stats else '' _call(self._commit, "%s%s" % (', '.join(self._states), stats_str), options=["-a"]) if stats: _call(stats.reset) else: lgr.info("Found branch non-dirty -- nothing was committed") if aggregate: from datalad.api import aggregate_metadata aggregate_metadata(dataset=self.repo.path, guess_native_type=True) if tag and stats: # versions survive only in total_stats total_stats = stats.get_total() if total_stats.versions: last_version = total_stats.versions[-1] if isinstance(tag, string_types): tag_ = tag.format(stats=total_stats, data=data, last_version=last_version) else: tag_ = last_version # TODO: config.tag.sign stats_str = "\n\n" + total_stats.as_str(mode='full') tags = self.repo.repo.tags if tag_ in tags: # TODO: config.tag.allow_override if existing_tag == '+suffix': lgr.warning( "There is already a tag %s in the repository. Delete it first if you want it updated" % tag_) tag_ = None elif existing_tag is None: suf = 1 while True: tag__ = '%s+%d' % (tag_, suf) if tag__ not in tags: break suf += 1 lgr.warning("There is already a tag %s in the repository. Tagging as %s" % (tag_, tag__)) tag_ = tag__ else: raise ValueError(existing_tag) self.repo.repo.create_tag(tag_, message="Automatically crawled and tagged by datalad %s.%s" % ( __version__, stats_str)) if cleanup: total_stats = stats.get_total() if total_stats.add_git or total_stats.add_annex or total_stats.merges: if cfg.obtain('datalad.crawl.pipeline.housekeeping', default=True): lgr.info("House keeping: gc, repack and clean") # gc and repack self.repo.gc(allow_background=False) clean(annex=self.repo) else: lgr.info("No git house-keeping performed as instructed by config") else: lgr.info("No git house-keeping performed as no notable changes to git") self._states = set() yield data return _finalize def remove_obsolete(self): """Remove obsolete files which were not referenced in queries to DB Note that it doesn't reset any state within statusdb upon call, so shouldn't be called multiple times for the same state. """ # made as a class so could be reset class _remove_obsolete(object): def __call__(self_, data): statusdb = self._statusdb obsolete = statusdb.get_obsolete() if obsolete: files_str = ": " + ', '.join(obsolete) if len(obsolete) < 10 else "" lgr.info('Removing %d obsolete files%s' % (len(obsolete), files_str)) stats = data.get('datalad_stats', None) _call(self.repo.remove, obsolete) if stats: _call(stats.increment, 'removed', len(obsolete)) for filepath in obsolete: statusdb.remove(filepath) yield data def reset(self_): if self._statusdb: self._statusdb.reset() return _remove_obsolete() def remove(self, data): """Removed passed along file name from git/annex""" stats = data.get('datalad_stats', None) self._states.add("Removed files") filename = self._get_fpath(data, stats) # TODO: not sure if we should may be check if exists, and skip/just complain if not if stats: _call(stats.increment, 'removed') if lexists(opj(self.repo.path, filename)): _call(self.repo.remove, filename) else: lgr.warning("Was asked to remove non-existing path %s", filename) yield data def initiate_dataset(self, *args, **kwargs): """Thin proxy to initiate_dataset node which initiates dataset as a subdataset to current annexificator """ # now we can just refer to initiate_dataset which uses create return initiate_dataset(*args, **kwargs) datalad-0.4.1/datalad/crawler/nodes/crawl_url.py000066400000000000000000000152461301115270200216130ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Basic crawler for the web """ from os.path import splitext, dirname, basename from ...utils import updated from ...dochelpers import exc_str from ...downloaders.base import DownloadError, UnhandledRedirectError from ...downloaders.providers import Providers from logging import getLogger lgr = getLogger('datalad.crawl.crawl_url') class crawl_url(object): """Given a source url, perform the initial crawling of the page, i.e. simply bloody fetch it and pass along """ def __init__(self, url=None, matchers=None, input='url', failed=None, cache_redirects=True, output=('response', 'url')): """If url is None, would try to pick it up from data[input] Parameters ---------- matchers: list of matchers Expect page content in and should produce url field failed: {skip}, optional What to do about failing urls. If None -- would consult (eventually) the config cache_redirects: bool, optional Either to remember redirects for subsequent invocations """ self._url = url self._matchers = matchers self._input = input self._output = output self._seen = set() self._providers = Providers.from_config_files() # partially to overcome stuck redirect # https://github.com/kennethreitz/requests/issues/2997 self._redirects_cache = {} if cache_redirects else None self.failed = failed def reset(self): """Reset cache of seen URLs""" self._seen = set() def _visit_url(self, url, data): if url in self._seen: return # this is just a cruel first attempt lgr.debug("Visiting %s" % url) try: retry = 0 orig_url = url if self._redirects_cache is not None: url = self._redirects_cache.get(url, url) while True: retry += 1 if retry > 100: raise DownloadError("We have followed 100 redirects already. Something is wrong!") try: self._seen.add(url) page = self._providers.fetch(url, allow_redirects=False) break except UnhandledRedirectError as exc: # since we care about tracking URL for proper full url construction # we should disallow redirects and handle them manually here lgr.debug("URL %s was redirected to %s" % (url, exc.url)) if url == exc.url: raise DownloadError("Was redirected to the same url upon %s" % exc_str(exc)) url = exc.url if self._redirects_cache is not None: self._redirects_cache[orig_url] = exc.url except DownloadError as exc: lgr.warning("URL %s failed to download: %s" % (url, exc_str(exc))) if self.failed in {None, 'skip'}: # TODO: config -- crawl.failed='skip' should be a config option, for now always skipping return raise # otherwise -- kaboom data_ = updated(data, zip(self._output, (page, url))) yield data_ # now recurse if matchers were provided matchers = self._matchers if matchers: lgr.debug("Looking for more URLs at %s using %s", url, matchers) for matcher in (matchers if isinstance(matchers, (list, tuple)) else [matchers]): for data_matched in matcher(data_): if 'url' not in data_matched: lgr.warning("Got data without a url from %s" % matcher) continue # proxy findings for data_matched_ in self._visit_url(data_matched['url'], data_matched): yield data_matched_ def __call__(self, data={}): #assert(data == {}) # atm assume we are the first of mogican url = data[self._input] if not self._url else self._url return self._visit_url(url, data) def recurse(self, data): """Recurse into the page - self._url gets ignored""" return self._visit_url(data[self._input], data) # for extractors, actions in conditionals: # extractors = assure_tuple_or_list(extractors) # actions = assure_tuple_or_list(actions) # seen_urls = set() # for extractor in extractors: # for url, meta_ in extractor(parent_url, meta=meta): # if url in seen_urls: # continue # file = None # # progress through actions while possibly augmenting the url, file, and/or meta_ # for action in actions: # # TODO: may be we should return a dict with whatever that action # # found necessary to change, update local state and pass into # url, file, meta_ = \ # action(parent_url=parent_url, url=url, file=file, meta=meta_) # seen_urls.add(url) # TODO: probably might sense to RF into just a generic TSV file parser def parse_checksums(digest=None): """Generates a node capable of parsing checksums file and generating new URLs Base of the available in data url is used for new URLs """ def _parse_checksums(data): url = data['url'] urlsplit = url.split('/') topurl = '/'.join(urlsplit[:-1]) if digest is None: # deduce from URL's file extension filename = urlsplit[-1] base, ext = splitext(filename) digest_ = ext if ext else digest content = data['response'] # split into separate lines, first entry is checksum, 2nd file path for line in content.split('\n'): if not line: # empty line continue checksum, fpath = line.split(None, 1) yield updated(data, {'digest': digest or digest_, 'checksum': checksum, 'path': dirname(fpath), 'filename': basename(fpath), 'url': "%s/%s" % (topurl, fpath) }) return _parse_checksums datalad-0.4.1/datalad/crawler/nodes/matches.py000066400000000000000000000156531301115270200212470ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Helpers to match and generate urls and extracted content""" __docformat__ = 'restructuredtext' import re import types from six import PY3 from ...utils import updated from ...support.network import dlurljoin from ...utils import auto_repr from logging import getLogger lgr = getLogger('datalad.crawler') try: from scrapy.http import Response from scrapy.selector import Selector except ImportError: lgr.debug("Failed to import Selector from scrapy, so matches would not be functional") class Selector(object): xpath = css = None Response = None # for now heavily based on scrapy but we might make the backend # replaceable @auto_repr class ExtractorMatch(object): """Generic matching extractor """ def __init__(self, query, input='response', output='match', pop_input=False, allow_multiple=False, xpaths=None, csss=None, min_count=None, max_count=None): """""" # TODO: define arguments self.query = query # allow_multiple concerns only extraction of additional xpaths and csss self._allow_multiple = allow_multiple self._xpaths = xpaths self._csss = csss self._input = input self._output = output self._pop_input = False self._min_count = min_count self._max_count = max_count def _select_and_extract(self, selector, query, data): raise NotImplementedError def __call__(self, data): input = data.pop(self._input) if self._pop_input else data[self._input] if isinstance(input, Response): selector = Selector(response=input) if hasattr(input, 'url') and input.url and ('url' not in data): # take the URL of the response object data = updated(data, {'url': input.url}) else: selector = Selector(text=input) count = 0 for entry, data_ in self._select_and_extract(selector, self.query, data): data_ = updated(data_, {self._output: entry.extract()}) # now get associated xpaths, css, etc for selectors_dict, entry_method in ((self._xpaths, entry.xpath), (self._csss, entry.css)): if not selectors_dict: continue for key in selectors_dict: selector_ = selectors_dict[key] key_extracted = entry_method(selector_).extract() if not len(key_extracted): # TODO: warning, make mandatory to have a hit if expected? continue if len(key_extracted) > 1: if self._allow_multiple: data_[key] = key_extracted # raise NotImplementedError("Don't know what to do yet with this one") else: lgr.warn( "Got multiple selections for xpath query %s. " "Keeping only the first one: %s" % (repr(selector_), key_extracted[0])) data_[key] = key_extracted[0] else: data_[key] = key_extracted[0] count += 1 yield data_ if self._min_count and count < self._min_count: raise ValueError("Did not match required %d matches (got %d) using %s" % (self._min_count, count, self)) if self._max_count and count > self._max_count: raise ValueError("Matched more than %d matches (got %d) using %s" % (self._max_count, count, self)) class ScrapyExtractorMatch(ExtractorMatch): EXTRACTOR = None # defined in subclasses def _select_and_extract(self, selector, query, data): # there must be some magic happening since originally # (when just parsing the code) those are unbound (func) # and then somehow become bound if PY3 and isinstance(self.EXTRACTOR, types.MethodType): self.EXTRACTOR = self.EXTRACTOR.__func__ for extracted in self.EXTRACTOR(selector, query): yield extracted, data class xpath_match(ScrapyExtractorMatch): EXTRACTOR = Selector.xpath class css_match(ScrapyExtractorMatch): EXTRACTOR = Selector.css class AExtractorMatch(ExtractorMatch): """Helper to simplify matching of URLs based on their HREF or text """ # abstract -- must be defined in subclass #_TARGET = None def _select_and_extract(self, selector, query, data): prev_url = data.get('url', None) url_query = re.compile(query) for url_e in selector.xpath('//a'): url = url_href = url_e.xpath('@href').extract_first() if not url: # it was an without href continue # make it a full URL, if there was an original URL if prev_url: url = dlurljoin(prev_url, url_href) if self._TARGET == 'href': regex_target = url elif self._TARGET == 'text': regex_target = url_e.xpath('text()').extract_first() else: raise ValueError("Unknown _TARGET=%r" % (self._TARGET,)) regex = url_query.match(regex_target) if not regex: continue # enrich data with extracted keywords data_ = data.copy() for k, v in regex.groupdict().items(): data_[k] = v # TODO: such actions we might want to perform also in other cases, # e.g. operating on some extracted with XPATH content data_['url'] = url data_['url_href'] = url_href data_['url_text'] = url_e.xpath('text()').extract_first() lgr.log(5, "Matched %(url)s" % data_) yield url_e, data_ class a_href_match(AExtractorMatch): """Helper to simplify matching based on URL while also extracting various tags from URL while at it Given a URL regular expression pattern, perform matching among available URLs and yield those of interest, while also yielding additional groups which matched. It will return keys 'url', 'url_href', 'url_text' and the original '.extracted()' entry for the url extractor in the field specified by the output argument """ _TARGET = 'href' class a_text_match(AExtractorMatch): """Helper to simplify matching based on a target's text while also extracting various tags from it while at it """ _TARGET = 'text' datalad-0.4.1/datalad/crawler/nodes/misc.py000066400000000000000000000433431301115270200205530ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Various small nodes """ import inspect import os import re import stat import pdb from stat import ST_MODE, S_IEXEC, S_IXOTH, S_IXGRP from os.path import curdir, isdir, isabs, exists, join as opj, split as ops from six import iteritems, string_types from datalad.support.network import get_url_disposition_filename, get_url_straight_filename from datalad.utils import updated from ..pipeline import FinishPipeline from ..pipeline import xrun_pipeline from ..pipeline import PIPELINE_TYPES from ...utils import auto_repr from ...utils import find_files as _find_files from logging import getLogger from nose.tools import eq_, assert_raises lgr = getLogger('datalad.crawler.nodes') @auto_repr class fix_permissions(object): """A node used to check the permissions of a file and set the executable bit """ def __init__(self, file_re='.*', executable=False, input='filename', path=None): """ Parameters ---------- file_re : str, optional Regular expression str to which the filename must contain (re.search) executable : boolean, optional If false, sets file to allow no one to execute. If true, sets file to be executable to those already allowed to read it input : str, optional Key which holds value that is the filename or absolute path in data path : str, optional If absolute path is not specifed in data, path must be expressed here """ self.file_re = file_re self.executable = executable self.input = input self.path = path def __call__(self, data): filename = data.get(self.input) # check that file matches regex if not re.search(self.file_re, filename): yield data return # early termination since nothing to do # check that absolute path exists if not isabs(filename): path = self.path or data.get('path') if path: filename = opj(path, filename) # check existance of file and that path is not a dir if exists(filename) and not isdir(filename): per = oct(os.stat(filename)[ST_MODE])[-3:] st = os.stat(filename) permissions = [S_IEXEC, S_IXGRP, S_IXOTH] if self.executable: # make is executable for those that can read it for i, scope in enumerate(permissions): if per[i] == '6' or '4': os.chmod(filename, st.st_mode | scope) st = os.stat(filename) else: # strip everyone away from executing current = stat.S_IMODE(os.lstat(filename).st_mode) os.chmod(filename, current & ~S_IEXEC & ~S_IXGRP & ~S_IXOTH) nper = oct(os.stat(filename)[ST_MODE])[-3:] lgr.debug('Changing permissions for file %s from %s to %s', filename, per, nper) yield data @auto_repr class Sink(object): """A rudimentary node to sink/collect all the data passed into it """ def __init__(self, keys=None, output=None, ignore_prefixes=['datalad_']): """ Parameters ---------- keys : list of str, optional List of keys to store. If not specified -- entire dictionaries are stored output : str, optional If specified, it will be the key in the yielded data to contain all sunk data ignore_prefixes : list, optional Keys with which prefixes to ignore. By default all 'datalad_' ignored """ self.data = [] self.keys = keys self.output = output self.ignore_prefixes = ignore_prefixes or [] def get_values(self, keys): return [[d[k] for k in keys] for d in self.data] def __call__(self, data): if self.keys: self.data.append({key: data[key] for key in self.keys if key in data}) else: data_ = {k: v for k, v in data.items() if not any(k.startswith(p) for p in self.ignore_prefixes)} self.data.append(data_) if self.output: data = updated(data, {self.output: self.data}) yield data def clean(self): """Clean out collected data""" self.data = [] @auto_repr class rename(object): """Rename some items (i.e. change keys) in data for subsequent nodes """ def __init__(self, assignments): """ Use OrderedDict when order of remapping matters """ assert (isinstance(assignments, dict)) self.assignments = assignments def __call__(self, data): # TODO: unittest data = data.copy() for from_, to_ in self.assignments.items(): if from_ in data: data[to_] = data.pop(from_) yield data # TODO: test @auto_repr class sub(object): """Apply re.sub regular expression substitutions to specified items""" def __init__(self, subs, ok_missing=False): """ Parameters ---------- subs: dict of key -> dict of pattern -> replacement """ self.subs = subs self.ok_missing = ok_missing def __call__(self, data): data = data.copy() for key, subs in self.subs.items(): for from_, to_ in subs.items(): if key not in data and self.ok_missing: continue data[key] = re.sub(from_, to_, data[key]) yield data @auto_repr class assign(object): """Class node to provide assignment of items in data With "interpolate" it allows for insertion of values """ # TODO: may be allow values to be a callable taking data in, so almost like # a node, but not yielding and just returning some value based on the data? def __init__(self, assignments, interpolate=False): """ Parameters ---------- assignments: dict keys: values pairs interpolate: bool, optional Either interpolate provided values using data """ assert (isinstance(assignments, dict)) self.assignments = assignments self.interpolate = interpolate def __call__(self, data): data_ = data.copy() # we need to operate on a copy for k, v in self.assignments.items(): data_[k] = v % data if self.interpolate else v yield data_ # class prune(object): def get_url_filename(data): yield updated(data, {'filename': get_url_straight_filename(data['url'])}) def get_disposition_filename(data): """For the URL request content filename disposition """ yield updated(data, {'filename': get_url_disposition_filename(data['url'])}) class _act_if(object): """Base class for nodes which would act if input data matches specified values Should generally be not used directly. If used directly, no action is taken, besides possibly regexp matched groups populating the `data` """ def __init__(self, values, re=False, negate=False): """ Parameters ---------- values: dict Key/value pairs to compare arrived data against. Would raise FinishPipeline if all keys have matched target values re: bool, optional If specified, values are to be treated as regular expression to be searched. If re and expression includes groups, those will also populate yielded data negate: bool, optional Reverses, so acts (skips, etc) if no match """ self.values = values self.re = re self.negate = negate def __call__(self, data): #import ipdb; ipdb.set_trace() comp = re.search if self.re else lambda x, y: x == y matched = True # finds if all match data_ = data.copy() for k, v in iteritems(self.values): res = k in data and comp(v, data[k]) if not res: # do nothing and pass the data further matched = False break if self.re: assert(res) data_.update(res.groupdict()) for v in (self._act_mismatch if matched != self.negate else self._act_match)(data_): yield v # By default, nothing really is done -- so just produce the same # data. Sub-classes will provide specific custom actions in one # or another case def _act_mismatch(self, data): return [data] def _act_match(self, data): return [data] @auto_repr class interrupt_if(_act_if): """Interrupt further pipeline processing whenever obtained data matches provided value(s)""" def _act_mismatch(self, data): raise FinishPipeline @auto_repr class skip_if(_act_if): """Skip (do not yield anything) further pipeline processing whenever obtained data matches provided value(s)""" def _act_mismatch(self, data): return [] # nothing will be yielded etc @auto_repr class continue_if(_act_if): """Continue if matched An inverse of skip_if""" def _act_match(self, data): return [] @auto_repr class range_node(object): """A node yielding incrementing integers in a data item (output by default) Primarily for testing """ def __init__(self, n, output='output'): self.n = n self.output = output def __call__(self, data={}): for i in range(self.n): yield updated(data, {self.output: i}) def _string_as_list(x): if isinstance(x, string_types): return [x] return x def func_to_node(func, data_args=(), data_kwargs=(), kwargs={}, outputs=(), **orig_kwargs): """Generate a node out of an arbitrary function If provided function returns a generator, each item is returned separately Parameters ---------- data_args: list or tuple, optional Names of keys in data values of which will be given to the func data_kwargs: list or tuple, optional Names of keys in data which will be given to the func as keyword arguments kwargs: dict, optional Additional keyword arguments to pass into the function outputs: str or list or tuple, optional Names of keys to store output of the function in """ def func_node(data): kwargs_ = kwargs.copy() kwargs_.update(orig_kwargs) args_ = [] for k in _string_as_list(data_args): args_.append(data[k]) for k in _string_as_list(data_kwargs): kwargs_[k] = data[k] out = func(*args_, **kwargs_) if not inspect.isgenerator(out): # for uniform handling below out = [out] for return_value in out: if outputs: outputs_ = _string_as_list(outputs) # we need to place it into the copy of data data_ = data.copy() if len(outputs_) > 1: # we were requested to have multiple outputs, # then the function should have provided matching multiple values assert (len(return_value) == len(outputs_)) else: return_value = [return_value] # for uniformity for k, v in zip(outputs_, return_value): data_[k] = v yield data_ else: # TODO: does it make sense to yield the same multiple times ever # if we don't modify it??? yield data in_args = list(_string_as_list(data_args)) + list(_string_as_list(data_kwargs)) func_node.__doc__ = """Function %s wrapped into a node. It expects %s keys to be provided in the data and output will be assigned to %s keys in the output %s """ % ( func.__name__ if hasattr(func, '__name__') else '', ', '.join(in_args) if in_args else "no", ', '.join(_string_as_list(outputs)) if outputs else "no", (' Additional keyword arguments: %s' % kwargs) if kwargs else "" ) return func_node # TODO: come up with a generic function_to_node adapter # actually there is already func_to_node but it is not clear how it works on # generators... in this commit also added **orig_kwargs to it @auto_repr class find_files(object): """Find files matching a regular expression in a pre-specified directory By default in current directory """ def __init__(self, regex, fail_if_none=False, dirs=False, topdir=curdir): """ Parameters ---------- regex: basestring topdir: basestring, optional Directory where to search dirs: bool, optional Either to match directories fail_if_none: bool, optional Fail if no file matched throughout the life-time of this object, i.e. counts through multiple runs (if any run had files matched -- it is ok to have no matched files on subsequent run) """ self.regex = regex self.topdir = topdir self.dirs = dirs self.fail_if_none = fail_if_none self._total_count = 0 def __call__(self, data): count = 0 for fpath in _find_files(self.regex, dirs=self.dirs, topdir=self.topdir): lgr.log(5, "Found file %s" % fpath) count += 1 path, filename = ops(fpath) yield updated(data, {'path': path, 'filename': filename}) self._total_count += count if not self._total_count and self.fail_if_none: raise RuntimeError("We did not match any file using regex %r" % self.regex) @auto_repr class switch(object): """Helper node which would decide which sub-pipeline/node to execute based on values in data Example ------- TODO """ def __init__(self, key, mapping, default=None, missing='raise', re=False): """ Parameters ---------- key: str mapping: dict re: bool, optional Either mapping keys define the regular expressions. Note that in case multiple regular expressions match, all of them would "work it" default: node or pipeline, optional Node or pipeline to use if no mapping was found missing: ('raise', 'stop', 'skip'), optional If value is missing in the mapping or key is missing in data (yeah, not differentiating atm), and no default is provided: 'raise' - would just raise KeyError, 'stop' - would not yield anything in effect stopping any down processing, 'skip' - would just yield input data """ self.key = key self.mapping = mapping self.re = re # trying self.missing assigning just for the sake of auto_repr self.missing = self._missing = missing self.default = default @property def missing(self): return self._missing @missing.setter def missing(self, value): assert (value in {'raise', 'skip', 'stop'}) self._missing = value def __call__(self, data): # make decision which sub-pipeline key_value = data[self.key] if not self.re: try: pipelines = [self.mapping[key_value]] except KeyError: pipelines = [] else: # find all the matches pipelines = [ pipeline for regex, pipeline in self.mapping.items() if re.match(regex, key_value) ] if not pipelines: # None has matched if self.default is not None: pipelines = [self.default] elif self.missing == 'raise': raise KeyError( "Found no matches for %s == %r %s %r" % ( self.key, key_value, "matching one of" if self.re else "among specified", list(self.mapping.keys())) ) elif self.missing == 'skip': yield data return elif self.missing == 'stop': return for pipeline in pipelines: if pipeline is None: yield data return if not isinstance(pipeline, PIPELINE_TYPES): # it was a node, return its output gen = pipeline(data) else: gen = xrun_pipeline(pipeline, data) # run and yield each result for out in gen: yield out @auto_repr class debug(object): """Helper node to fall into debugger (pdb for now) whenever node is called""" def __init__(self, node, when='before'): """ Parameters ---------- node: callable when: str, optional Determines when to enter the debugger: - before: right before running the node - after: after it was ran - empty: when node produced no output """ self.node = node self.when = when def __call__(self, data): node = self.node when = self.when if when == 'before': lgr.info("About to run %s node", node) pdb.set_trace() n = 0 for out in node(data): n += 1 yield out if when == 'after' or (when == 'empty' and n == 0): lgr.info("Ran node %s which yielded %d times", node, n) pdb.set_trace() datalad-0.4.1/datalad/crawler/nodes/s3.py000066400000000000000000000273051301115270200201450ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Basic crawler for the web """ import re from distutils.version import LooseVersion import os from os import unlink from os.path import splitext, dirname, basename, curdir from os.path import lexists from os.path import join as opj from boto.s3.key import Key from boto.s3.prefix import Prefix from boto.s3.deletemarker import DeleteMarker import time from six import iteritems from ...utils import updated from ...dochelpers import exc_str from ...support.s3 import get_key_url from ...support.network import iso8601_to_epoch from ...downloaders.providers import Providers from ...downloaders.s3 import S3Downloader from ...downloaders.base import TargetFileAbsent from ..dbs.versions import SingleVersionDB from logging import getLogger lgr = getLogger('datalad.crawl.s3') def get_version_for_key(k, fmt='0.0.%Y%m%d'): """Given a key return a version it identifies to be used for tagging Uses 0.0.YYYYMMDD by default """ t = iso8601_to_epoch(k.last_modified) # format it return time.strftime(fmt, time.gmtime(t)) def _strip_prefix(s, prefix): """A helper to strip the prefix from the string if present""" return s[len(prefix):] if s and s.startswith(prefix) else s class crawl_s3(object): """Given a source bucket and optional prefix, generate s3:// urls for the content """ def __init__(self, bucket, prefix=None, strip_prefix=True, # either to strip leading prefix if provided url_schema='s3', strategy='naive', versionfx=get_version_for_key, repo=None, ncommits=None, recursive=False, versioned=True, ): """ Parameters ---------- bucket: str prefix: str, optional Either to remember redirects for subsequent invocations strip_prefix: bool, optional Either to strip the prefix (if given) off the target paths versionfx: function, optional If not None, to define a version from the last processed key repo: GitRepo, optional Under which to store information about latest scraped version strategy: {'naive', 'commit-versions'}, optional With `naive` strategy no commits are made if there is a deletion, or update event, so a single run should result in a single commit even though interim different "load" could be added to annex under the same filename ncommits: int or None, optional If specified, used as max number of commits to perform. ??? In principle the same effect could be achieved by a node raising FinishPipeline after n'th commit recursive: bool, optional Either to traverse recursively or just list elements at that level versioned: bool, optional Either to expect bucket to be versioned and demand all versions per prefix and generate versioned urls """ self.bucket = bucket if prefix and not prefix.endswith('/'): lgr.warning("ATM we assume prefixes to correspond only to directories, adding /") prefix += "/" self.prefix = prefix self.strip_prefix = strip_prefix self.url_schema = url_schema assert(strategy in {'naive', 'commit-versions'}) self.strategy = strategy self.versionfx = versionfx self.repo = repo self.ncommits = ncommits self.recursive = recursive self.versioned = versioned def __call__(self, data): stats = data.get('datalad_stats', None) url = "s3://%s" % self.bucket if self.prefix: url += "/" + self.prefix.lstrip('/') providers = Providers.from_config_files() downloader = providers.get_provider(url).get_downloader(url) # bucket = provider.authenticator.authenticate(bucket_name, provider.credential) try: _ = downloader.get_status(url) # just to authenticate and establish connection except TargetFileAbsent as exc: lgr.debug("Initial URL %s lead to not something downloader could fetch: %s", url, exc_str(exc)) pass bucket = downloader.bucket assert(bucket is not None) if self.repo: versions_db = SingleVersionDB(self.repo) prev_version = versions_db.version else: prev_version, versions_db = None, None # TODO: we could probably use headers to limit from previously crawled last-modified # for now will be inefficient -- fetch all, sort, proceed kwargs = {} if self.recursive else {'delimiter': '/'} all_versions = (bucket.list_versions if self.versioned else bucket.list)(self.prefix, **kwargs) # Comparison becomes tricky whenever as if in our test bucket we have a collection # of rapid changes within the same ms, so they couldn't be sorted by last_modified, so we resolve based # on them being marked latest, or not being null (as could happen originally), and placing Delete after creation # In real life last_modified should be enough, but life can be as tough as we made it for 'testing' def kf(k, f): """Some elements, such as Prefix wouldn't have any of attributes to sort by""" return getattr(k, f, None) # So ATM it would sort Prefixes first, but that is not necessarily correct... # Theoretically the only way to sort Prefix'es with the rest is traverse that Prefix # and take latest last_modified there but it is expensive, so -- big TODO if ever ;) # ACTUALLY -- may be there is an API call to return sorted by last_modified, then we # would need only a single entry in result to determine the last_modified for the Prefix, thus TODO cmp = lambda k: ( kf(k, 'last_modified'), k.name, kf(k, 'is_latest'), kf(k, 'version_id') != 'null', isinstance(k, DeleteMarker) ) versions_sorted = sorted(all_versions, key=cmp) # attrgetter('last_modified')) # print '\n'.join(map(str, [cmp(k) for k in versions_sorted])) version_fields = ['last-modified', 'name', 'version-id'] def get_version_cmp(k): # this one will return action version_id so we could uniquely identify return kf(k, 'last_modified'), k.name, kf(k, 'version_id') if prev_version: last_modified_, name_, version_id_ = [prev_version[f] for f in version_fields] # roll forward until we get to the element > this # to not breed list copies for i, k in enumerate(versions_sorted): lm, n, vid = get_version_cmp(k) if lm > last_modified_: start = i break elif lm == last_modified_: # go by name/version_id to be matched and then switch to the next one if (n, vid) == (name_, version_id_): start = i+1 # from the next one if stats: stats.increment('skipped') break stats.increment('skipped') versions_sorted = versions_sorted[start:] # a set of items which we have already seen/yielded so hitting any of them again # would mean conflict/versioning is necessary since two actions came for the same item staged = set() strategy = self.strategy e_prev = None ncommits = self.ncommits or 0 # adding None so we could deal with the last commit within the loop without duplicating # logic later outside def update_versiondb(e, force=False): # this way we could recover easier after a crash # TODO: config crawl.crawl_s3.versiondb.saveaftereach=True if e is not None and (force or True): versions_db.version = dict(zip(version_fields, get_version_cmp(e))) for e in versions_sorted + [None]: filename = e.name if e is not None else None if (self.strip_prefix and self.prefix): filename = _strip_prefix(filename, self.prefix) if filename in staged or e is None: # we should finish this one and commit if staged: if self.versionfx and e_prev is not None: version = self.versionfx(e_prev) if version not in stats.versions: stats.versions.append(version) if versions_db: # save current "version" DB so we would know where to pick up from # upon next rerun. Record should contain # last_modified, name, versionid # TODO? what if e_prev was a DeleteMarker??? update_versiondb(e_prev, force=True) if strategy == 'commit-versions': yield updated(data, {'datalad_action': 'commit'}) if self.ncommits: ncommits += 1 if self.ncommits <= ncommits: lgr.debug("Interrupting on %dth commit since asked to do %d", ncommits, self.ncommits) break staged.clear() if e is None: break # we are done staged.add(filename) if isinstance(e, Key): if e.name.endswith('/'): # signals a directory for which we don't care explicitly (git doesn't -- we don't! ;) ) continue url = get_key_url(e, schema=self.url_schema, versioned=self.versioned) # generate and pass along the status right away since we can yield updated( data, { 'url': url, 'url_status': S3Downloader.get_key_status(e, dateformat='iso8601'), 'filename': filename, 'datalad_action': 'annex', }) update_versiondb(e) elif isinstance(e, DeleteMarker): if strategy == 'commit-versions': # Since git doesn't care about empty directories for us makes sense only # in the case when DeleteMarker is not pointing to the subdirectory if not filename.endswith('/'): yield updated(data, {'filename': filename, 'datalad_action': 'remove'}) else: lgr.info("Ignoring DeleteMarker for %s", filename) update_versiondb(e) elif isinstance(e, Prefix): # so we were provided a directory (in non-recursive traversal) assert(not self.recursive) yield updated( data, { 'url': url, 'filename': filename.rstrip('/'), 'datalad_action': 'directory', } ) else: raise ValueError("Don't know how to treat %s" % e) e_prev = e datalad-0.4.1/datalad/crawler/nodes/tests/000077500000000000000000000000001301115270200204015ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/nodes/tests/__init__.py000066400000000000000000000007651301115270200225220ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from datalad.tests.utils import skip_if_no_module skip_if_no_module('scrapy') # e.g. not present under Python3datalad-0.4.1/datalad/crawler/nodes/tests/test_annex.py000066400000000000000000000361321301115270200231300ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from os import listdir from os.path import join as opj, exists, lexists, basename from collections import OrderedDict from datalad.tests.utils import with_tempfile, eq_, ok_, SkipTest from mock import patch from ..annex import initiate_dataset from ..annex import Annexificator from ....tests.utils import assert_equal, assert_in from ....tests.utils import assert_raises from ....tests.utils import assert_true, assert_false from ....tests.utils import with_tree, serve_path_via_http from ....tests.utils import ok_file_under_git from ....tests.utils import ok_file_has_content from ....tests.utils import assert_cwd_unchanged from ....tests.utils import put_file_under_git from ...pipeline import load_pipeline_from_config from ....consts import CRAWLER_META_CONFIG_PATH, DATALAD_SPECIAL_REMOTE, ARCHIVES_SPECIAL_REMOTE from ....support.stats import ActivityStats from ....support.annexrepo import AnnexRepo @with_tempfile(mkdir=True) def test_annexificator_no_git_if_dirty(outdir): eq_(listdir(outdir), []) filepath = opj(outdir, '.myfile') with open(filepath, 'w'): pass eq_(listdir(outdir), ['.myfile']) assert_raises(RuntimeError, Annexificator, path=outdir) eq_(listdir(outdir), ['.myfile']) Annexificator(path=outdir, allow_dirty=True) eq_(sorted(listdir(outdir)), sorted(['.myfile', '.git'])) @with_tempfile(mkdir=True) @with_tempfile() def test_initiate_dataset(path, path2): dataset_path = opj(path, 'test') datas = list(initiate_dataset('template', 'testdataset', path=dataset_path)()) assert_equal(len(datas), 1) data = datas[0] eq_(data['dataset_path'], dataset_path) crawl_cfg = opj(dataset_path, CRAWLER_META_CONFIG_PATH) ok_(exists, crawl_cfg) pipeline = load_pipeline_from_config(crawl_cfg) # by default we should initiate to MD5E backend fname = 'test.dat' f = opj(dataset_path, fname) annex = put_file_under_git(f, content="test", annexed=True) eq_(annex.get_file_backend(f), 'MD5E') # and even if we clone it -- nope -- since persistence is set by Annexificator # so we don't need to explicitly to commit it just in master since that might # not be the branch we will end up working in annex2 = AnnexRepo(path2, url=dataset_path) annex3 = put_file_under_git(path2, 'test2.dat', content="test2", annexed=True) eq_(annex3.get_file_backend('test2.dat'), 'MD5E') raise SkipTest("TODO much more") @with_tree(tree=[ ('d1', ( ('1.dat', '1.dat load'), )) ]) @serve_path_via_http() @with_tempfile(mkdir=True) def _test_annex_file(mode, topdir, topurl, outdir): annex = Annexificator(path=outdir, mode=mode, statusdb='fileattr', options=["-c", "annex.largefiles=exclude=*.txt"]) input = {'url': "%sd1/1.dat" % topurl, 'filename': '1-copy.dat'} tfile = opj(outdir, '1-copy.dat') expected_output = [input.copy()] # nothing to be added/changed output = list(annex(input)) assert_equal(output, expected_output) # addurl is batched, and we haven't forced annex flushing so there should # be a batched process assert_equal(len(annex.repo._batched), 1) assert_raises(AssertionError, ok_file_under_git, tfile, annexed=True) # if we finalize, it should flush batched annexes and commit list(annex.finalize()({})) assert(lexists(tfile)) ok_file_under_git(tfile, annexed=True) if mode == 'full': ok_file_has_content(tfile, '1.dat load') else: # in fast or relaxed mode there must not be any content assert_raises(AssertionError, ok_file_has_content, tfile, '1.dat load') whereis = annex.repo.whereis(tfile) assert_in(annex.repo.WEB_UUID, whereis) # url must have been added assert_equal(len(whereis), 1 + int(mode == 'full')) # TODO: check the url # Neither file should not be attempted to download again, since nothing changed # and by default we do use files db output = list(annex(input)) assert_equal(output, []) # nothing was done, so annex didn't yield data annex.yield_non_updated = True input_with_stats = input.copy() input_with_stats['datalad_stats'] = ActivityStats() output = list(annex(input_with_stats)) assert_equal(output[0]['datalad_stats'], ActivityStats(files=1, urls=1, skipped=1)) # but if we change that file, it should re-download it now with open(opj(topdir, 'd1', '1.dat'), 'a') as f: f.write("+") output = list(annex(input_with_stats)) stats = output[0]['datalad_stats'] stats.downloaded_time = 0 # 2 since we are reusing the same stats download_stats = dict(downloaded=1, downloaded_size=11) if mode == 'full' else {} addskip_stats = dict(add_annex=0, skipped=2, overwritten=0) if mode == 'relaxed' else dict(add_annex=1, skipped=1, overwritten=1) kwargs = download_stats.copy() kwargs.update(addskip_stats) assert_equal(stats, ActivityStats(files=2, urls=2, **kwargs)) # Download into a file which will be added to git # TODO: for now added to git only in full mode. in --fast or --relaxed, still goes to annex # http://git-annex.branchable.com/bugs/treatment_of_largefiles_is_not_working_for_addurl_--fast___40__or_--relaxed__41__/ input = {'url': "%sd1/1.dat" % topurl, 'filename': '1.txt', 'datalad_stats': ActivityStats()} tfile = opj(outdir, '1.txt') output = list(annex(input)) annexed = mode not in {'full'} list(annex.finalize()({})) if not annexed: ok_file_has_content(tfile, '1.dat load+') else: assert_raises(AssertionError, ok_file_has_content, tfile, '1.dat load+') ok_file_under_git(tfile, annexed=annexed) assert_equal(len(output), 1) stats = output[0]['datalad_stats'] # reset varying metric stats.downloaded_time = 0 assert_equal(stats, ActivityStats(files=1, urls=1, add_git=1-int(annexed), add_annex=int(annexed), **download_stats)) # Let's add a file without specifying URL sfilepath = opj(outdir, 'sample.txt') with open(sfilepath, 'w') as f: f.write("sample") ok_file_has_content(sfilepath, "sample") output = list(annex({'filename': 'sample.txt', 'datalad_stats': ActivityStats()})) ok_file_under_git(sfilepath, annexed=False) assert(output) assert_equal(output[0]['datalad_stats'], ActivityStats(files=1, add_git=1)) def test_annex_file(): for mode in ('full', 'fast', 'relaxed',): yield _test_annex_file, mode @assert_cwd_unchanged() # we are passing annex, not chpwd @with_tree(tree={'1.tar': {'file.txt': 'load', '1.dat': 'load2'}}) def _test_add_archive_content_tar(direct, repo_path): mode = 'full' annex = Annexificator(path=repo_path, allow_dirty=True, mode=mode, direct=direct, special_remotes=[DATALAD_SPECIAL_REMOTE, ARCHIVES_SPECIAL_REMOTE], options=["-c", "annex.largefiles=exclude=*.txt and exclude=SOMEOTHER"]) output_add = list(annex({'filename': '1.tar'})) # adding it to annex assert_equal(output_add, [{'filename': '1.tar'}]) #stats = ActivityStats() #output_add[0]['datalad_stats'] = ActivityStats() output_addarchive = list( annex.add_archive_content( existing='archive-suffix', delete=True, strip_leading_dirs=True,)(output_add[0])) assert_equal(output_addarchive, [{'datalad_stats': ActivityStats(add_annex=1, add_git=1, files=3, renamed=2), 'filename': '1.tar'}]) if not direct: # Notimplemented otherwise assert_true(annex.repo.dirty) annex.repo.commit("added") ok_file_under_git(repo_path, 'file.txt', annexed=False) ok_file_under_git(repo_path, '1.dat', annexed=True) assert_false(lexists(opj(repo_path, '1.tar'))) if not direct: # Notimplemented otherwise assert_false(annex.repo.dirty) def test_add_archive_content_tar(): for direct in (True, False): yield _test_add_archive_content_tar, direct @assert_cwd_unchanged() @with_tempfile(mkdir=True) @with_tree(tree={'file': 'load'}) @serve_path_via_http def test_add_dir_file(repo_path, p, topurl): # test whenever file becomes a directory and then back a file. Should all work! annex = Annexificator(path=repo_path, auto_finalize=False) url = "%s/file" % topurl path1 = opj(repo_path, 'd') data1 = {'filename': 'd', 'url': url} out1 = list(annex(data1)) # becomes a directory which carries a file data2 = {'filename': 'f', 'url': url, 'path': 'd'} # but since we didn't commit previous file yet -- should puke! assert_raises(RuntimeError, list, annex(data2)) list(annex.finalize()({})) # so it gets committed ok_file_under_git(path1, annexed=True) # and after that it should proceed normally #import pdb; pdb.set_trace() out2 = list(annex(data2)) path2 = opj(repo_path, 'd', 'f') ok_(exists(path2)) # tricky one -- becomes back a file... what if repo was dirty and files under dir were staged? TODO assert_raises(RuntimeError, list, annex(data1)) list(annex.finalize()({})) # so it gets committed ok_file_under_git(path2, annexed=True) list(annex(data1)) list(annex.finalize()({})) # so it gets committed ok_file_under_git(path1, annexed=True) # with auto_finalize (default) it should go smoother ;) annex = Annexificator(path=repo_path) list(annex(data2)) # wouldn't happen without explicit finalize to commit whatever new is staged # ok_file_under_git(path2, annexed=True) list(annex(data1)) list(annex.finalize()({})) # so it gets committed ok_file_under_git(path1, annexed=True) def test_commit_versions(): raise SkipTest("TODO: is tested only as a part of test_openfmri.py") @with_tempfile(mkdir=True) def test_remove_other_versions(repo_path): annex = Annexificator(path=repo_path, create=True) class version_db: version = '1.0.0' versions = OrderedDict([ ('10.0.0', {}), # wrong order -- will fail and we will pop it ('1.0.0', {'a': 'a_1.0.0'}), ('2.0.0', {'b': 'b_2.0.0', 'c': 'c_2.0.0'}), ('2.0.1', {'c': 'c_2.0.1'}), # if no overlay, b is gone, overlay=2 should keep b ('2.1.1', {'c': 'c_2.1.1'}), # b should disappear if overlay=2, but stay if ('3', {'d': 'd_3'}), ]) assert_raises(ValueError, annex.remove_other_versions, 'name', db=version_db) data = {'datalad_stats': ActivityStats()} rov = annex.remove_other_versions(db=version_db) # generator # we have that incorrectly ordered version inside assert_raises(AssertionError, next, rov(data)) version_db.versions.pop('10.0.0') # remove the abuser class Unlinker(object): def __init__(self): self.unlinked = [] self.allfiles = set() # Let's also record all present files for vfs in version_db.versions.values(): self.allfiles.update(vfs.values()) self._remaining = self.allfiles.copy() def Find_files(self): """To provide another mock callable""" def find_files(*args, **kwargs): assert_equal(kwargs.get('topdir'), repo_path) # return full path return [opj(repo_path, x) for x in self._remaining] return find_files def __call__(self, s): bs = basename(s) assert(bs in self._remaining) self.unlinked.append(bs) self._remaining.remove(bs) @property def remaining(self): # strip the repopath return self._remaining def _check(version, remaining=None, unlinked=None, **kwargs): """Helper - for a given version and kwargs for remove_ - what to expect""" version_db.version = version data = {'datalad_stats': ActivityStats()} rov = annex.remove_other_versions(db=version_db, **kwargs) # generator with patch('os.unlink', new_callable=Unlinker) as cmunlink, \ patch('os.path.lexists', return_value=True),\ patch('datalad.crawler.nodes.annex.find_files', new_callable=cmunlink.Find_files): out = list(rov(data)) assert_equal(len(out), 1) eq_(out[0]['datalad_stats'].versions, [version]) if remaining is not None: eq_(cmunlink.remaining, remaining) if unlinked is not None: eq_(cmunlink.unlinked, unlinked) eq_(cmunlink.remaining.union(cmunlink.unlinked), cmunlink.allfiles) def check(*args, **kwargs): _check(*args, **kwargs) kwargs = kwargs.copy() # without remove unversioned # in our test results should be the same kwargs['remove_unversioned'] = True _check(*args, **kwargs) # remove_unversioned=True check('1.0.0', remaining={'a_1.0.0'}) # even though due to overlay=0 all versions are identical, there were no # version before 1.0.0, and all later are removed regardless or overlay check('1.0.0', remaining={'a_1.0.0'}, overlay=0) check('2.0.0', remaining={'b_2.0.0', 'c_2.0.0'}) check('2.0.0', remaining={'b_2.0.0', 'c_2.0.0'}, overlay=1) check('2.0.0', remaining={'b_2.0.0', 'c_2.0.0'}, overlay=lambda x: x[:1]) # but with overlay=0 we would also get files from 1 check('2.0.0', remaining={'a_1.0.0', 'b_2.0.0', 'c_2.0.0'}, overlay=0) check('2.0.1', remaining={'c_2.0.1'}) check('2.0.1', remaining={'b_2.0.0', 'c_2.0.1'}, overlay=1) check('2.0.1', remaining={'b_2.0.0', 'c_2.0.1'}, overlay=2) check('2.0.1', remaining={'a_1.0.0', 'b_2.0.0', 'c_2.0.1'}, overlay=0) check('2.1.1', remaining={'c_2.1.1'}) check('2.1.1', remaining={'c_2.1.1'}, overlay=2) check('2.1.1', remaining={'b_2.0.0', 'c_2.1.1'}, overlay=1) check('2.1.1', remaining={'a_1.0.0', 'b_2.0.0', 'c_2.1.1'}, overlay=0) check('3', remaining={'d_3'}) check('3', remaining={'d_3'}, overlay=1) check('3', remaining={'d_3'}, overlay=2) check('3', remaining={'a_1.0.0', 'b_2.0.0', 'c_2.1.1', 'd_3'}, overlay=0) # if name changes version_db.versions['2.1.1'] = {'c00.dat': 'c00_2.1.1'} # if we don't do anything special, it would have its own life: check('2.1.1', remaining={'c00_2.1.1'}) check('2.1.1', remaining={'c00_2.1.1'}, overlay=2) check('2.1.1', remaining={'b_2.0.0', 'c_2.0.1', 'c00_2.1.1'}, overlay=1) check('2.1.1', remaining={'a_1.0.0', 'b_2.0.0', 'c_2.0.1', 'c00_2.1.1'}, overlay=0) # but we should be able to specify to "unify" unversioned name more by providing # replacement pattern kw = dict(fpath_subs=[('c00', 'c'), ('\.dat', '')]) check('2.1.1', remaining={'b_2.0.0', 'c00_2.1.1'}, overlay=1, **kw) check('2.1.1', remaining={'a_1.0.0', 'b_2.0.0', 'c00_2.1.1'}, overlay=0, **kw) datalad-0.4.1/datalad/crawler/nodes/tests/test_crawl_url.py000066400000000000000000000056351301115270200240150ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # now with some recursive structure of directories from glob import glob from os.path import exists, join as opj from datalad.tests.utils import eq_, ok_ from datalad.tests.utils import serve_path_via_http, with_tree from datalad.tests.utils import skip_if_scrapy_without_selector skip_if_scrapy_without_selector() from ..crawl_url import crawl_url from ..crawl_url import parse_checksums from ..matches import a_href_match from ...pipeline import run_pipeline from ....tests.utils import assert_equal from ....tests.utils import assert_false from ....tests.utils import SkipTest from ....utils import updated pages_loop = dict( tree=( ('index.html', """ mypage2 mypage4 """), ('page2.html', 'mypage3'), ('page3.html', 'root'), ('page4.html', 'mypage5'), ('page5.html', 'mypage3'), ) ) @with_tree(**pages_loop) @serve_path_via_http() def test_recurse_loop_http(path, url): def visit(url, matchers): return sorted((d['url'].replace(url, '') for d in crawl_url(url, matchers=matchers)())) target_pages = ['', 'page2.html', 'page3.html', 'page4.html', 'page5.html'] eq_(visit(url, [a_href_match('.*')]), target_pages) # test recursive loop as implemented by pipeline crawler = crawl_url(url) pipeline = [ {'output': 'outputs'}, # just to simplify testing here crawler, [ {'loop': True, 'output': 'input+outputs'}, a_href_match('.*'), crawler.recurse ] ] pipeline_results = run_pipeline(pipeline) eq_(sorted([d['url'].replace(url, '') for d in pipeline_results]), target_pages) def test_parse_checksums(): response = """\ abc f1.txt bcd d1/f1.txt """ out = list(parse_checksums(digest='md5')( {'response': response, 'url': 'http://example.com/subdir/checksums.md5'})) assert_equal(len(out), 2) assert_equal(out[0]['path'], '') assert_equal(out[0]['filename'], 'f1.txt') assert_equal(out[0]['url'], 'http://example.com/subdir/f1.txt') assert_equal(out[1]['path'], 'd1') assert_equal(out[1]['filename'], 'f1.txt') assert_equal(out[1]['url'], 'http://example.com/subdir/d1/f1.txt') datalad-0.4.1/datalad/crawler/nodes/tests/test_matches.py000066400000000000000000000102451301115270200234400ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import inspect from nose import SkipTest from datalad.tests.utils import ok_, eq_, assert_raises from datalad.tests.utils import skip_if_scrapy_without_selector skip_if_scrapy_without_selector() from ..matches import * try: import scrapy except ImportError: raise SkipTest("Needs scrapy") class sample1: # there could be some html normalization effects, so using " within for now a_htmls = ( '/', 'somewhere', 'else' ) a_texts = ['/', 'somewhere', 'else'] a_url_hrefs = ['/', 'buga/duga/du', 'http://example.com'] class1_texts = [None, 'somewhere', None] response = """
' %s%s%s magic
""" % a_htmls def _test_match_basic(matcher, query): extracts = dict( xpaths={'text': 'text()'}, csss={'favorite': '.class1::text'} ) m = matcher(query, **extracts) mg = m(dict(response="
")) ok_(inspect.isgenerator(mg)) eq_(list(mg), []) # there should be no hits mg = m(dict(response=sample1.response)) ok_(inspect.isgenerator(mg)) hits = list(mg) eq_(len(hits), 3) for hit, a_html, a_text, class1_text in zip( hits, sample1.a_htmls, sample1.a_texts, sample1.class1_texts): ok_(hit['response']) eq_(hit['match'], a_html) eq_(hit['text'], a_text) eq_(hit.get('favorite', None), class1_text) m = matcher(query, min_count=4, **extracts) mg = m(dict(response=sample1.response)) ok_(inspect.isgenerator(mg)) assert_raises(ValueError, list, mg) m = matcher(query, max_count=2, **extracts) mg = m(dict(response=sample1.response)) ok_(inspect.isgenerator(mg)) assert_raises(ValueError, list, mg) def test_match_basic(): yield _test_match_basic, xpath_match, '//a' yield _test_match_basic, css_match, 'a' yield _test_match_basic, a_href_match, '.*' def test_a_href_match_basic(): m = a_href_match('.*') mg = m(dict(response=sample1.response)) ok_(inspect.isgenerator(mg)) hits = list(mg) eq_(len(hits), 3) eq_([u['url_text'] for u in hits], sample1.a_texts) eq_([u['url_href'] for u in hits], sample1.a_url_hrefs) # nothing done to url eq_([u['url'] for u in hits], sample1.a_url_hrefs) # if we do provide original url where it comes from -- result urls should be full mg = m(dict(response=sample1.response, url="http://w.example.com:888/d/")) ok_(inspect.isgenerator(mg)) hits = list(mg) eq_(len(hits), 3) eq_([u['url_text'] for u in hits], sample1.a_texts) eq_([u['url_href'] for u in hits], sample1.a_url_hrefs) eq_([u['url'] for u in hits], ['http://w.example.com:888/', 'http://w.example.com:888/d/buga/duga/du', 'http://example.com']) def test_a_href_match_pattern1(): m = a_href_match('.*buga/(?P.*)/.*') hits = list(m(dict(response=sample1.response))) eq_(len(hits), 1) hit = hits[0] eq_(hit['url'], 'buga/duga/du') eq_(hit['custom'], 'duga') def test_a_href_match_pattern2(): m = a_href_match('.*(?P.a).*') hits = list(m(dict(response=sample1.response))) eq_(len(hits), 2) eq_([u['url'] for u in hits], ['buga/duga/du', 'http://example.com']) eq_([u['custom'] for u in hits], ['ga', 'xa']) def test_a_href_match_pattern3(): # that we would match if top url was provided as well m = a_href_match('.*(?Pbu..).*') hits = list(m(dict(response=sample1.response, url="http://w.buxxxx.com/"))) eq_(len(hits), 2) eq_([u['url'] for u in hits], ['http://w.buxxxx.com/', 'http://w.buxxxx.com/buga/duga/du']) eq_([u['custom'] for u in hits], ['buxx', 'buga']) datalad-0.4.1/datalad/crawler/nodes/tests/test_misc.py000066400000000000000000000414311301115270200227500ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import os from stat import * from os import chmod from os.path import join as opj from datalad.tests.utils import with_tempfile, eq_, ok_, SkipTest from six import next from collections import OrderedDict from ..misc import get_disposition_filename from ..misc import fix_permissions from ..misc import get_url_filename from ..misc import range_node from ..misc import interrupt_if from ..misc import skip_if from ..misc import continue_if from ..misc import func_to_node from ..misc import sub from ..misc import find_files from ..misc import switch from ..misc import assign from ..misc import _act_if from ..misc import rename from ..misc import debug from ..misc import Sink from ...pipeline import FinishPipeline from ....tests.utils import with_tree from ....utils import updated from ...tests.test_pipeline import _out from datalad.tests.utils import skip_if_no_network from datalad.tests.utils import use_cassette from datalad.tests.utils import ok_generator from datalad.tests.utils import assert_in from datalad.tests.utils import assert_re_in from datalad.tests.utils import assert_equal from datalad.tests.utils import assert_false from datalad.tests.utils import swallow_logs import logging from mock import patch from nose.tools import eq_, assert_raises from nose import SkipTest # TODO: redo on a local example # TODO: seems vcr fetches entire response not just the header which makes this test url # in particular not appropriate #@skip_if_no_network #@use_cassette('brain-map.org-1', return_body='') def test_get_disposition_filename(): input = {'url': 'http://human.brain-map.org/api/v2/well_known_file_download/157722290'} with patch('datalad.crawler.nodes.misc.get_url_disposition_filename', return_value="T1.nii.gz"): output = list(get_disposition_filename(input)) eq_(len(output), 1) eq_(output[0]['filename'], 'T1.nii.gz') @with_tempfile(mkdir=True) def test_fix_permissions(outdir): filepath = opj(outdir, 'myfile.txt') filepath2 = opj(outdir, 'badfile.py') filepath3 = opj(outdir, 'nopath.txt') with open(filepath, 'w'), open(filepath2, 'w'), open(filepath3, 'w'): pass gen = fix_permissions('\.(txt|csv)', True, 'filename') # make file executable for those that can read it filename = opj(outdir, 'myfile.txt') chmod(filename, 0o643) data = {'url': 'http://mapping.org/docs/?num=45', 'filename': filename} eq_(list(gen(data)), [{'url': 'http://mapping.org/docs/?num=45', 'filename': filename}]) eq_(oct(os.stat(filename)[ST_MODE])[-3:], '753') # file that does not match regex badfile = opj(outdir, 'badfile.py') chmod(badfile, 0o666) baddata = {'url': 'http://mapping.org/docs/?num=45', 'filename': badfile} eq_(list(gen(baddata)), [{'url': 'http://mapping.org/docs/?num=45', 'filename': badfile}]) eq_(oct(os.stat(badfile)[ST_MODE])[-3:], '666') # file that is actually a dir dirdata = {'url': 'http://mapping.org/docs/?num=45', 'filename': outdir} eq_(list(gen(dirdata)), [{'url': 'http://mapping.org/docs/?num=45', 'filename': outdir}]) # path given in args nopath = opj(outdir, 'nopath.txt') chmod(nopath, 0o643) datafile = {'url': 'http://mapping.org/docs/?num=45', 'filename': 'nopath.txt'} gen = fix_permissions('\.txt', True, 'filename', outdir) eq_(list(gen(datafile)), [{'url': 'http://mapping.org/docs/?num=45', 'filename': 'nopath.txt'}]) eq_(oct(os.stat(filename)[ST_MODE])[-3:], '753') # take permissions away from everyone gen = fix_permissions('.txt', False, 'filename') filename = opj(outdir, 'myfile.txt') chmod(filename, 0o743) data = {'url': 'http://mapping.org/docs/?num=45', 'filename': filename} eq_(list(gen(data)), [{'url': 'http://mapping.org/docs/?num=45', 'filename': filename}]) eq_(oct(os.stat(filename)[ST_MODE])[-3:], '642') def test_get_url_filename(): input = {'url': 'http://human.brain-map.org/api/v2/well_known_file_download/157722290'} output = list(get_url_filename(input)) eq_(len(output[0]), 2) eq_(len(output), 1) eq_(output, [{'url': 'http://human.brain-map.org/api/v2/well_known_file_download/157722290', 'filename': '157722290'}]) def test_sink(): data = {'x': 'y', 'g': 'h', 'a': 'b'} keys = ['x', 'a'] nomatch = ['z'] # no arguments genempty = Sink() eq_(list(genempty(data)), [{'a': 'b', 'x': 'y', 'g': 'h'}]) # check the internal state eq_(list(genempty.data), [{'a': 'b', 'x': 'y', 'g': 'h'}]) # if key for sunk data is specified gen = Sink(output='result') eq_(list(gen(data)), [{'a': 'b', 'x': 'y', 'result': [{'a': 'b', 'x': 'y', 'g': 'h'}], 'g': 'h'}]) eq_(list(gen.data), [{'a': 'b', 'x': 'y', 'g': 'h'}]) # if list of keys is specified genkeys = Sink(keys, 'result') eq_(list(genkeys(data)), [{'a': 'b', 'x': 'y', 'result': [{'a': 'b', 'x': 'y'}], 'g': 'h'}]) eq_(list(genkeys.data), [{'a': 'b', 'x': 'y'}]) # if list of keys has no match gentwo = Sink(nomatch, 'result') eq_(list(gentwo(data)), [{'a': 'b', 'result': [{}], 'x': 'y', 'g': 'h'}]) eq_(list(gentwo.data), [{}]) # check that data's key/value pair matches will be sunk again eq_(list(gentwo(data)), [{'a': 'b', 'x': 'y', 'result': [{}, {}], 'g': 'h'}]) eq_(list(gentwo.data), [{}, {}]) def test_get_values(): data = {'x': 'y', 'g': 'h', 'a': 'b'} keys = ['x', 'a'] gen = Sink() list(gen(data)) eq_(gen.get_values(keys), [['y', 'b']]) def test_assign(): data = {'x': 'y'} gen = assign({'key': 'value', 'aa': 'bb %(x)s'}, interpolate=True) eq_(list(gen(data)), [{'key': 'value', 'aa': 'bb y', 'x': 'y'}]) genf = assign({'key': 'value %(x)s'}, interpolate=False) eq_(list(genf(data)), [{'key': 'value %(x)s', 'x': 'y'}]) datadup = {'x': 'z'} gen = assign({'x': 'value', 'g': 'y %(x)s'}, interpolate=True) eq_(list(gen(datadup)), [{'x': 'value', 'g': 'y z'}]) def test__act_if(): values = {'x': 'y', 'a': 'b'} data = {'x': 'y'} datamatch = {'x': 'y', 'a': 'b'} # matched = true gen = _act_if(values) eq_(list(gen(datamatch)), [datamatch]) # matched = false genfal = _act_if(values, re=True, negate=False) eq_(list(genfal(data)), [{'x': 'y'}]) # matched = true gent = _act_if(values, re=True, negate=True) eq_(list(gent(datamatch)), [{'a': 'b', 'x': 'y'}]) def test_rename(): data = {'x': 'y'} datamulti = {'x': 'y', 'aa': 'bb'} gen = rename({'x': 'newkey', 'aa': 'bb'}) genmulti = rename({'x': 'newkey', 'aa': 'newerkey'}) eq_(list(gen(data)), [{'newkey': 'y'}]) eq_(list(genmulti(datamulti)), [{'newkey': 'y', 'newerkey': 'bb'}]) def test_range_node(): eq_(list(range_node(1)()), [{'output': 0}]) eq_(list(range_node(2)()), [{'output': 0}, {'output': 1}]) def test_interrupt_if(): n = interrupt_if({'v1': 'done'}) assert_raises(FinishPipeline, next, n(dict(v1='done'))) assert_raises(FinishPipeline, next, n(dict(v1='done', someother=123))) tdict = dict(v1='not yet', someother=123) # and that we would interrupt while matching multiple values eq_(list(n(tdict)), [tdict]) assert_raises(FinishPipeline, next, interrupt_if(tdict)(tdict)) eq_(list(interrupt_if({'v1': 'ye.$'})(tdict)), [tdict]) assert_raises(FinishPipeline, next, interrupt_if({'v1': 'ye.$'}, re=True)(tdict)) def test_skip_if(): n = skip_if({'v1': 'done'}) eq_(list(n(dict(v1='done'))), []) eq_(list(n(dict(v1='not done'))), [{'v1': 'not done'}]) eq_(list(n(dict(v1='done', someother=123))), []) tdict = dict(v1='not yet', someother=123) # and that we would interrupt while matching multiple values eq_(list(n(tdict)), [tdict]) eq_(list(skip_if(tdict)(tdict)), []) eq_(list(skip_if({'v1': 'ye.$'})(tdict)), [tdict]) eq_(list(skip_if({'v1': 'ye.$'}, re=True)(tdict)), []) def test_continue_if(): d = {'v1': 'done'} n = continue_if(d) #eq_(list(n(d)), [d]) eq_(list(n(dict(v1='not done'))), []) eq_(list(n(dict(v1='done', someother=123))), [dict(v1='done', someother=123)]) tdict = dict(v1='not yet', someother=123) # and that we would interrupt while matching multiple values eq_(list(n(tdict)), []) eq_(list(continue_if(tdict)(tdict)), [tdict]) # regexp eq_(list(continue_if({'v1': '^(?Pnot +)yet$'}, re=True)(tdict)), [updated(tdict, {'negate': 'not '})]) def test_skip_if_negate(): n = skip_if({'v1': 'done'}, negate=True) eq_(list(n(dict(v1='done'))), [dict(v1='done')]) eq_(list(n(dict(v1='not done'))), []) eq_(list(n(dict(v1='done', someother=123))), [dict(v1='done', someother=123)]) tdict = dict(v1='done', someother=123) # and that we would interrupt while matching multiple values eq_(list(n(tdict)), [tdict]) eq_(list(skip_if(tdict, negate=True)(tdict)), [tdict]) eq_(list(skip_if({'v1': 'don.$'}, negate=True)(tdict)), []) eq_(list(skip_if({'v1': 'don.$'}, re=True, negate=True)(tdict)), [tdict]) def test_func_to_node(): int_node = func_to_node(int) # node which requires nothing and nothing of output is used assert int_node.__doc__ in_dict = {'in': 1} ok_generator(int_node(in_dict)) # xrange is not considered to be a generator def xrange_(n, offset=0): for x in range(offset, offset + n): yield x xrange_node = func_to_node(xrange_, data_args='in', outputs='out') assert_in('assigned to out', xrange_node.__doc__) assert_false('Additional keyword arguments' in xrange_node.__doc__) range_node_gen = xrange_node(in_dict) ok_generator(range_node_gen) assert_equal(list(range_node_gen), [{'in': 1, 'out': 0}]) # with additional kwargs xrange_node = func_to_node(xrange_, data_args='in', outputs='out', kwargs={'offset': 10}) assert_in('assigned to out', xrange_node.__doc__) assert_in('Additional keyword arguments', xrange_node.__doc__) range_node_gen = xrange_node(in_dict) ok_generator(range_node_gen) assert_equal(list(range_node_gen), [{'in': 1, 'out': 10}]) # testing func_node data = {'offset': 5, 'in': 1} xrange_node = func_to_node(xrange_, data_args='in', data_kwargs=['offset'], outputs='out') assert_in('assigned to out', xrange_node.__doc__) assert_false('Additional keyword arguments' in xrange_node.__doc__) gen = xrange_node(data) ok_generator(gen) assert_equal(list(gen), [{'offset': 5, 'out': 5, 'in': 1}]) # with multiple outputs def split_(s, num): yield s.split('/', num) data = {'num': 3, 'in': 'datalad/crawler/nodes'} split_node = func_to_node(split_, data_args='in', data_kwargs=['num'], outputs=['a', 'b', 'c']) assert_in('assigned to a, b, c', split_node.__doc__) assert_false('Additional keyword arguments' in split_node.__doc__) split_node_gen = split_node(data) assert_equal(list(split_node_gen), [{'a': 'datalad', 'c': 'nodes', 'b': 'crawler', 'num': 3, 'in': 'datalad/crawler/nodes'}]) def test_sub(): s = sub({ 'url': { '(http)s?(://.*openfmri\.s3\.amazonaws.com/|://s3\.amazonaws\.com/openfmri/)': r'\1\2' } }) ex1 = {'url': 'http://example.com'} assert_equal(list(s(ex1)), [ex1]) assert_equal(list(s({'url': "https://openfmri.s3.amazonaws.com/tarballs/ds001_raw.tgz?param=1"})), [{'url': "http://openfmri.s3.amazonaws.com/tarballs/ds001_raw.tgz?param=1"}]) assert_equal( list(s({ 'url': "https://s3.amazonaws.com/openfmri/tarballs/ds031_retinotopy.tgz?versionId=HcKd4prWsHup6nEwuIq2Ejdv49zwX5U"})), [{ 'url': "http://s3.amazonaws.com/openfmri/tarballs/ds031_retinotopy.tgz?versionId=HcKd4prWsHup6nEwuIq2Ejdv49zwX5U"}] ) @with_tree(tree={'1': '1', '1.txt': '2'}) def test_find_files(d): assert_equal(sorted(list(sorted(x.items())) for x in find_files('.*', topdir=d)({})), [[('filename', '1'), ('path', d)], [('filename', '1.txt'), ('path', d)]]) assert_equal(list(find_files('.*\.txt', topdir=d)({})), [{'path': d, 'filename': '1.txt'}]) assert_equal(list(find_files('notmatchable', topdir=d)({})), []) assert_raises(RuntimeError, list, find_files('notmatchable', topdir=d, fail_if_none=True)({})) # and fail_if_none should operate globally i.e. this should be fine ff = find_files('.*\.txt', topdir=d, fail_if_none=True) assert_equal(list(ff({})), [{'path': d, 'filename': '1.txt'}]) os.unlink(opj(d, '1.txt')) assert_equal(list(ff({})), []) def test_switch(): ran = [] def n2(data): for i in range(2): ran.append(len(ran)) yield updated(data, {'f2': 'x_%d' % i}) switch_node = switch( 'f1', { 1: sub({'f2': {'_': '1'}}), # should be able to consume nodes and pipelines 2: [n2], } ) out = list(switch_node({'f1': 1, 'f2': 'x_'})) assert_equal(out, [{'f1': 1, 'f2': 'x1'}]) assert_equal(ran, []) # but in the 2nd case, the thing is a sub-pipeline so it behaves as such without spitting # out its output out = list(switch_node({'f1': 2, 'f2': 'x_'})) assert_equal(out, _out([{'f1': 2, 'f2': 'x_'}])) assert_equal(ran, [0, 1]) # but does execute just fine # if there is a value mapping doesn't exist for, by default would fail data_missing = {'f1': 3, 'f2': 'x_'} with assert_raises(KeyError) as cme: list(switch_node(data_missing)) assert_in('Found no matches for f1 == 3 among', str(cme.exception)) assert_raises(KeyError, list, switch_node(data_missing)) switch_node.missing = 'skip' assert_equal(list(switch_node(data_missing)), [data_missing]) switch_node.missing = 'stop' assert_equal(list(switch_node(data_missing)), []) # and if there is a default -- we should be all good switch_node.default = sub({'f2': {'_': '_default'}}) assert_equal(list(switch_node(data_missing)), [{'f1': 3, 'f2': 'x_default'}]) # and if we make it output all outputs, we would get them! switch_node.mapping[2].insert(0, {'output': 'outputs'}) out = list(switch_node({'f1': 2, 'f2': 'x_'})) assert_equal(out, _out([{'f1': 2, 'f2': 'x_0'}, {'f1': 2, 'f2': 'x_1'}])) def test_switch_re(): ran = [] def n2(data): for i in range(2): ran.append(len(ran)) yield updated(data, {'f2': 'x_%d' % i}) switch_node = switch( 'f1', OrderedDict([ ('m[13]', sub({'f2': {'_': '1'}})), # should be able to consume nodes and pipelines ('m[23]', [n2]), ('emp.*', None), # just return input ]), re=True ) out = list(switch_node({'f1': 'm123', 'f2': 'x_'})) assert_equal(out, [{'f1': 'm123', 'f2': 'x1'}]) assert_equal(ran, []) # if there is a value mapping doesn't exist for, by default would fail data_missing = {'f1': 'xxxxx', 'f2': 'x_'} with assert_raises(KeyError) as cme: list(switch_node(data_missing)) assert_re_in('Found no matches for f1 == .xxxxx. matching one of', cme.exception.args) # but in the 2nd case, the thing is a sub-pipeline so it behaves as such without spitting # out its output out = list(switch_node({'f1': 'm2', 'f2': 'x_'})) assert_equal(out, _out([{'f1': 'm2', 'f2': 'x_'}])) assert_equal(ran, [0, 1]) # but does execute just fine # and if matches both -- we need to get all outputs for i in range(len(ran)): ran.remove(i) out = list(switch_node({'f1': 'm3', 'f2': 'x_'})) assert_equal(out, [{'f1': 'm3', 'f2': 'x1'}] + _out([{'f1': 'm3', 'f2': 'x_'}])) assert_equal(ran, [0, 1]) # and does execute just as fine # empty match out = list(switch_node({'f1': 'empty', 'f2': 'x_'})) assert_equal(out, [{'f1': 'empty', 'f2': 'x_'}]) def _test_debug(msg, args=()): if 'empty' in args: def node(d): # just so Python marks it as a generator if False: yield d # pragma: no cover else: return else: def node(d): yield updated(d, {'debugged': True}) d1 = debug(node, *args) data = {'data': 1} with patch('pdb.set_trace') as set_trace: with swallow_logs(new_level=logging.INFO) as cml: list(d1(data)) set_trace.assert_called_once_with() cml.assert_logged(msg, level='INFO') def test_debug(): yield _test_debug, "About to run" yield _test_debug, "Ran node .* which yielded 1 times", ('after',) yield _test_debug, "Ran node .* which yielded 0 times", ('empty',)datalad-0.4.1/datalad/crawler/nodes/tests/test_s3.py000066400000000000000000000221551301115270200223440ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # now with some recursive structure of directories import logging from ..s3 import crawl_s3 from ..s3 import _strip_prefix from ..s3 import get_key_url from ..misc import switch from ..annex import Annexificator from ...pipeline import run_pipeline from ....consts import DATALAD_SPECIAL_REMOTE from ....downloaders.tests.utils import get_test_providers from ....support.stats import ActivityStats from ....utils import swallow_logs from ....utils import rmtree from ....tests.utils import eq_ from ....tests.utils import assert_not_equal from ....tests.utils import assert_in, assert_not_in from ....tests.utils import skip_if_no_network from ....tests.utils import use_cassette from ....tests.utils import externals_use_cassette from ....tests.utils import with_tempfile def _annex(path): annex = Annexificator(path, special_remotes=[DATALAD_SPECIAL_REMOTE]) url = 's3://datalad-test0-versioned' providers = get_test_providers(url) # to skip if no credentials # start with a fresh bucket each time so we could reuse the same vcr tapes work providers.get_provider(url).get_downloader(url).reset() return annex target_version = '0.0.20151107' @skip_if_no_network @use_cassette('test_crawl_s3') @with_tempfile def test_crawl_s3(path): annex = _annex(path) # For now a very simple one which doesn't give a damn about files being removed # so we just get the "most recent existed" view of all of them without having commits # for previous versions but annex processing them thus doing all house-keeping # necessary pipeline = [ [ crawl_s3('datalad-test0-versioned', strategy='naive', repo=annex.repo), annex ], annex.finalize() ] with externals_use_cassette('test_crawl_s3-pipeline1'): out = run_pipeline(pipeline) # things are committed and thus stats are empty eq_(out, [{'datalad_stats': ActivityStats()}]) total_stats = out[0]['datalad_stats'].get_total() eq_(set(total_stats.versions), {target_version}) # we have a bunch of them since not uniq'ing them and they are all the same total_stats.versions = [] eq_(total_stats, ActivityStats(files=14, overwritten=5, downloaded=14, urls=14, add_annex=14, downloaded_size=112)) # if we rerun -- nothing new should have been done. I.e. it is the # and ATM we can reuse the same cassette with externals_use_cassette('test_crawl_s3-pipeline1'): out = run_pipeline(pipeline) eq_(out, [{'datalad_stats': ActivityStats(skipped=17)}]) eq_(out[0]['datalad_stats'].get_total(), ActivityStats(skipped=17)) @skip_if_no_network @use_cassette('test_crawl_s3_commit_versions') @with_tempfile def test_crawl_s3_commit_versions(path): annex = _annex(path) # Fancier setup so we could do any of desired actions within a single sweep pipeline = [ crawl_s3('datalad-test0-versioned', strategy='commit-versions', repo=annex.repo), switch('datalad_action', { 'commit': annex.finalize(tag=True), 'remove': annex.remove, 'annex': annex, }) ] with externals_use_cassette('test_crawl_s3-pipeline1'): with swallow_logs(new_level=logging.WARN) as cml: out = run_pipeline(pipeline) assert_in("There is already a tag %s" % target_version, cml.out) # things are committed and thus stats are empty eq_(out, [{'datalad_stats': ActivityStats()}]) total_stats = out[0]['datalad_stats'].get_total() eq_(set(total_stats.versions), {target_version}) # we have a bunch of them since not uniq'ing them and they are all the same # override for easier checking total_stats.versions = [] eq_(total_stats, # Deletions come as 'files' as well atm ActivityStats(files=17, overwritten=3, downloaded=14, urls=14, add_annex=14, removed=3, downloaded_size=112)) tags = annex.repo.repo.tags assert_in(target_version, tags) # and we actually got 7 more commits for t in range(1, 8): assert_in(target_version + "+%d" % t, tags) # if we rerun -- nothing new should have been done. I.e. it is the # and ATM we can reuse the same cassette with externals_use_cassette('test_crawl_s3-pipeline1'): with swallow_logs() as cml: out = run_pipeline(pipeline) assert_not_in("There is already a tag %s" % target_version, cml.out) eq_(out, [{'datalad_stats': ActivityStats(skipped=17)}]) eq_(out[0]['datalad_stats'].get_total(), ActivityStats(skipped=17)) # Really nothing was done @skip_if_no_network @use_cassette('test_crawl_s3_commit_versions_one_at_a_time') @with_tempfile def test_crawl_s3_commit_versions_one_at_a_time(path): annex = _annex(path) # Fancier setup so we could do any of desired actions within a single sweep pipeline = [ crawl_s3('datalad-test0-versioned', strategy='commit-versions', repo=annex.repo, ncommits=1), switch('datalad_action', { 'commit': annex.finalize(tag=True), 'remove': annex.remove, 'annex': annex, }) ] with externals_use_cassette('test_crawl_s3-pipeline1'): with swallow_logs(new_level=logging.WARN) as cml: out = run_pipeline(pipeline) assert_not_in("There is already a tag %s" % target_version, cml.out) # things are committed and thus stats are empty eq_(out, [{'datalad_stats': ActivityStats()}]) total_stats_all = total_stats = out[0]['datalad_stats'].get_total() eq_(total_stats, # Deletions come as 'files' as well atm ActivityStats(files=3, downloaded=3, urls=3, add_annex=3, downloaded_size=24, versions=[target_version])) # and there should be 7 more, every time changing the total stats for t in range(1, 8): with externals_use_cassette('test_crawl_s3-pipeline1'): with swallow_logs(new_level=logging.WARN) as cml: out = run_pipeline(pipeline) assert_in("There is already a tag %s" % target_version, cml.out) total_stats_ = out[0]['datalad_stats'].get_total() assert_not_equal(total_stats, total_stats_) total_stats = total_stats_ total_stats_all += total_stats # with total stats at the end to be the same as if all at once total_stats_all.versions = [] eq_(total_stats_all, # Deletions come as 'files' as well atm ActivityStats(files=17, skipped=72, overwritten=3, downloaded=14, urls=14, add_annex=14, removed=3, downloaded_size=112)) # # Theoretically could be a test without invoking S3 where file gets later renamed into a directory # and the other way around. annex should handle that. So this one serves more as integration # test # @skip_if_no_network @use_cassette('test_crawl_s3_file_to_directory') @with_tempfile def test_crawl_s3_file_to_directory(path): annex = _annex(path) # with auto_finalize (default), Annexificator will finalize whenever it runs into a conflict pipeline = [ crawl_s3('datalad-test1-dirs-versioned', repo=annex.repo, recursive=True), # annex switch('datalad_action', { 'commit': annex.finalize(tag=True), 'remove': annex.remove, 'annex': annex, }) ] with externals_use_cassette('test_crawl_s3_file_to_directory-pipeline1'): with swallow_logs() as cml: out = run_pipeline(pipeline) assert(annex.repo.dirty) list(annex.finalize()(out[0])) # things are committed and thus stats are empty eq_(out, [{'datalad_stats': ActivityStats()}]) total_stats_all = total_stats = out[0]['datalad_stats'].get_total() eq_(total_stats, # Deletions come as 'files' as well atm ActivityStats(files=3, downloaded=3, overwritten=2, urls=3, add_annex=3, downloaded_size=12, versions=['0.0.20160303'])) def test_strip_prefix(): eq_(_strip_prefix('', ''), '') eq_(_strip_prefix('abab', 'ab'), 'ab') eq_(_strip_prefix('cabab', 'ab'), 'cabab') eq_(_strip_prefix('', 'ab'), '') eq_(_strip_prefix('ab', 'ab'), '') def test_crawl_s3_prefix(): # verify correct handling of prefix while defining the node node = crawl_s3('datalad-test0-versioned') eq_(node.prefix, None) with swallow_logs(new_level=logging.WARN) as cml: node = crawl_s3('datalad-test0-versioned', prefix="dir") assert_in('adding /', cml.out) eq_(node.prefix, 'dir/') def test_get_key_url(): # Just to provide necessary structure class e: name = 'e' version_id = '123' class bucket: name = "bucket" eq_(get_key_url(e), 'http://bucket.s3.amazonaws.com/e?versionId=123') eq_(get_key_url(e, versioned=False), 'http://bucket.s3.amazonaws.com/e')datalad-0.4.1/datalad/crawler/oldconfig/000077500000000000000000000000001301115270200200735ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/oldconfig/__init__.py000066400000000000000000000010041301115270200221770ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Old configuration handling for the crawler """ __docformat__ = 'restructuredtext' from .base import EnhancedConfigParser datalad-0.4.1/datalad/crawler/oldconfig/base.py000066400000000000000000000143251301115270200213640ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from six import iteritems from datalad.support.configparserinc import SafeConfigParserWithIncludes from datalad.support.archives import DECOMPRESSORS import logging lgr = logging.getLogger('datalad.config') # # Configuration # class EnhancedConfigParser(SafeConfigParserWithIncludes): """Enhanced ConfigParser to support evaluations of the values etc """ @staticmethod def get_default(sections={}): cfg = EnhancedConfigParser(defaults=dict( mode='download', # TODO -- check, use orig='auto', # TODO -- now we don't use it meta_info='True', # TODO -- now we just use it directory="%(__name__)s", archives_re='(%s)' % ('|'.join(DECOMPRESSORS.keys())), # 'keep' -- just keep original without annexing it etc # 'annex' -- git annex addurl ... # 'drop' -- 'annex add(url)' and then 'annex drop' upon extract # 'rm' -- remove upon extraction if archive # 'auto': # if incoming == public: # 'auto' == 'rm' if is_archive and 'annex' if # else: # 'auto' == 'annex' incoming_destiny="auto", # TODO: # maintain - preserve the ones in the archive and place them # at the same level as the archive file (TODO) # strip - would remove leading extracted directory if a single one archives_directories="strip", incoming="repos/incoming/EXAMPLE", public="repos/public/EXAMPLE", include_href='', include_href_a='', exclude_href='', exclude_href_a='', filename='filename', # Checks! check_url_limit='0', # no limits # unused... we might like to enable it indeed and then be # smart with our actions in extracting archives into # directories which might contain those files, so some might # need to be annexed and some directly into .git # TODO #annex='.*', # # Q: or should we utilize annex.largefiles syntax etc? # cons: for fast/relaxed mode etc we might not kow the size # NOTE: that in rm, keep modes those will not be added either since # those modes set it up for handling archives git_add=None, # which files (regexp) should be added to git directly. Implies that # they need to be downloaded first recurse=None, # do not recurse by default, otherwise regex on urls to assume being for directories )) # TODO: verify correct input config. Currently would not fail if some # unknown option (e.g. add_git) is used for section, options in iteritems(sections): if section != 'DEFAULT': cfg.add_section(section) for opt, value in iteritems(options): cfg.set(section, opt, value) return cfg def get_section(self, section): """Return container representing the section """ return ConfigSection(self, section) class ConfigSection(object): """Configuration section providing additional enhanced handling of option specifications """ def __init__(self, config, section): self._config = config self._section = section self._env = {} # If present -- execute the 'exec' option to populate the environment if self._config.has_option(section, 'exec'): exec_ = self._config.get(section, 'exec') lgr.debug("Executing exec=%r in section %s to populate _env" % (exec_, section)) exec(exec_, self._env) def get(self, option, default=None, vars=None, raw=False, **kwargs): """get() with support of _e (evaluate) in `context` For an option X supports a set of ways how to obtain the actual value 'X' plain value 'X_e' evaluation of the expression in the 'context'. `vars` in this case are not passed into the expression for the interpolation, and should be used as regular variables in evaluation N.B. String interpolations are also in effect as provided by stock ConfigParser If multiple ways (e.g. X and X_e) are present -- _e takes precedence. """ options = [option + suf for suf in ('_e', '') if self._config.has_option(self._section, option+suf)] if len(options) > 1: lgr.debug("Found multiple specifications: %s. Using first one" % (options,)) elif len(options) == 0: return default option = options[0] # what environment/variables would be available for # interpolation/evaluation vars_all = {} if self._env: vars_all.update(self._env) if vars: vars_all.update(vars) value = self._config.get(self._section, option, vars={} if option.endswith('_e') else vars_all, raw=raw, **kwargs) if option.endswith('_e') and not raw: value_ = eval(value, vars_all) return value_ else: return value def __getitem__(self, k): return self.get(k) def load_config(configs): # Load configuration cfg = EnhancedConfigParser.get_default() cfg_read = cfg.read(configs) assert set(configs).issubset(cfg_read), \ "Not all configs were read. Wanted: %s Read: %s" % (configs, cfg_read) return cfg datalad-0.4.1/datalad/crawler/oldconfig/tests/000077500000000000000000000000001301115270200212355ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/oldconfig/tests/__init__.py000066400000000000000000000000001301115270200233340ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/oldconfig/tests/test_config.py000066400000000000000000000040421301115270200241130ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from datalad.tests.utils import * from ..base import load_config @with_tempfile(suffix='.cfg') def test_eval_value(filename): with open(filename, 'w') as f: f.write(""" [DEFAULT] v1 = 1+2 # simple evaluation v2_e = str(1+2) # Simple string interpolations v4 = %(var)s # mixing both string interpolations and evaluation exec = from math import e func = lambda x: "fun^%%d" %% x # Function to be evaluated with some local variable substitution funccall_e = func(%(l)s) [section1] v5_e = "%%.2f(%(v1)s)" %% (var+1) var_e = "%%.2f" %% (var+2) e_e = e v1_array_e = [%(v1)s] l = 3 [section2] l = 4 [section3] l = 1 exec = func = lambda x: "NEW" """) # this one is a tricky one -- probably could still work via iterative refinement of vars # while catching InterpolationMissingOptionError # v3 = %(v2)s.bak cfg = load_config([filename]) dcfg = cfg.get_section('DEFAULT') eq_(dcfg.get('v1'), '1+2') eq_(dcfg.get('v2_e'), '3') eq_(dcfg.get('v2_e', raw=True), 'str(1+2)') eq_(dcfg.get('v2'), '3') eq_(dcfg.get('v4', vars=dict(var='1.3333')), '1.3333') scfg1 = cfg.get_section('section1') eq_(scfg1.get('v5', vars=dict(var=1.3333)), '2.33(1+2)') eq_(scfg1.get('var', vars=dict(var=1.3333)), '3.33') # now check if exec worked correctly and enriched environment # for those evaluations import math eq_(scfg1.get('e'), math.e) eq_(scfg1.get('v1_array'), [ 3 ]) eq_(scfg1.get('funccall'), 'fun^3') scfg2 = cfg.get_section('section2') eq_(scfg2.get('funccall'), 'fun^4') scfg3 = cfg.get_section('section3') # exec was redefined eq_(scfg3.get('funccall'), 'NEW') eq_(scfg3.get('l'), '1') datalad-0.4.1/datalad/crawler/pipeline.py000066400000000000000000000522221301115270200203110ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Pipeline functionality. A pipeline is represented by a simple list or tuple of nodes or other nested pipelines. Each pipeline node is a callable which receives a dictionary (commonly named `data`), does some processing, and yields (once or multiple times) a derived dictionary (commonly a shallow copy of original dict). For a node to be parametrized it should be implemented as a callable (i.e. define __call__) class, which could obtain parameters in its constructor. TODO: describe PIPELINE_OPTS and how to specify them for a given (sub-)pipeline. The `data` dictionary is used primarily to carry the scraped/produced data, but besides that it will carry few items which some nodes might use. All those item names will start with the `datalad_` prefix, and will be intended for 'inplace' modifications or querying. The following items are planned to be provided by the pipeline runner: `datalad_settings` PipelineSettings object which could be used to provide configuration for the current run of the pipeline. E.g.: - dry: either nodes are intended not to perform any changes which would reflect on disk - skip_existing: `datalad_stats` ActivityStats/dict object to accumulate statistics on what has been done by the nodes so far To some degree, we could make an analogy when `blood` is to `data` and `venous system` is to `pipeline`. Blood delivers various elements which are picked up by various parts of our body when they know what to do with the corresponding elements. To the same degree nodes can consume, augment, or produce new items to the `data` and send it down the stream. Since there is no strict typing or specification on what nodes could consume or produce (yet), no verification is done and things can go utterly wrong. So nodes must be robust and provide informative logging. """ __dev_doc__ = """ somewhat similar loose/flexible pipelining in Python approaches - https://github.com/freeman-lab/pipeit """ import sys from glob import glob from os.path import dirname, join as opj, isabs, exists, curdir, basename from os import makedirs from ..consts import CRAWLER_META_DIR, HANDLE_META_DIR, CRAWLER_META_CONFIG_PATH from ..consts import CRAWLER_META_CONFIG_FILENAME from ..utils import updated from ..dochelpers import exc_str from ..support.gitrepo import GitRepo from ..support.network import parse_url_opts from ..support.stats import ActivityStats from ..support.exceptions import PipelineNotSpecifiedError from ..support.configparserinc import SafeConfigParserWithIncludes from logging import getLogger lgr = getLogger('datalad.crawler.pipeline') # name of the section in the config file which would define pipeline parameters CRAWLER_PIPELINE_SECTION = 'crawl:pipeline' CRAWLER_PIPELINE_SECTION_DEPRECATED = 'crawler' class FinishPipeline(Exception): """Exception to use to signal that any given pipeline should be stopped """ pass # options which could augment behavior of the pipeline, could be specified # only on top of it PIPELINE_OPTS = dict( # nested_pipeline_inherits_opts=True, # would use or not values yielded by the nested pipeline output='input', # last-output, outputs, input+outputs loop=False, # either to feed results into itself (until None returned) ) # which data types depict object being a pipeline PIPELINE_TYPES = (list, tuple) def reset_pipeline(pipeline): """Given a pipeline, traverse its nodes and call .reset on them Note: it doesn't try to call reset if a node doesn't have it """ if pipeline: for node in pipeline: if isinstance(node, PIPELINE_TYPES): reset_pipeline(node) elif hasattr(node, '__call__') and hasattr(node, 'reset'): lgr.log(2, "Resetting node %s" % node) node.reset() def run_pipeline(*args, **kwargs): """Run pipeline and assemble results into a list By default, the pipeline returns only its input (see PIPELINE_OPTS), so if no options for the pipeline were given to return additional items, a `[{}]` will be provided as output """ output = list(xrun_pipeline(*args, **kwargs)) if output: if 'datalad_stats' in output[-1]: stats = output[-1]['datalad_stats'].get_total() stats_str = stats.as_str(mode='line') else: stats_str = 'no stats collected' else: stats_str = "no output" lgr.info("Finished running pipeline: %s" % stats_str) return output if output else None def _get_pipeline_opts(pipeline): """Return options and pipeline steps to be ran given the pipeline "definition" Definition might have options as the first element """ opts = PIPELINE_OPTS.copy() if isinstance(pipeline[0], dict): newopts, pipeline = (pipeline[0], pipeline[1:]) opts = updated(opts, newopts) return opts, pipeline def xrun_pipeline(pipeline, data=None, stats=None, reset=True): """Yield results from the pipeline. """ id_pipeline = "Pipe #%s" % id(pipeline) def _log(msg, *args): """Helper for uniform debug messages""" lgr.log(5, "%s: " + msg, id_pipeline, *args) _log("%s", pipeline) if reset: _log("Resetting pipeline") reset_pipeline(pipeline) # just for paranoids and PEP8-disturbed, since theoretically every node # should not change the data, so having default {} should be sufficient data = data or {} if 'datalad_stats' in data: if stats is not None: raise ValueError("We were provided stats to use, but data has already datalad_stats") else: data = updated(data, {'datalad_stats': stats or ActivityStats()}) if not len(pipeline): return # options for this pipeline opts, pipeline = _get_pipeline_opts(pipeline) # verify that we know about all specified options unknown_opts = set(opts).difference(set(PIPELINE_OPTS)) if unknown_opts: raise ValueError("Unknown pipeline options %s" % str(unknown_opts)) data_to_process = [data] output = opts['output'] if output not in ('input', 'last-output', 'outputs', 'input+outputs'): raise ValueError("Unknown output=%r" % output) if opts['loop'] and output == 'input': lgr.debug("Assigning output='last-output' for sub-pipeline since we want " "to loop until pipeline returns anything") output_sub = 'last-output' else: output_sub = output log_level = lgr.getEffectiveLevel() data_out = None while data_to_process: _log("processing data. %d left to go", len(data_to_process)) data_in = data_to_process.pop(0) try: for idata_out, data_out in enumerate(xrun_pipeline_steps(pipeline, data_in, output=output_sub)): if log_level <= 3: # provide details of what keys got changed # TODO: unify with 2nd place where it was invoked lgr.log(3, "O3: +%s, -%s, ch%s, ch?%s", *_compare_dicts(data_in, data_out)) _log("got new %dth output", idata_out) if opts['loop']: _log("extending list of data to process due to loop option") data_to_process.append(data_out) if 'outputs' in output: _log("yielding output") yield data_out except FinishPipeline as e: # TODO: decide what we would like to do -- skip that particular pipeline run # or all subsequent or may be go back and only skip that generated result _log("got a signal that pipeline is 'finished'") # TODO: this implementation is somewhat bad since all the output logic is # duplicated within xrun_pipeline_steps, but it is probably unavoidable because of # loop option if output == 'last-output': if data_out: _log("yielding last-output") yield data_out # Input should be yielded last since otherwise it might ruin the flow for typical # pipelines which do not expect anything beyond going step by step # We should yield input data even if it was empty if 'input' in output: _log("finally yielding input data as instructed") yield data def xrun_pipeline_steps(pipeline, data, output='input'): """Actually run pipeline steps, feeding yielded results to the next node and yielding results back. Recursive beast which runs a single node and then recurses to run the rest, possibly multiple times if the current node is a generator. It yields output from the node/nested pipelines, as directed by the output argument. """ if not len(pipeline): return node, pipeline_tail = pipeline[0], pipeline[1:] if isinstance(node, (list, tuple)): lgr.debug("Pipe: %s" % str(node)) # we have got a step which is yet another entire pipeline pipeline_gen = xrun_pipeline(node, data, reset=False) if pipeline_gen: # should be similar to as running a node data_in_to_loop = pipeline_gen else: # pipeline can return None, and in such a case # just do not process further, since if it completed # normally, its input would have been provided back lgr.log(7, "Pipeline generator %s returned None", node) data_in_to_loop = [] prev_stats = None # we do not care to check if entire pipeline drops stats # since it is done below at the node level else: # it is a "node" which should generate (or return) us an iterable to feed # its elements into the rest of the pipeline lgr.debug("Node: %s" % node) prev_stats = data.get('datalad_stats', None) # so we could check if the node doesn't dump it data_in_to_loop = node(data) log_level = lgr.getEffectiveLevel() data_out = None if data_in_to_loop: for data_ in data_in_to_loop: if prev_stats is not None: new_stats = data_.get('datalad_stats', None) if new_stats is None or new_stats is not prev_stats: lgr.debug("Node %s has changed stats to %s from %s. Updating and using previous one", node, prev_stats, new_stats) if new_stats is not None: prev_stats += new_stats data_['datalad_stats'] = prev_stats if log_level <= 4: # provide details of what keys got changed stats_str = data_['datalad_stats'].as_str(mode='line') if 'datalad_stats' in data_ else '' lgr.log(4, "O1: +%s, -%s, ch%s, ch?%s %s", *(_compare_dicts(data, data_) + (stats_str,))) if pipeline_tail: lgr.log(7, " pass %d keys into tail with %d elements", len(data_), len(pipeline_tail)) lgr.log(5, " passed keys: %s", data_.keys()) for data_out in xrun_pipeline_steps(pipeline_tail, data_, output=output): if log_level <= 3: # provide details of what keys got changed # TODO: difference from previous stats! stats_str = data_['datalad_stats'].as_str(mode='line') if 'datalad_stats' in data_ else '' lgr.log(3, "O2: +%s, -%s, ch%s, ch?%s %s", *(_compare_dicts(data, data_out) + (stats_str,))) if 'outputs' in output: yield data_out else: data_out = data_ if 'outputs' in output: yield data_out elif pipeline_tail: lgr.warning("%s returned None, although there is still a tail in the pipeline" % node) if output == 'last-output' and data_out: yield data_out def _compare_dicts(d1, d2): """Given two dictionaries, return what keys were added, removed, changed or might be changed """ added, removed, changed, maybe_changed = [], [], [], [] all_keys = set(d1).union(set(d2)) for k in all_keys: if k not in d1: added.append(k) elif k not in d2: removed.append(k) else: if d1[k] is d2[k]: continue else: try: if d1[k] != d2[k]: changed.append(k) except: # MIH: TypeError? maybe_changed.append(k) return added, changed, removed, maybe_changed def initiate_pipeline_config(template, template_func=None, template_kwargs=None, path=curdir, commit=False): """ TODO Gergana ;) """ lgr.debug("Creating crawler configuration for template %s under %s", template, path) crawl_config_dir = opj(path, CRAWLER_META_DIR) if not exists(crawl_config_dir): lgr.log(2, "Creating %s", crawl_config_dir) makedirs(crawl_config_dir) crawl_config_repo_path = opj(CRAWLER_META_DIR, CRAWLER_META_CONFIG_FILENAME) crawl_config = opj(crawl_config_dir, CRAWLER_META_CONFIG_FILENAME) cfg_ = SafeConfigParserWithIncludes() cfg_.add_section(CRAWLER_PIPELINE_SECTION) cfg_.set(CRAWLER_PIPELINE_SECTION, 'template', template) if template_func: cfg_.set(CRAWLER_PIPELINE_SECTION, 'func', template_func) for k, v in (template_kwargs or {}).items(): cfg_.set(CRAWLER_PIPELINE_SECTION, "_" + k, str(v)) with open(crawl_config, 'w') as f: cfg_.write(f) if commit: repo = GitRepo(path) repo.add(crawl_config_repo_path) if repo.dirty: repo.commit("Initialized crawling configuration to use template %s" % template, _datalad_msg=True) else: lgr.debug("Repository is not dirty -- not committing") return crawl_config def load_pipeline_from_module(module, func=None, args=None, kwargs=None, return_only=False): """Load pipeline from a Python module Parameters ---------- module: str Module name or filename of the module from which to load the pipeline func: str, optional Function within the module to use. Default: `pipeline` args: list or tuple, optional Positional arguments to provide to the function. kwargs: dict, optional Keyword arguments to provide to the function. return_only: bool, optional flag true if only to return pipeline """ func = func or 'pipeline' args = args or tuple() kwargs = kwargs or {} # mod = __import__('datalad.crawler.pipelines.%s' % module, fromlist=['datalad.crawler.pipelines']) dirname_ = dirname(module) assert(module.endswith('.py')) try: sys.path.insert(0, dirname_) modname = basename(module)[:-3] # to allow for relative imports within "stock" pipelines if dirname_ == opj(dirname(__file__), 'pipelines'): mod = __import__('datalad.crawler.pipelines.%s' % modname, fromlist=['datalad.crawler.pipelines']) else: mod = __import__(modname, level=0) if return_only: return getattr(mod, func) return getattr(mod, func)(*args, **kwargs) except Exception as e: raise RuntimeError("Failed to import pipeline from %s: %s" % (module, exc_str(e))) finally: if dirname_ in sys.path: path = sys.path.pop(0) if path != dirname_: lgr.warning("Popped %s when expected %s. Restoring!!!" % (path, dirname_)) sys.path.insert(0, path) def _find_pipeline(name): """Given a name for a pipeline, looks for the pipeline under common locations """ def candidates(name): if not name.endswith('.py'): name += '.py' # first -- current directory repo_path = GitRepo.get_toppath(curdir) if repo_path: yield opj(repo_path, CRAWLER_META_DIR, 'pipelines', name) # TODO: look under other .datalad locations as well # last -- within datalad code yield opj(dirname(__file__), 'pipelines', name) # datalad's module shipped within it for candidate in candidates(name): if exists(candidate): lgr.debug("Found pipeline %s under %s", name, candidate) return candidate lgr.log(5, "No pipeline %s under %s", name, candidate) return None def load_pipeline_from_template(name, func=None, args=None, kwargs=None, return_only=False): """Given a name, loads that pipeline from datalad.crawler.pipelines and later from other locations Parameters ---------- name: str Name of the pipeline (the template) defining the filename, or the full path to it (TODO), example: openfmri func: str Name of function from which pipeline to run example: superdataset_pipeline args: dict, optional Positional args for the pipeline, passed as `*args` into the pipeline call kwargs: dict, optional Keyword args for the pipeline, passed as `**kwargs` into the pipeline call, example: {'dataset': 'ds000001'} return_only: bool, optional flag true if only to return pipeline """ if isabs(name) or exists(name): raise NotImplementedError("Don't know how to import straight path %s yet" % name) # explicit isabs since it might not exist filename = name \ if (isabs(name) or exists(name)) \ else _find_pipeline(name) if filename: if not exists(filename): raise PipelineNotSpecifiedError("Pipeline file %s is N/A" % filename) else: raise PipelineNotSpecifiedError("could not find pipeline for %s" % name) return load_pipeline_from_module(filename, func=func, args=args, kwargs=kwargs, return_only=return_only) # TODO: we might need to find present .datalad/crawl in another branch if not # present currently def load_pipeline_from_config(path): """Given a path to the pipeline configuration file, instantiate a pipeline Typical example description [crawl:pipeline] pipeline = standard func = pipeline1 _kwarg1 = 1 which would instantiate a pipeline from standard.py module by calling `standard.pipeline1` with `_kwarg1='1'`. This definition is identical to [crawl:pipeline] pipeline = standard?func=pipeline1&_kwarg1=1 so that theoretically we could specify basic pipelines completely within a URL """ cfg_ = SafeConfigParserWithIncludes() cfg_.read([path]) pipeline = None for sec in (CRAWLER_PIPELINE_SECTION, CRAWLER_PIPELINE_SECTION_DEPRECATED): if not cfg_.has_section(sec): continue if sec == CRAWLER_PIPELINE_SECTION_DEPRECATED: lgr.warning("Crawler section was renamed from %s to %s and format has changed" " please adjust", CRAWLER_PIPELINE_SECTION_DEPRECATED, CRAWLER_PIPELINE_SECTION) opts = cfg_.options(sec) # must have template if 'template' not in opts: raise PipelineNotSpecifiedError("%s lacks %r field within %s section" % (path, 'template', sec)) template = cfg_.get(sec, 'template') # parse template spec template_name, url_opts = parse_url_opts(template) # so we will allow to specify options in the url and then also in the section definitions all_opts = updated(url_opts, {o: cfg_.get(sec, o) for o in opts}) template_opts = {k: v for k, v in all_opts.items() if not k.startswith('_')} pipeline_opts = {k[1:]: v for k, v in all_opts.items() if k.startswith('_')} assert not set(template_opts).difference({'template', 'func'}), "ATM we understand only 'func'" pipeline = load_pipeline_from_template( template_name, func=template_opts.get('func', None), kwargs=pipeline_opts) break if pipeline is None: raise IOError("Did not find section %r within %s" % (CRAWLER_PIPELINE_SECTION, path)) return pipeline def get_repo_pipeline_config_path(repo_path=curdir): """Given a path within a repo, return path to the crawl.cfg""" if not exists(opj(repo_path, HANDLE_META_DIR)): # we need to figure out top path for the repo repo_path = GitRepo.get_toppath(repo_path) if not repo_path: return None return opj(repo_path, CRAWLER_META_CONFIG_PATH) def get_repo_pipeline_script_path(repo_path=curdir): """If there is a single pipeline present among 'pipelines/', return path to it""" # TODO: somewhat adhoc etc -- may be improve with some dedicated name being # tracked or smth like that if not exists(opj(repo_path, HANDLE_META_DIR)): # we need to figure out top path for the repo repo_path = GitRepo.get_toppath(repo_path) if not repo_path: return None pipelines = glob(opj(repo_path, CRAWLER_META_DIR, 'pipelines', '*.py')) if len(pipelines) > 1 or not pipelines: return None return pipelines[0] datalad-0.4.1/datalad/crawler/pipelines/000077500000000000000000000000001301115270200201175ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/pipelines/__init__.py000066400000000000000000000007121301115270200222300ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Module containing pre-crafted pipeline generators for crawling """datalad-0.4.1/datalad/crawler/pipelines/abstractsonline.py000066400000000000000000000061451301115270200236720ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Pipeline to scrape abstracts information from online resource (e.g. for SfN 2015)""" from ..nodes.crawl_url import crawl_url from ..nodes.matches import a_href_match, xpath_match from ..nodes.misc import Sink # TODO: migrate to stock nodes package/module. make an io module probably class dump_csv(object): """IO node to dump collected data as a csv""" def __init__(self, keys, filename=None, delimiter='\t'): self.keys = keys self.filename = filename self.delimiter = delimiter def __call__(self, data): # should be taken from data filename = data['filename'] if not self.filename else self.filename # TODO do actual dumping of keys of data as csv file yield data def parse_abstract(data): data = data.copy() # TODO do parsing here and assign all interesting fields to data[] fields yield data def pipeline(mkey=None, outputfile=None): crawler = crawl_url('http://www.abstractsonline.com/plan/start.aspx?mkey={%s}' % mkey) sink_abstracts = Sink(output='abstracts') # fields_sink = sink_dict(key='field', values='raw_value', # output='abstract_fields', # exclude_keys=('', 'Disclosures:')), return [ [ # {'return_last': True, }, crawler, a_href_match('.*/Browse.aspx'), crawler.recurse, a_href_match('.*/BrowseResults.aspx?date=(?P[/0-9]*)'), crawler.recurse, a_href_match('.*/ViewSession.aspx?.*'), crawler.recurse, a_href_match('.*/ViewAbstract.aspx?mID=.*'), #[ crawler.recurse, #fields_sink.reset, #xpath_match('//td[@class="ViewAbstractDataLabel"]', # xpaths={'field': 'normalize-space(text())', # 'raw_value': 'following-sibling::td'}), xpath_match('//table[@cellpadding=3]'), # TODO: do not match one by one, just get entire DIV and parse itout # into interesting fields parse_abstract, sink_abstracts, #return_from_pipeline(['abstract_fields']) #], # process_fields(key='abstract_fields', # funcs={'Program#/Poster#:': extract_td_text, # 'Authors:': fancy_authors_extractor, # {'Presentation time:', 'Presenter at Poster:'}: extract_date, # ...}, # default_func=extract_td_text), # dump_dict(keys=('...'), filename='output.csv') ], dump_csv(keys=('title', 'authors', '...'), filename=outputfile) ]datalad-0.4.1/datalad/crawler/pipelines/balsa.py000066400000000000000000000233001301115270200215510ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """A pipeline for crawling BALSA datasets""" import json import os import re from shutil import rmtree from os import curdir, makedirs, rmdir from os.path import lexists, join as opj, abspath, exists, normpath from collections import OrderedDict # Import necessary nodes from ..nodes.crawl_url import crawl_url from ..nodes.matches import xpath_match, a_href_match from ..nodes.misc import assign, skip_if, find_files, continue_if from ..nodes.misc import debug from ..nodes.misc import sub, switch from ..nodes.annex import Annexificator from ...consts import ARCHIVES_SPECIAL_REMOTE, DATALAD_SPECIAL_REMOTE from ...utils import find_files as f_f, _path_ from ...utils import auto_repr from ...utils import updated from logging import getLogger lgr = getLogger("datalad.crawler.pipelines.balsa") TOPURL = "https://balsa.wustl.edu/" def superdataset_pipeline(url=TOPURL): """ Parameters ---------- url: str URL point to all datasets, hence the URL at the top """ url = opj(url, 'study/') hardcodeNames = {'RVVG': '2016-Glasser_et_al-Multimodal_parcellation', 'W336': '2016-Donahue_et_al-Connection_strength_and_distance', 'WG33': '2016-VanEssen_et_al-BALSA', 'pkGn': '2016-Glasser_et_al-HCP_Neuroimaging_approach', } annex = Annexificator(no_annex=True) lgr.info("Creating a BALSA collection pipeline") return [ crawl_url(url), a_href_match('.*/study/show/(?P.*)'), # skip the empty dataset used by BALSA for testing skip_if({'dataset_id': 'Jvw1'}, re=True), crawl_url(), xpath_match('substring(//*/h3/text(), 8, string-length(//*/h3/text()))', output='dataset'), # assign({'dataset_name': '%(dataset)s'}, interpolate=True), switch('dataset_id', {idkey: assign({'dataset_name': name}) for idkey, name in hardcodeNames.items()}), annex.initiate_dataset( template="balsa", data_fields=['dataset_id'], existing='skip' ) ] def extract_meta(data): content = [x['match'] for x in xpath_match('//*[@class="attributeLabel"]/..')(data)] content = [(re.sub('', ', ', x)) for x in content] content = [(re.sub('<[^<]+?>|[\t|\n|\r]', '', (str(x.encode('ascii', 'ignore'))))).strip() for x in content] [content.remove(x) for x in content if x.find('SCENES:' or 'OWNERS:') >= 0] json_dict = OrderedDict(map(str, x.split(':', 1)) for x in content) if not exists(".datalad/meta"): makedirs(".datalad/meta") with open(_path_(".datalad/meta/balsa.json"), "w") as fi: json.dump(json_dict, fi, indent=1) lgr.info("Generated descriptor file") yield {'filename': ".datalad/meta/balsa.json"} @auto_repr class BalsaSupport(object): def __init__(self, repo): """Verifies that the canoncial tarball contains all files that are individually listed Parameters ---------- repo: str annex repo to which dataset is being annexed """ self.repo = repo def verify_files(self, data): files_path = opj(abspath(curdir), '_files') # list of files that exist from canonical tarball con_files = set(f_f('.*', topdir=curdir, exclude='./(_files|.datalad)')) assert con_files # list of files that are individually downloaded files = set(f_f('.*', topdir='_files')) assert files for item_from_con in con_files: item_compare = normpath(opj('_files', item_from_con)) if item_compare in files: key_item = self.repo.get_file_key(item_from_con) key_files_item = self.repo.get_file_key(item_compare) if key_item == key_files_item: pass else: lgr.warning("%s varies in content from the individually downloaded " "file with the same name, it is removed and file " "from canonical tarball is kept" % item_from_con) files.discard(item_compare) else: lgr.warning("%s does not exist in the individually listed files by name, " "but will be kept from canonical tarball" % item_compare) if files: lgr.warning("The following files do not exist in the canonical tarball, but are " "individually listed files and will not be kept: %s" % files) rmtree(files_path) yield data def fix_the_filename(data): from datalad.utils import file_basename download_ext = file_basename(data['filename'], return_ext=True)[-1] orig_filename, orig_ext = file_basename(data['target_filename'], return_ext=True) # data['filename_orig'] if orig_ext != download_ext: assert(download_ext == 'zip') # we are not aware of other cases assert(orig_ext == 'scene') data = data.copy() # They have a bug that the same name of the archive provided for multiple .scene files # available within the study data['filename'] = orig_filename + '_scene' + '.' + download_ext yield data def pipeline(dataset_id, url=TOPURL): lgr.info("Creating a pipeline for the BALSA dataset %s" % dataset_id) annex = Annexificator(create=False, statusdb='json', special_remotes=[ARCHIVES_SPECIAL_REMOTE, DATALAD_SPECIAL_REMOTE], options=["-c", "annex.largefiles=" "exclude=Makefile and exclude=LICENSE* and exclude=ISSUES*" " and exclude=CHANGES* and exclude=README*" " and exclude=*.[mc] and exclude=dataset*.json" " and exclude=*.txt" " and exclude=*.json" " and exclude=*.tsv" ]) if not exists("_files"): makedirs("_files") def splitpath(data): data = data.copy() fullpath = data.pop('path') path = os.path.dirname(fullpath) if path: data['path'] = path data['filename'] = os.path.basename(fullpath) yield data files_url = opj(url, 'file/show/') # files_url = https://balsa.wustl.edu/file/show/ url = opj(url, 'study/') # url = https://balsa.wustl.edu/study/ dataset_url = '%sshow/%s' % (url, dataset_id) # url = https://balsa.wustl.edu/study/show/[dataset_id] balsa = BalsaSupport(repo=annex.repo) from datalad.downloaders.providers import Providers providers = Providers.from_config_files() balsa_downloader = providers.get_provider(url).get_downloader(url) def get_disposition_filename(data): yield updated(data, {'filename': balsa_downloader.get_status(data['url']).filename}) # BALSA has no versioning atm, so no changelog either return [ annex.switch_branch('incoming'), [ crawl_url(url), [ assign({'dataset': dataset_id}), skip_if({'dataset': 'test study upload'}, re=True), ], [ crawl_url(dataset_url), [ extract_meta, annex, ], [ # canonical tarball a_href_match('.*/download/.*', min_count=1), # TODO: in case of .zip files we must not use checksum backends # since those are regenerated irreproducibly each time # so we should use URL backend with those annex, ], [ a_href_match(files_url, min_count=2), assign({'path': '_files/%(url_text)s'}, interpolate=True), sub({'path': {' / ': '/'}}), splitpath, crawl_url(), a_href_match('.*/download/.*', max_count=1), # so we could use it in our magical function # because get_disposition will override it assign({'target_filename': '%(filename)s'}, interpolate=True), get_disposition_filename, fix_the_filename, annex, ], ], ], annex.remove_obsolete(), [ annex.switch_branch('incoming-processed'), annex.merge_branch('incoming', one_commit_at_a_time=True, strategy='theirs', commit=False), [ {'loop': True}, find_files("\.(zip|tgz|tar(\..+)?)$", fail_if_none=True), annex.add_archive_content( existing='archive-suffix', strip_leading_dirs=True, leading_dirs_depth=1, delete=True, exclude=['(^|%s)\._' % os.path.sep], ), ], balsa.verify_files, annex.switch_branch('master'), annex.merge_branch('incoming-processed', commit=True, allow_unrelated=True), annex.finalize(tag=True), ], annex.switch_branch('master'), annex.finalize(cleanup=True), ] datalad-0.4.1/datalad/crawler/pipelines/crcns.py000066400000000000000000000171131301115270200216040ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """A pipeline for crawling a crcns dataset""" # Import necessary nodes import os from ..nodes.crawl_url import crawl_url from ..nodes.crawl_url import parse_checksums from ..nodes.matches import css_match, a_href_match from ..nodes.misc import assign from ..nodes.misc import find_files from ..nodes.misc import sub from ..nodes.misc import skip_if from ..nodes.annex import Annexificator from ...consts import DATALAD_SPECIAL_REMOTE, ARCHIVES_SPECIAL_REMOTE from ...support.strings import get_replacement_dict # Possibly instantiate a logger if you would like to log # during pipeline creation from logging import getLogger lgr = getLogger("datalad.crawler.pipelines.crcns") def superdataset_pipeline(): lgr.info("Creating a CRCNS collection pipeline") # Should return a list representing a pipeline annex = Annexificator(no_annex=True) return [ crawl_url("http://crcns.org/data-sets", matchers=[a_href_match('.*/data-sets/[^#/]+$')]), # a_href_match('.*/data-sets/[\S+/\S+'),]), # TODO: such matchers don't have state so if they get to the same url from multiple # pages they pass that content twice. Implement state to remember yielded results + # .reset() for nodes with state so we could first get through the pipe elements and reset # them all a_href_match("(?P.*/data-sets/(?P[^/#]+)/(?P[^_/#]+))$"), # http://crcns.org/data-sets/vc/pvc-1 assign({'dataset_name': '%(dataset)s'}, interpolate=True), annex.initiate_dataset( template="crcns", data_fields=['dataset_category', 'dataset'], # branch='incoming', # there will be archives etc existing='skip', # further any additional options ) ] def extract_readme(data): # TODO - extract data from the page/response but not into README I guess since majority of datasets # already provide README if os.path.exists("README.txt"): os.unlink("README.txt") with open("README.txt", "w") as f: f.write("CRCNS dataset from %(url)s" % data) lgr.info("Generated README.txt") yield {'filename': "README.txt"} # we might need to explicitly specify for some datasets to use_current_dir since # archives are already carrying the leading directory anyways... # actually probably wouldn't scale since within the same dataset we might need # some tarballs extracted one way, some other... so we might better rely on # stripping dirs in general BUT need to provide some sophistication, e.g. strip # iff directory matches archive name (without archive suffix) # hc-3 dataset is a good example of a mix etc... or may be just a parameter for # how many to strip (currently 2), but then we need to take care bout converting # if provided as a str from config def pipeline(dataset, dataset_category, versioned_urls=False, tarballs=True, data_origin='checksums', use_current_dir=False, leading_dirs_depth=2, rename=None): """Pipeline to crawl/annex an crcns dataset""" if not isinstance(leading_dirs_depth, int): leading_dirs_depth = int(leading_dirs_depth) dataset_url = 'http://crcns.org/data-sets/{dataset_category}/{dataset}'.format(**locals()) lgr.info("Creating a pipeline for the crcns dataset %s" % dataset) annex = Annexificator( create=False, # must be already initialized etc backend="MD5E", statusdb='json', special_remotes=[DATALAD_SPECIAL_REMOTE, ARCHIVES_SPECIAL_REMOTE], # many datasets are actually quite small, so we can simply git them up # below one didn't work out as it should have -- caused major headache either due to bug here or in annex # and comitting to git large .mat and .h5 files # options=["-c", "annex.largefiles=exclude=*.txt and exclude=README and (largerthan=100kb or include=*.gz or include=*.zip)"] # # CRCNS requires authorization, so only README* should go straight under git options=["-c", "annex.largefiles=exclude=README*"] ) crawler = crawl_url(dataset_url) if data_origin == 'checksums': urls_pipe = [ # Download from NERSC # don't even bother finding the link (some times only in about, some times also on the main page # just use https://portal.nersc.gov/project/crcns/download/ # actually to not mess with crawling a custom index let's just go by checksums.md5 crawl_url("https://portal.nersc.gov/project/crcns/download/{dataset}/checksums.md5".format(**locals())), parse_checksums(digest='md5'), # they all contain filelist and checksums.md5 which we can make use of without explicit crawling # no longer valid # TODO: do not download checksums.md (annex would do it) and filelist.txt (includes download # instructions which might confuse, not help) skip_if({'url': '(checksums.md5|filelist.txt)$'}, re=True), ] elif data_origin == 'urls': urls_pipe = [ # Download all the archives found on the project page crawler, a_href_match('.*/.*\.(tgz|tar.*|zip)', min_count=1), ] else: raise ValueError(data_origin) if rename: urls_pipe += [sub({'filename': get_replacement_dict(rename)})] return [ annex.switch_branch('incoming'), [ # nested pipeline so we could quit it earlier happen we decided that nothing todo in it # but then we would still return to 'master' branch # [ # README # crawler, # # Somewhat sucks here since 'url' from above would be passed all the way to annex # # So such nodes as extract_readme should cleans the data so only relevant pieces are left # a_href_match(".*/data.*sets/.*about.*"), # crawler.recurse, # extract_readme, # annex, # ], urls_pipe + [ annex, ], ], annex.switch_branch('incoming-processed'), [ # nested pipeline so we could skip it entirely if nothing new to be merged annex.merge_branch('incoming', strategy='theirs', commit=False), #, skip_no_changes=False), [ # Pipeline to augment content of the incoming and commit it to master find_files("\.(zip|tgz|tar(\..+)?)$", fail_if_none=tarballs), # So we fail if none found -- there must be some! ;)), annex.add_archive_content( existing='archive-suffix', # Since inconsistent and seems in many cases no leading dirs to strip, keep them as provided strip_leading_dirs=True, delete=True, leading_dirs_consider=['crcns.*', dataset], leading_dirs_depth=leading_dirs_depth, use_current_dir=use_current_dir, rename=rename, exclude='.*__MACOSX.*', # some junk penetrates ), ], ], annex.switch_branch('master'), annex.merge_branch('incoming-processed', allow_unrelated=True), annex.finalize(cleanup=True), ] datalad-0.4.1/datalad/crawler/pipelines/fcptable.py000066400000000000000000000157751301115270200222700ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """A pipeline for crawling FCP Classic Data Table""" import os, re from os.path import lexists # Import necessary nodes from ..nodes.crawl_url import crawl_url from ..nodes.matches import xpath_match, a_href_match from ..nodes.misc import assign, skip_if, sub from ..nodes.misc import assign, continue_if from ..nodes.misc import switch from ..nodes.misc import get_disposition_filename from ..nodes.misc import find_files from ..nodes.misc import debug from ..nodes.annex import Annexificator from ...consts import ARCHIVES_SPECIAL_REMOTE from ...consts import DATALAD_SPECIAL_REMOTE from logging import getLogger lgr = getLogger("datalad.crawler.pipelines.fcptable") TOPURL = "http://fcon_1000.projects.nitrc.org/fcpClassic/FcpTable.html" def superdataset_pipeline(url=TOPURL): """ Parameters ---------- url: str URL point to all datasets, hence the URL at the top """ annex = Annexificator() lgr.info("Creating a FCP collection pipeline") return [ crawl_url(url), xpath_match('//*[@class="tableHdr"]/td/strong/text()', output='dataset'), # skipping NewYork due to URL redirects, Durham due to lack of dataset tarball skip_if({'dataset': 'Durham_Madden'}, re=True), assign({'dataset_name': '%(dataset)s'}, interpolate=True), annex.initiate_dataset( template="fcptable", data_fields=['dataset'], existing='skip' ) ] def extract_readme(data): if lexists("README.txt"): os.unlink("README.txt") with open("README.txt", "w") as fi: fi.write("""\ FCP1000 sub-dataset %(dataset)s ------------------------ Author(s): %(PI)s Details: %(details)s """ % data) lgr.info("Generated README.txt") yield {'filename': "README.txt"} class find_dataset(object): def __init__(self, dataset): """ Pipeline to crawl/annex a FCP Table dataset Parameters ---------- dataset: str Id of the FCP Table dataset (e.g. Baltimore) """ self.dataset = dataset def __call__(self, data): titles = [x['match'] for x in xpath_match('//*/tr [@class="tableHdr"]/td/strong/text()')(data)] tar_tds = [x['match'] for x in xpath_match('//*/tr [@class="tableDownload"]/td')(data)] details = [x['match'] for x in xpath_match('//*[@class="tableHdr"]/td|strong/text()')(data)] if len(titles) != len(tar_tds): assert tar_tds[-1] == '' assert len(tar_tds) == len(titles) + 1 for title, tar, detail in zip(titles, tar_tds, details): if title == self.dataset: data['title'] = title data['response'] = tar pi = (re.search(('
.*
'), detail)).group() cleaner = re.compile('<.*?>') data['PI'] = re.sub(cleaner, '', pi) data['details'] = (re.search(('n =.*[0-9]'), detail)).group() yield data return raise RuntimeError("Failed to find a cell for the dataset %s" % self.dataset) def pipeline(dataset): lgr.info("Creating a pipeline for the FCP Table dataset %s" % dataset) annex = Annexificator(create=False, statusdb='json', # only some require authentication, so we need to add DATALAD_SPECIAL_REMOTE there special_remotes= [ARCHIVES_SPECIAL_REMOTE] \ + ([DATALAD_SPECIAL_REMOTE] if dataset in {'Cleveland CCF'} else []), options=["-c", "annex.largefiles=" "exclude=Makefile and exclude=LICENSE* and exclude=ISSUES*" " and exclude=CHANGES* and exclude=README*" " and exclude=*.[mc] and exclude=dataset*.json" " and exclude=*.txt" " and exclude=*.json" " and exclude=*.tsv" ]) # FCP Table has no versioning, so no changelog either return [ annex.switch_branch('incoming'), [ crawl_url(TOPURL), [ assign({'dataset': dataset}), # skipping Cleveland and NewYork due to URL redirects, Durham due to lack of dataset tarball skip_if({'dataset': 'Durham_Madden'}, re=True), # first row was formatted differently so we need to condition it a bit sub({'response': {'
([^<]*)
': r'\1'}}), find_dataset(dataset), [ # README extract_readme, annex, ], a_href_match('.*'), switch( 'url', { # e.g. for Cleveland CCF -- we need to crawl one of thos pages '(http://fcon_1000.projects.nitrc.org/indi/' # or it might lead to the nitrc project page '|http://www.nitrc.org/projects/' '|http://www.nitrc.org/frs/downloadlink.php/1635$)': [ {'output': 'outputs'}, crawl_url(matchers=[a_href_match('http://www.nitrc.org/frs/\?group_id=[0-9]+$')]), a_href_match('(http://www.nitrc.org)?/frs/download(link)?.php/'), # , min_count=1), ], }, re=True, missing='skip', ), # and then collect all URLs pointing to tarballs annex, ], ], annex.remove_obsolete(), [ annex.switch_branch('incoming-processed'), annex.merge_branch('incoming', one_commit_at_a_time=True, strategy='theirs', commit=False), [ {'loop': True}, find_files("\.(zip|tgz|tar(\..+)?)$", fail_if_none=True), annex.add_archive_content( existing='archive-suffix', strip_leading_dirs=True, leading_dirs_depth=1, delete=True, exclude=['(^|%s)\._' % os.path.sep], ), ], annex.switch_branch('master'), annex.merge_branch('incoming-processed', commit=True, allow_unrelated=True), annex.finalize(tag=True), ], annex.switch_branch('master'), annex.finalize(cleanup=True), ] datalad-0.4.1/datalad/crawler/pipelines/nda.py000066400000000000000000000137441301115270200212440ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """A pipeline for crawling NIMH data archive""" # Import necessary nodes from ..nodes.misc import assign from ..nodes.misc import switch from ..nodes.misc import continue_if from ..nodes.matches import a_href_match from ..nodes.s3 import crawl_s3 from ..nodes.annex import Annexificator from ...consts import DATALAD_SPECIAL_REMOTE from datalad.support.nda_ import get_oracle_db from datalad.support.nda_ import image03_fields, image03_file_fields from datalad.support.nda_ import image03_Record from datalad.utils import auto_repr, updated # Possibly instantiate a logger if you would like to log # during pipeline creation from logging import getLogger lgr = getLogger("datalad.crawler.pipelines.nda") DEFAULT_BUCKET = 'NDAR_Central' def collection_pipeline(bucket=DEFAULT_BUCKET, prefix=None): """Pipeline to crawl/annex an entire openfmri bucket""" lgr.info("Creating a pipeline for the openfmri bucket") annex = Annexificator( create=False, # must be already initialized etc ) sprefix = prefix + '/' if prefix else '' return [ crawl_s3(bucket, prefix=prefix, recursive=False, strategy='commit-versions', repo=annex.repo, versioned=False), switch('datalad_action', { # TODO: we should actually deal with subdirs primarily 'commit': annex.finalize(tag=True), # should we bother removing anything? not sure # 'remove': annex.remove, 'annex': annex, 'directory': [ # for initiate_dataset we should replicate filename as handle_name, prefix assign({ 'prefix': sprefix + '%(filename)s/', 'bucket': bucket, 'handle_name': '%(filename)s' }, interpolate=True), annex.initiate_dataset( template='nda', data_fields=['bucket', 'prefix'], ) ] }, missing='skip', # ok to not remove ), ] def bucket_pipeline(bucket=DEFAULT_BUCKET, prefix=None): """Pipeline to crawl/annex NDA bucket""" lgr.info("Creating a pipeline for the NDA bucket") annex = Annexificator( create=False, # must be already initialized etc special_remotes=[DATALAD_SPECIAL_REMOTE], backend='MD5E' # Primary purpose of this one is registration of all URLs with our # upcoming "ultimate DB" so we don't get to git anything # options=["-c", "annex.largefiles=exclude=CHANGES* and exclude=changelog.txt and exclude=dataset_description.json and exclude=README* and exclude=*.[mc]"] ) return [ crawl_s3(bucket, prefix=prefix, strategy='commit-versions', repo=annex.repo, versioned=False), switch('datalad_action', { 'commit': annex.finalize(tag=True), 'remove': annex.remove, 'annex': annex, 'directory': None, }) ] @auto_repr class crawl_mindar_images03(object): """Crawl miNDAR DB for a given collection TODO: generalize for other data structures other than image03, with their own sets of "File" fields Parameters ---------- collection """ def __init__(self, collection): self.collection = collection def __call__(self, data): db = get_oracle_db() query = "SELECT %s FROM IMAGE03 WHERE COLLECTION_ID=%s" \ % (','.join(image03_fields), self.collection) c = db.cursor() c.execute(query) # query and wrap into named tuples to ease access #import ipdb; ipdb.set_trace() for rec in c.fetchall(): # TODO -- better access method? rec = image03_Record(*rec) for field in image03_file_fields: url = getattr(rec, field) if url: # generate a new yield updated(data, { 'url': url, }) c.close() def pipeline(collection, archives=None): """Pipeline to crawl/annex NDA Parameters ---------- archives: Idea is to be able to control how archives treated -- extracted within the same repository, or extracted into a submodule. TODO """ assert archives is None, "nothing else is implemented" lgr.info("Creating a pipeline for the NDA bucket") annex = Annexificator( create=False, # must be already initialized etc special_remotes=[DATALAD_SPECIAL_REMOTE], backend='MD5E', skip_problematic=True, # TODO: make it cleaner for detection of s3 "directories" # Primary purpose of this one is registration of all URLs with our # upcoming "ultimate DB" so we don't get to git anything # options=["-c", "annex.largefiles=exclude=CHANGES* and exclude=changelog.txt and exclude=dataset_description.json and exclude=README* and exclude=*.[mc]"] ) return [ [ assign( {'url': 'https://ndar.nih.gov/edit_collection.html?id=%s' % collection, 'filename': 'collection.html'} ), annex, ], [ crawl_mindar_images03(collection), continue_if({'url': "s3://(?P[^/]*)/submission_(?P[0-9]*)/(?P.*[^/])$"}, re=True), annex, # TODO: add annex tags may be for dataset_id, submission_id, ], ]datalad-0.4.1/datalad/crawler/pipelines/openfmri.py000066400000000000000000000325401301115270200223140ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """A pipeline for crawling openfmri dataset""" import os import re from os.path import lexists # Import necessary nodes from ..nodes.crawl_url import crawl_url from ..nodes.matches import css_match, a_href_match from ..nodes.misc import assign from ..nodes.misc import sub from ..nodes.misc import switch from ..nodes.misc import func_to_node from ..nodes.misc import find_files from ..nodes.misc import skip_if from ..nodes.misc import debug from ..nodes.misc import fix_permissions from ..nodes.annex import Annexificator from ...support.s3 import get_versioned_url from ...utils import updated from ...consts import ARCHIVES_SPECIAL_REMOTE, DATALAD_SPECIAL_REMOTE from datalad.downloaders.providers import Providers # For S3 crawling from ..nodes.s3 import crawl_s3 from .openfmri_s3 import pipeline as s3_pipeline from datalad.api import ls from datalad.dochelpers import exc_str # Possibly instantiate a logger if you would like to log # during pipeline creation from logging import getLogger lgr = getLogger("datalad.crawler.pipelines.openfmri") TOPURL = "https://openfmri.org/dataset/" # define a pipeline factory function accepting necessary keyword arguments # Should have no strictly positional arguments def superdataset_pipeline(url=TOPURL, **kwargs): annex = Annexificator(no_annex=True, allow_dirty=True) lgr.info("Creating a pipeline with kwargs %s" % str(kwargs)) return [ crawl_url(url), a_href_match("(?P.*/dataset/(?Pds0*(?P[0-9a-z]*)))/*$"), # https://openfmri.org/dataset/ds000001/ assign({'dataset_name': '%(dataset)s'}, interpolate=True), skip_if({'dataset_name': 'ds000017'}), # was split into A/B # TODO: crawl into the dataset url, and check if there is any tarball available # if not -- do not bother (e.g. ds000053) annex.initiate_dataset( template="openfmri", data_fields=['dataset'], # let's all specs and modifications reside in master # branch='incoming', # there will be archives etc existing='skip' # further any additional options ) ] def extract_readme(data): # TODO - extract data from the page/response if lexists("README.txt"): os.unlink("README.txt") with open("README.txt", "w") as f: f.write("OpenfMRI dataset from %(url)s" % data) lgr.info("Generated README.txt") yield {'filename': "README.txt", # TODO: think how we should sweat about this one # 'datalad_stats': data['datalad_stats'] } def pipeline(dataset, versioned_urls=True, topurl=TOPURL, versions_overlay_level=2, leading_dirs_depth=1, prefix='', s3_prefix=None): """Pipeline to crawl/annex an openfmri dataset Parameters ---------- dataset: str Id of the OpenfMRI dataset (e.g. ds000001) versioned_urls: bool, optional Request versioned URLs. OpenfMRI bucket is versioned, but if original data resides elsewhere, set to False topurl: str, optional Top level URL to the datasets. prefix: str, optional Prefix regular expression in urls to identifying subgroup of data to be fetched in the dataset (e.g. in case of ds000017 there is A and B) s3_prefix: str or None, optional Either to crawl per-dataset subdirectory in the bucket into incoming-s3 branch, to also annex also all the extracted files available from openfmri bucket. If None -- we determine depending on availability of the sub-directory on S3 bucket """ skip_no_changes = True # to redo incoming-processed, would finish dirty in incoming-processed # when commit would fail since nothing to commit leading_dirs_depth = int(leading_dirs_depth) versions_overlay_level = int(versions_overlay_level) dataset_url = '%s%s/' % (topurl, dataset) lgr.info("Creating a pipeline for the openfmri dataset %s" % dataset) special_remotes = [ARCHIVES_SPECIAL_REMOTE] if s3_prefix is None: # some datasets available (fresh enough or old) from S3, so let's sense if this one is s3_prefix = re.sub('^ds0*([0-9]{3})/*', r'ds\1/', dataset) if dataset == 'ds000017': # we had some custom prefixing going on assert(prefix) suf = prefix[-3] assert suf in 'AB' s3_prefix = 'ds017' + suf openfmri_s3_prefix = 's3://openfmri/' try: if not ls('%s%s' % (openfmri_s3_prefix, s3_prefix)): s3_prefix = None # not there except Exception as exc: lgr.warning( "Failed to access %s, not attempting to crawl S3: %s", s3_prefix, exc_str(exc) ) s3_prefix = None if s3_prefix: # actually not needed here since we are remapping them to public http # urls # special_remotes += [DATALAD_SPECIAL_REMOTE] pass annex = Annexificator( create=False, # must be already initialized etc # leave in Git only obvious descriptors and code snippets -- the rest goes to annex # so may be eventually we could take advantage of git tags for changing layout statusdb='json', special_remotes=special_remotes, # all .txt and .json in root directory (only) go into git! options=["-c", "annex.largefiles=" # ISSUES LICENSE Makefile "exclude=Makefile and exclude=LICENSE* and exclude=ISSUES*" " and exclude=CHANGES* and exclude=README* and exclude=ReadMe.txt" " and exclude=*.[mc] and exclude=dataset*.json and exclude=license.txt" " and (exclude=*.txt or include=*/*.txt)" " and (exclude=*.json or include=*/*.json)" " and (exclude=*.tsv or include=*/*.tsv)" ]) if s3_prefix: # a sub-pipeline to crawl s3 bucket s3_pipeline_here = \ [ [ annex.switch_branch('incoming-s3'), s3_pipeline(s3_prefix, tag=False), # for 31 ;) skip_problematic=True), annex.switch_branch('master'), ] ] else: s3_pipeline_here = [] # common kwargs which would later would be tuned up def add_archive_content(**kw): if 'leading_dirs_depth' not in kw: kw['leading_dirs_depth'] = leading_dirs_depth if 'strip_leading_dirs' not in kw: kw['strip_leading_dirs'] = bool(leading_dirs_depth) return annex.add_archive_content( existing='archive-suffix', delete=True, exclude=['(^|%s)\._' % os.path.sep], # some files like '._whatever' **kw # overwrite=True, # TODO: we might need a safeguard for cases when multiple subdirectories within a single tarball ) return s3_pipeline_here + [ # optionally "log" to annex extracted content available on openfmri S3 annex.switch_branch('incoming'), [ # nested pipeline so we could quit it earlier happen we decided that nothing todo in it # but then we would still return to 'master' branch crawl_url(dataset_url), [ # changelog XXX there might be multiple, e.g. in case of ds000017 a_href_match(".*%srelease_history.txt" % prefix), # , limit=1 assign({'filename': 'changelog.txt'}), annex, ], # Moving to proper meta-data descriptors, so no need to generate and possibly conflict # with distributed one README # [ # README # # Somewhat sucks here since 'url' from above would be passed all the way to annex # # So such nodes as extract_readme should cleans the data so only relevant pieces are left # extract_readme, # annex, # ], [ # and collect all URLs pointing to tarballs a_href_match('.*/%s.*\.(tgz|tar.*|zip)' % prefix, min_count=1), # Since all content of openfmri is anyways available openly, no need atm # to use https which complicates proxying etc. Thus replace for AWS urls # to openfmri S3 from https to http # TODO: might want to become an option for get_versioned_url? sub({ 'url': { '(http)s?(://.*openfmri\.s3\.amazonaws.com/|://s3\.amazonaws\.com/openfmri/)': r'\1\2'}}), func_to_node(get_versioned_url, data_args=['url'], outputs=['url'], kwargs={'guarantee_versioned': versioned_urls, 'verify': True}), annex, ], # TODO: describe_dataset # Now some true magic -- possibly multiple commits, 1 per each detected **new** version! # this one doesn't go through all files, but only through the freshly staged! annex.commit_versions( '_R(?P\d+[\.\d]*)(?=[\._])', always_versioned='ds\d\d+.*', unversioned='default', default='1.0.0'), ], annex.remove_obsolete(), # should be called while still within incoming but only once # TODO: since it is a very common pattern -- consider absorbing into e.g. add_archive_content? [ # nested pipeline so we could skip it entirely if nothing new to be merged {'loop': not skip_no_changes}, # loop for multiple versions merges annex.switch_branch('incoming-processed'), annex.merge_branch('incoming', one_commit_at_a_time=True, strategy='theirs', commit=False, skip_no_changes=skip_no_changes ), # still we would have all the versions present -- we need to restrict only to the current one! # TODO: we often need ability to augment next node options by checks etc in the previous ones # e.g. ehere overlay option depending on which dataset/version being processed annex.remove_other_versions('incoming', remove_unversioned=True, # ds001.tar.gz could then become ds0000001.zip fpath_subs=[ # ad-hoc fixups for some datasets ('ds005\.tgz', 'ds005_raw.tgz'), # had it split into this one with derived data separately and then joined ('ds007_01-20\.tgz', 'ds007_raw.tgz'), # generic ('^ds0*', '^ds'), ('\.(zip|tgz|tar\.gz)$', '.ext') ], # Had manually to do this for this one since there was a switch from # overlay layout to even bigger single one within a minor 2.0.1 "release" overlay=None if dataset in ('ds000007', 'ds000114', 'ds000119') else versions_overlay_level, # use major.minor to define overlays #overlay=None, # use major.minor to define overlays exclude='(README|changelog).*'), [ # Pipeline to augment content of the incoming and commit it to master # There might be archives within archives, so we need to loop {'loop': True}, find_files("\.(zip|tgz|tar(\..+)?)$", fail_if_none=True), # we fail if none found -- there must be some! ;)), assign({'dataset_file': dataset + '///%(filename)s'}, interpolate=True), switch( 'dataset_file', { 'ds0*158///aalmasks\.zip$': add_archive_content(add_archive_leading_dir=True), '.*///ds000030_R1\.0\.1_metadata\.tgz': add_archive_content(leading_dirs_depth=4), }, default=add_archive_content(), re=True, ), ], [ find_files("(\.(tsv|csv|txt|json|gz|bval|bvec|hdr|img|m|mat|pdf|png|zip|nii|jpg|fif|fig)|README|CHANGES)$"), fix_permissions(executable=False) ], annex.switch_branch('master'), annex.merge_branch('incoming-processed', commit=True, allow_unrelated=True), annex.finalize(tag=True, aggregate=True), ], annex.switch_branch('master'), annex.finalize(cleanup=True, aggregate=True), ] datalad-0.4.1/datalad/crawler/pipelines/openfmri_s3.py000066400000000000000000000077311301115270200227250ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """A pipeline for crawling openfmri s3 bucket""" import os from os.path import lexists # Import necessary nodes from ..nodes.misc import switch, assign, sub from ..nodes.s3 import crawl_s3 from ..nodes.annex import Annexificator from ...consts import DATALAD_SPECIAL_REMOTE # Possibly instantiate a logger if you would like to log # during pipeline creation from logging import getLogger lgr = getLogger("datalad.crawler.pipelines.openfmri") # Since all content of openfmri is anyways available openly, no need atm # to use https which complicates proxying etc. Thus provide a node which # would replace s3:// urls with regular http # TODO: we might want to make it an option for crawl_s3 to yield http urls # so then we could just generalize this whole shebang into a single helper # for crawling any S3 bucket. # Right away think about having an 'incoming' branch and handling of versioned files sub_s3_to_http = sub({ 'url': {'^s3://openfmri/': 'http://openfmri.s3.amazonaws.com/'} }, ok_missing=True ) def collection_pipeline(prefix=None): """Pipeline to crawl/annex an entire openfmri bucket""" lgr.info("Creating a pipeline for the openfmri bucket") annex = Annexificator( create=False, # must be already initialized etc # Primary purpose of this one is registration of all URLs with our # upcoming "ultimate DB" so we don't get to git anything # options=["-c", "annex.largefiles=exclude=CHANGES* and exclude=changelog.txt and exclude=dataset_description.json and exclude=README* and exclude=*.[mc]"] ) return [ crawl_s3('openfmri', prefix=prefix, recursive=False, strategy='commit-versions', repo=annex.repo), sub_s3_to_http, switch('datalad_action', { # TODO: we should actually deal with subdirs primarily 'commit': annex.finalize(tag=True), # should we bother removing anything? not sure # 'remove': annex.remove, 'annex': annex, 'directory': [ # for initiate_dataset we should replicate filename as handle_name, prefix assign({'prefix': '%(filename)s/', 'handle_name': '%(filename)s'}, interpolate=True), annex.initiate_dataset( template='openfmri_s3', data_fields=['prefix'], ) ] }, missing='skip', # ok to not remove ) ] # TODO: make a unittest for all of this on a simple bucket def pipeline(prefix=None, tag=True, skip_problematic=False): """Pipeline to crawl/annex an entire openfmri bucket""" lgr.info("Creating a pipeline for the openfmri bucket") annex = Annexificator( create=False, # must be already initialized etc special_remotes=[DATALAD_SPECIAL_REMOTE], backend='MD5E', skip_problematic=skip_problematic, # Primary purpose of this one is registration of all URLs with our # upcoming "ultimate DB" so we don't get to git anything # options=["-c", "annex.largefiles=exclude=CHANGES* and exclude=changelog.txt and exclude=dataset_description.json and exclude=README* and exclude=*.[mc]"] ) return [ crawl_s3('openfmri', prefix=prefix, strategy='commit-versions', repo=annex.repo, recursive=True), sub_s3_to_http, switch('datalad_action', { 'commit': annex.finalize(tag=tag), 'remove': annex.remove, 'annex': annex, }) ] datalad-0.4.1/datalad/crawler/pipelines/simple_with_archives.py000066400000000000000000000123761301115270200247120ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """A pipeline for crawling a crcns dataset""" # Import necessary nodes import os from ..nodes.crawl_url import crawl_url from ..nodes.crawl_url import parse_checksums from ..nodes.matches import css_match, a_href_match from ..nodes.misc import assign from ..nodes.misc import find_files from ..nodes.misc import sub from ..nodes.misc import skip_if from ..nodes.annex import Annexificator from ...consts import DATALAD_SPECIAL_REMOTE, ARCHIVES_SPECIAL_REMOTE from ...support.strings import get_replacement_dict # Possibly instantiate a logger if you would like to log # during pipeline creation from logging import getLogger lgr = getLogger("datalad.crawler.pipelines.kaggle") # # def superdataset_pipeline(): # lgr.info("Creating a CRCNS collection pipeline") # # Should return a list representing a pipeline # annex = Annexificator(no_annex=True) # return [ # crawl_url("http://crcns.org/data-sets", # matchers=[a_href_match('.*/data-sets/[^#/]+$')]), # # a_href_match('.*/data-sets/[\S+/\S+'),]), # # TODO: such matchers don't have state so if they get to the same url from multiple # # pages they pass that content twice. Implement state to remember yielded results + # # .reset() for nodes with state so we could first get through the pipe elements and reset # # them all # a_href_match("(?P.*/data-sets/(?P[^/#]+)/(?P[^_/#]+))$"), # # http://crcns.org/data-sets/vc/pvc-1 # assign({'dataset_name': '%(dataset)s'}, interpolate=True), # annex.initiate_dataset( # template="crcns", # data_fields=['dataset_category', 'dataset'], # # branch='incoming', # there will be archives etc # existing='skip', # # further any additional options # ) # ] # # # def extract_readme(data): # # TODO - extract data from the page/response but not into README I guess since majority of datasets # # already provide README # if os.path.exists("README.txt"): # os.unlink("README.txt") # with open("README.txt", "w") as f: # f.write("CRCNS dataset from %(url)s" % data) # lgr.info("Generated README.txt") # yield {'filename': "README.txt"} # def pipeline(url, a_href_match_='.*/download/.*\.(tgz|tar.*|zip)', tarballs=True, datalad_downloader=False, use_current_dir=False, leading_dirs_depth=1, rename=None, backend='MD5E'): """Pipeline to crawl/annex an crcns dataset""" if not isinstance(leading_dirs_depth, int): leading_dirs_depth = int(leading_dirs_depth) if not tarballs: raise NotImplementedError("yet to simplify for no tarballs case") lgr.info("Creating a pipeline to crawl data files from %s", url) special_remotes = [] if tarballs: special_remotes.append(ARCHIVES_SPECIAL_REMOTE) if datalad_downloader: special_remotes.append(DATALAD_SPECIAL_REMOTE) annex = Annexificator( create=False, # must be already initialized etc backend=backend, statusdb='json', special_remotes=special_remotes, options=["-c", "annex.largefiles=exclude=README* and exclude=LICENSE* and exclude=*.txt and exclude=*.json" " and exclude=*.cfg"] ) crawler = crawl_url(url) urls_pipe = [ # Download all the archives found on the project page crawler, a_href_match(a_href_match_, min_count=1), ] if rename: urls_pipe += [sub({'filename': get_replacement_dict(rename)})] return [ annex.switch_branch('incoming'), [ urls_pipe + [ annex, ], ], annex.switch_branch('incoming-processed'), [ # nested pipeline so we could skip it entirely if nothing new to be merged annex.merge_branch('incoming', strategy='theirs', commit=False), #, skip_no_changes=False), [ # Pipeline to augment content of the incoming and commit it to master find_files("\.(zip|tgz|tar(\..+)?)$", fail_if_none=tarballs), # So we fail if none found -- there must be some! ;)), annex.add_archive_content( existing='archive-suffix', # Since inconsistent and seems in many cases no leading dirs to strip, keep them as provided strip_leading_dirs=bool(leading_dirs_depth), delete=True, leading_dirs_depth=leading_dirs_depth, use_current_dir=use_current_dir, rename=rename, exclude='.*__MACOSX.*', # some junk penetrates ), ], ], annex.switch_branch('master'), annex.merge_branch('incoming-processed', allow_unrelated=True), annex.finalize(cleanup=True), ] datalad-0.4.1/datalad/crawler/pipelines/template.py000066400000000000000000000020761301115270200223110ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """A template (non-functional) for crawler pipelines""" # Import necessary nodes from ..nodes.crawl_url import crawl_url # Possibly instantiate a logger if you would like to log # during pipeline creation from logging import getLogger lgr = getLogger("datalad.crawler.pipelines.template") # define a pipeline generator as a function accepting necessary keyword arguments # Should have no strictly positional arguments def pipeline( # e.g. could be listed as explicit keyword arguments #url="http://example.com", depth=2, ... **kwargs ): lgr.info("Creating a pipeline with kwargs %s" % str(kwargs)) # Should return a list representing a pipeline return []datalad-0.4.1/datalad/crawler/pipelines/tests/000077500000000000000000000000001301115270200212615ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/pipelines/tests/__init__.py000066400000000000000000000002611301115270200233710ustar00rootroot00000000000000from datalad.tests.utils import skip_if_no_module skip_if_no_module('scrapy') from datalad.tests.utils import skip_if_scrapy_without_selector skip_if_scrapy_without_selector() datalad-0.4.1/datalad/crawler/pipelines/tests/test_balsa.py000066400000000000000000000276301301115270200237640ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from datalad.crawler.pipelines.tests.utils import _test_smoke_pipelines from ..balsa import pipeline as ofpipeline, superdataset_pipeline import os import logging from os.path import exists from glob import glob from os.path import join as opj from mock import patch from ...nodes.crawl_url import crawl_url from ...nodes.matches import * from ...pipeline import run_pipeline, FinishPipeline from ...nodes.misc import Sink, assign, range_node, interrupt_if from ...nodes.annex import Annexificator, initiate_dataset from ...pipeline import load_pipeline_from_module from ....support.stats import ActivityStats from ....support.gitrepo import GitRepo from ....support.annexrepo import AnnexRepo from ....utils import chpwd from ....utils import find_files from ....utils import swallow_logs from ....tests.utils import with_tree from ....tests.utils import SkipTest from ....tests.utils import eq_, assert_not_equal, ok_, assert_raises from ....tests.utils import assert_in, assert_not_in, assert_true from ....tests.utils import skip_if_no_module from ....tests.utils import with_tempfile from ....tests.utils import serve_path_via_http from ....tests.utils import skip_if_no_network from ....tests.utils import use_cassette from ....tests.utils import ok_file_has_content from ....tests.utils import ok_file_under_git from logging import getLogger lgr = getLogger('datalad.crawl.tests') def test_smoke_pipelines(): yield _test_smoke_pipelines, superdataset_pipeline, [] TEST_TREE1 = { 'study': { 'show': { 'WG33': { 'index.html': """

SPECIES:
Human

DESCRIPTION:
DC Van Essen, J Smith, MF Glasser (2016) PMID: 27074495

PUBLICATION:
NeuroImage

AUTHORS:
  • DC Van Essen
  • J Smith
  • MF Glasser
""" }, }, 'download': { 'WG33.tar.gz': { 'file1.nii': "content of file1.nii", 'dir1': { 'file2.nii': "content of file2.nii", } } } }, 'file': { 'show': { 'JX5V': { 'index.html': """ Download (120 MB) """ }, 'R1BX': { 'index.html': """ Download (26 MB) """ } }, 'download': { 'file1.nii': "content of file1.nii", 'dir1': { 'file2.nii': "content of file2.nii", } } }, } @with_tree(tree=TEST_TREE1, archives_leading_dir=False) @serve_path_via_http @with_tempfile @with_tempfile def test_balsa_extract_meta(ind, topurl, outd, clonedir): list(initiate_dataset( template="balsa", dataset_name='dataladtest-WG33', path=outd, data_fields=['dataset_id'])({'dataset_id': 'WG33'})) with chpwd(outd): pipeline = ofpipeline('WG33', url=topurl) out = run_pipeline(pipeline) eq_(len(out), 1) with chpwd(outd): assert_true(exists(".datalad/meta/balsa.json")) f = open(".datalad/meta/balsa.json", 'r') contents = f.read() assert_true("SPECIES" and "DESCRIPTION" and "PUBLICATION" and "AUTHORS" in contents) _PLUG_HERE = '' @with_tree(tree={ 'study': { 'show': { 'WG33': { 'index.html': """ Download (146 MB) file1.nii dir1 / file2.nii %s """ % _PLUG_HERE, }, }, 'download': { 'WG33.tar.gz': { 'file1.nii': "content of file1.nii", 'dir1': { 'file2.nii': "content of file2.nii", } } } }, 'file': { 'show': { 'JX5V': { 'index.html': """ Download (120 MB) """ }, 'R1BX': { 'index.html': """ Download (26 MB) """ } }, 'download': { 'file1.nii': "content of file1.nii", 'dir1': { 'file2.nii': "content of file2.nii", } } }, }, archives_leading_dir=False ) @serve_path_via_http @with_tempfile @with_tempfile def test_balsa_pipeline1(ind, topurl, outd, clonedir): list(initiate_dataset( template="balsa", dataset_name='dataladtest-WG33', path=outd, data_fields=['dataset_id'])({'dataset_id': 'WG33'})) with chpwd(outd): pipeline = ofpipeline('WG33', url=topurl) out = run_pipeline(pipeline) eq_(len(out), 1) repo = AnnexRepo(outd, create=False) # to be used in the checks # Inspect the tree -- that we have all the branches branches = {'master', 'incoming', 'incoming-processed', 'git-annex'} eq_(set(repo.get_branches()), branches) assert_not_equal(repo.get_hexsha('master'), repo.get_hexsha('incoming-processed')) # and that one is different from incoming assert_not_equal(repo.get_hexsha('incoming'), repo.get_hexsha('incoming-processed')) commits = {b: list(repo.get_branch_commits(b)) for b in branches} eq_(len(commits['incoming']), 1) eq_(len(commits['incoming-processed']), 2) eq_(len(commits['master']), 5) # all commits out there -- init ds + init crawler + 1*(incoming, processed, merge) with chpwd(outd): eq_(set(glob('*')), {'dir1', 'file1.nii'}) all_files = sorted(find_files('.')) fpath = opj(outd, 'file1.nii') ok_file_has_content(fpath, "content of file1.nii") ok_file_under_git(fpath, annexed=True) fpath2 = opj(outd, 'dir1', 'file2.nii') ok_file_has_content(fpath2, "content of file2.nii") ok_file_under_git(fpath2, annexed=True) target_files = { './.datalad/crawl/crawl.cfg', './.datalad/crawl/statuses/incoming.json', './.datalad/meta/balsa.json', './.datalad/config', './file1.nii', './dir1/file2.nii', } eq_(set(all_files), target_files) # this test should raise warning that canonical tarball does not have one of the files listed # and that a listed file differs in content _PLUG_HERE = '' @with_tree(tree={ 'study': { 'show': { 'WG33': { 'index.html': """ Download (172 MB) file1.nii dir1 / file2.nii file1b.nii %s """ % _PLUG_HERE, }, }, 'download': { 'WG33.tar.gz': { 'file1.nii': "content of file1.nii", 'dir1': { 'file2.nii': "content of file2.nii", } } } }, 'file': { 'show': { 'JX5V': { 'index.html': """ Download (120 MB) """ }, 'R1BX': { 'index.html': """ Download (26 MB) """ }, 'GSRD': { 'index.html': """ Download (26 MB) """ } }, 'download': { 'file1.nii': "content of file1.nii is different", 'file1b.nii': "content of file1b.nii", 'dir1': { 'file2.nii': "content of file2.nii", } } }, }, archives_leading_dir=False ) @serve_path_via_http @with_tempfile @with_tempfile def test_balsa_pipeline2(ind, topurl, outd, clonedir): list(initiate_dataset( template="balsa", dataset_name='dataladtest-WG33', path=outd, data_fields=['dataset_id'])({'dataset_id': 'WG33'})) with chpwd(outd): with swallow_logs(new_level=logging.WARN) as cml: pipeline = ofpipeline('WG33', url=topurl) out = run_pipeline(pipeline) assert_true('The following files do not exist in the canonical tarball, ' 'but are individually listed files and will not be kept:' in cml.out) assert_true('./file1.nii varies in content from the individually downloaded ' 'file with the same name, it is removed and file from canonical ' 'tarball is kept' in cml.out) eq_(len(out), 1) with chpwd(outd): eq_(set(glob('*')), {'dir1', 'file1.nii'}) all_files = sorted(find_files('.')) fpath = opj(outd, 'file1.nii') ok_file_has_content(fpath, "content of file1.nii") ok_file_under_git(fpath, annexed=True) fpath2 = opj(outd, 'dir1', 'file2.nii') ok_file_has_content(fpath2, "content of file2.nii") ok_file_under_git(fpath2, annexed=True) target_files = { './.datalad/config', './.datalad/crawl/crawl.cfg', './.datalad/crawl/statuses/incoming.json', './.datalad/meta/balsa.json', './file1.nii', './dir1/file2.nii', } eq_(set(all_files), target_files) datalad-0.4.1/datalad/crawler/pipelines/tests/test_crcns.py000066400000000000000000000011771301115270200240100ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from .utils import _test_smoke_pipelines from ..crcns import pipeline, superdataset_pipeline def test_smoke_pipelines(): yield _test_smoke_pipelines, pipeline, ['bogus', "bogusgroup"] yield _test_smoke_pipelines, superdataset_pipeline, [] datalad-0.4.1/datalad/crawler/pipelines/tests/test_fcptable.py000066400000000000000000000047451301115270200244640ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from os.path import exists from ....utils import chpwd from ....tests.utils import assert_true, assert_raises, assert_false from ....tests.utils import with_tempfile, skip_if_no_network, use_cassette from datalad.crawler.pipelines.tests.utils import _test_smoke_pipelines from datalad.crawler.pipelines.fcptable import * from datalad.crawler.pipeline import run_pipeline import logging from logging import getLogger lgr = getLogger('datalad.crawl.tests') from ..fcptable import pipeline, superdataset_pipeline def test_smoke_pipelines(): yield _test_smoke_pipelines, pipeline, ['bogus'] yield _test_smoke_pipelines, superdataset_pipeline, [] @use_cassette('test_fcptable_dataset') @skip_if_no_network @with_tempfile(mkdir=True) def _test_dataset(dataset, error, create, skip, tmpdir): TOPURL = "http://fcon_1000.projects.nitrc.org/fcpClassic/FcpTable.html" with chpwd(tmpdir): if create: with open("README.txt", 'w') as f: f.write(" ") pipe = [ crawl_url(TOPURL), [ assign({'dataset': dataset}), skip_if({'dataset': 'Cleveland CCF|Durham_Madden|NewYork_Test-Retest_Reliability'}, re=True), sub({'response': {'
([^<]*)
': r'\1'}}), find_dataset(dataset), extract_readme, ] ] if error: assert_raises(RuntimeError, run_pipeline, pipe) return run_pipeline(pipe) if skip: assert_false(exists("README.txt")) return assert_true(exists("README.txt")) f = open("README.txt", 'r') contents = f.read() assert_true("Author(s)" and "Details" in contents) def test_dataset(): yield _test_dataset, 'Baltimore', None, False, False yield _test_dataset, 'AnnArbor_b', None, False, False yield _test_dataset, 'Ontario', None, False, False yield _test_dataset, 'Boston', RuntimeError, False, False yield _test_dataset, "AnnArbor_b", None, True, False yield _test_dataset, "Cleveland CCF", None, False, True datalad-0.4.1/datalad/crawler/pipelines/tests/test_nda.py000066400000000000000000000020511301115270200234320ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from ....support.annexrepo import AnnexRepo from ....utils import chpwd from ....utils import swallow_logs from ....tests.utils import eq_, assert_not_equal, ok_, assert_raises from ....tests.utils import with_tempfile from ..nda import pipeline, bucket_pipeline, collection_pipeline from logging import getLogger lgr = getLogger('datalad.crawl.tests') @with_tempfile(mkdir=True) def test_smoke_pipelines(d): # Just to verify that we can correctly establish the pipelines AnnexRepo(d, create=True) with chpwd(d): with swallow_logs(): for p in [pipeline('bogus'), bucket_pipeline(), collection_pipeline()]: ok_(len(p) > 1) datalad-0.4.1/datalad/crawler/pipelines/tests/test_openfmri.py000066400000000000000000000546451301115270200245270ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import os from glob import glob from os.path import join as opj from os.path import exists from mock import patch from ...nodes.crawl_url import crawl_url from ...nodes.matches import * from ...pipeline import run_pipeline, FinishPipeline from ...nodes.misc import Sink, assign, range_node, interrupt_if from ...nodes.annex import Annexificator, initiate_dataset from ...pipeline import load_pipeline_from_module from ....support.stats import ActivityStats from ....support.gitrepo import GitRepo from ....support.annexrepo import AnnexRepo from ....api import clean from ....utils import chpwd from ....utils import find_files from ....utils import swallow_logs from ....tests.utils import with_tree from ....tests.utils import SkipTest from ....tests.utils import eq_, assert_not_equal, ok_, assert_raises from ....tests.utils import assert_in, assert_not_in from ....tests.utils import skip_if_no_module from ....tests.utils import with_tempfile from ....tests.utils import serve_path_via_http from ....tests.utils import skip_if_no_network from ....tests.utils import use_cassette from ....tests.utils import ok_file_has_content from ....tests.utils import ok_file_under_git from .. import openfmri from ..openfmri import pipeline as ofpipeline import logging from logging import getLogger lgr = getLogger('datalad.crawl.tests') # # Some helpers # def check_dropall_get(repo): # drop all annex content for all revisions, clean the cache, get the content for all files in # master in all of its revisions t1w_fpath = opj(repo.path, 'sub-1', 'anat', 'sub-1_T1w.dat') ok_file_has_content(t1w_fpath, "mighty load 2.0.0") # --force since it would fail to verify presence in case we remove archives keys... TODO repo._annex_custom_command([], ["git", "annex", "drop", "--all", "--force"]) clean(annex=repo) # remove possible extracted archives with assert_raises(AssertionError): ok_file_has_content(t1w_fpath, "mighty load 2.0.0") repo.get('.') ok_file_has_content(t1w_fpath, "mighty load 2.0.0") def add_to_index(index_file, content): with open(index_file) as f: old_index = f.read() with open(index_file, 'w') as f: f.write(old_index.replace(_PLUG_HERE, content + _PLUG_HERE)) def remove_from_index(index_file, regexp): with open(index_file) as f: old_index = f.read() with open(index_file, 'w') as f: f.write(re.sub(regexp, '', old_index)) @skip_if_no_network @use_cassette('openfmri') def __test_basic_openfmri_top_pipeline(): skip_if_no_module('scrapy') # e.g. not present under Python3 sink1 = Sink() sink2 = Sink() sink_licenses = Sink() pipeline = [ crawl_url("https://openfmri.org/data-sets"), a_href_match(".*/dataset/(?Pds0*(?P[1-9][0-9]*))$"), # if we wanted we could instruct to crawl inside [ crawl_url(), [# and collect all URLs under "AWS Link" css_match('.field-name-field-aws-link a', xpaths={'url': '@href', 'url_text': 'text()'}), sink2 ], [# and license information css_match('.field-name-field-license a', xpaths={'url': '@href', 'url_text': 'text()'}), sink_licenses ], ], sink1 ] run_pipeline(pipeline) # we should have collected all the URLs to the datasets urls = [e['url'] for e in sink1.data] ok_(len(urls) > 20) # there should be at least 20 listed there ok_(all([url.startswith('https://openfmri.org/dataset/ds00') for url in urls])) # got our dataset_dir entries as well ok_(all([e['dataset_dir'].startswith('ds0') for e in sink1.data])) # and sink2 should collect everything downloadable from under AWS Link section # test that we got all needed tags etc propagated properly! all_aws_entries = sink2.get_values(['dataset', 'url_text', 'url']) ok_(len(all_aws_entries) > len(urls)) # that we have at least as many ;-) #print('\n'.join(map(str, all_aws_entries))) all_licenses = sink_licenses.get_values(['dataset', 'url_text', 'url']) eq_(len(all_licenses), len(urls)) #print('\n'.join(map(str, all_licenses))) @skip_if_no_network @use_cassette('openfmri-1') @with_tempfile(mkdir=True) def __test_basic_openfmri_dataset_pipeline_with_annex(path): skip_if_no_module('scrapy') # e.g. not present under Python3 dataset_index = 1 dataset_name = 'ds%06d' % dataset_index dataset_url = 'https://openfmri.org/dataset/' + dataset_name # needs to be a non-existing directory dataset_path = opj(path, dataset_name) # we need to pre-initiate dataset list(initiate_dataset('openfmri', dataset_index, path=dataset_path)()) annex = Annexificator( dataset_path, create=False, # must be already initialized etc options=["-c", "annex.largefiles=exclude=*.txt and exclude=README"]) pipeline = [ crawl_url(dataset_url), [ # changelog a_href_match(".*release_history.txt"), # , limit=1 assign({'filename': 'changelog.txt'}), annex, ], [ # and collect all URLs under "AWS Link" css_match('.field-name-field-aws-link a', xpaths={'url': '@href', 'url_text': 'text()'}), annex, ], [ # and license information css_match('.field-name-field-license a', xpaths={'url': '@href', 'url_text': 'text()'}), assign({'filename': 'license.txt'}), annex, ], ] run_pipeline(pipeline) _PLUG_HERE = '' _versioned_files = """ Raw data on AWS version 1 Raw data on AWS version 2 Beh data on AWS version 2 """ @with_tree(tree={ 'ds666': { # there could also be a case of a file with "unique" name without versioned counterpart # e.g. ds666_models.tar.gz which seems to be not treated correctly (not placed into any # version in case of ds000017) 'index.html': """ Release History Raw data on AWS, no version %s """ % _PLUG_HERE, 'release_history.txt': '1.0.1 fixed\n1.0.0 whatever', 'ds666.tar.gz': {'ds-666': {'sub1': {'anat': {'sub-1_T1w.dat': "mighty load in old format"}}}}, 'ds666_R1.0.0.tar.gz': {'ds666': {'sub-1': {'anat': {'sub-1_T1w.dat': "mighty load 1.0.0"}}}}, 'ds666_R1.0.1.tar.gz': {'ds666': {'sub-1': {'anat': {'sub-1_T1w.dat': "mighty load 1.0.1"}}}}, 'ds666-beh_R1.0.1.tar.gz': {'beh.tar.gz': {'ds666': {'sub-1': {'beh': {'responses.tsv': "1"}}}}}, 'ds666_R2.0.0.tar.gz': {'ds666': {'sub-1': {'anat': {'sub-1_T1w.dat': "mighty load 2.0.0"}}}}, }}, archives_leading_dir=False ) @serve_path_via_http @with_tempfile @with_tempfile def test_openfmri_pipeline1(ind, topurl, outd, clonedir): index_html = opj(ind, 'ds666', 'index.html') list(initiate_dataset( template="openfmri", dataset_name='dataladtest-ds666', path=outd, data_fields=['dataset'])({'dataset': 'ds666'})) with chpwd(outd): pipeline = ofpipeline('ds666', versioned_urls=False, topurl=topurl) out = run_pipeline(pipeline) eq_(len(out), 1) repo = AnnexRepo(outd, create=False) # to be used in the checks # Inspect the tree -- that we have all the branches branches = {'master', 'incoming', 'incoming-processed', 'git-annex'} eq_(set(repo.get_branches()), branches) # We do not have custom changes in master yet, so it just follows incoming-processed atm # eq_(repo.get_hexsha('master'), repo.get_hexsha('incoming-processed')) # Since we did initiate_dataset -- now we have separate master! assert_not_equal(repo.get_hexsha('master'), repo.get_hexsha('incoming-processed')) # and that one is different from incoming assert_not_equal(repo.get_hexsha('incoming'), repo.get_hexsha('incoming-processed')) t1w_fpath_nover = opj(outd, 'sub1', 'anat', 'sub-1_T1w.dat') ok_file_has_content(t1w_fpath_nover, "mighty load in old format") # # And now versioned files were specified! # add_to_index(index_html, content=_versioned_files) with chpwd(outd): pipeline = ofpipeline('ds666', versioned_urls=False, topurl=topurl) out = run_pipeline(pipeline) eq_(len(out), 1) ok_(not exists(t1w_fpath_nover), "%s file should no longer be there if unversioned files get removed correctly" % t1w_fpath_nover) repo = AnnexRepo(outd, create=False) # to be used in the checks # Inspect the tree -- that we have all the branches branches = {'master', 'incoming', 'incoming-processed', 'git-annex'} eq_(set(repo.get_branches()), branches) # We do not have custom changes in master yet, so it just follows incoming-processed atm # eq_(repo.get_hexsha('master'), repo.get_hexsha('incoming-processed')) # Since we did initiate_dataset -- now we have separate master! assert_not_equal(repo.get_hexsha('master'), repo.get_hexsha('incoming-processed')) # and that one is different from incoming assert_not_equal(repo.get_hexsha('incoming'), repo.get_hexsha('incoming-processed')) # actually the tree should look quite neat with 1.0.0 tag having 1 parent in incoming # 1.0.1 having 1.0.0 and the 2nd commit in incoming as parents commits = {b: list(repo.get_branch_commits(b)) for b in branches} commits_hexsha = {b: list(repo.get_branch_commits(b, value='hexsha')) for b in branches} commits_l = {b: list(repo.get_branch_commits(b, limit='left-only')) for b in branches} eq_(len(commits['incoming']), 3) eq_(len(commits_l['incoming']), 3) eq_(len(commits['incoming-processed']), 6) eq_(len(commits_l['incoming-processed']), 4) # because original merge has only 1 parent - incoming eq_(len(commits['master']), 12) # all commits out there -- dataset init, crawler init + 3*(incoming, processed, meta data aggregation, merge) eq_(len(commits_l['master']), 6) # Check tags for the versions eq_(out[0]['datalad_stats'].get_total().versions, ['1.0.0', '1.0.1']) # +1 because original "release" was assumed to be 1.0.0 eq_([x.name for x in repo.repo.tags], ['1.0.0', '1.0.0+1', '1.0.1']) eq_(repo.repo.tags[0].commit.hexsha, commits_l['master'][-4].hexsha) # next to the last one eq_(repo.repo.tags[-1].commit.hexsha, commits_l['master'][0].hexsha) # the last one def hexsha(l): return l.__class__(x.hexsha for x in l) # Verify that we have desired tree of merges eq_(hexsha(commits_l['incoming-processed'][0].parents), (commits_l['incoming-processed'][1].hexsha, commits_l['incoming'][0].hexsha)) eq_(hexsha(commits_l['incoming-processed'][2].parents), (commits_l['incoming'][2].hexsha,)) eq_(hexsha(commits_l['master'][0].parents), (commits_l['master'][1].hexsha, commits_l['incoming-processed'][0].hexsha)) eq_(hexsha(commits_l['master'][1].parents), (commits_l['master'][2].hexsha, commits_l['incoming-processed'][1].hexsha)) with chpwd(outd): eq_(set(glob('*')), {'changelog.txt', 'sub-1'}) all_files = sorted(find_files('.')) t1w_fpath = opj(outd, 'sub-1', 'anat', 'sub-1_T1w.dat') ok_file_has_content(t1w_fpath, "mighty load 1.0.1") ok_file_under_git(opj(outd, 'changelog.txt'), annexed=False) ok_file_under_git(t1w_fpath, annexed=True) target_files = { './.datalad/config', './.datalad/meta/meta.json', './.datalad/crawl/crawl.cfg', # no more! # './.datalad/config.ttl', './.datalad/datalad.ttl', './.datalad/crawl/statuses/incoming.json', './.datalad/crawl/versions/incoming.json', './changelog.txt', './sub-1/anat/sub-1_T1w.dat', './sub-1/beh/responses.tsv'} target_incoming_files = { '.gitattributes', # we marked default backend right in the incoming 'changelog.txt', 'ds666.tar.gz', 'ds666-beh_R1.0.1.tar.gz', 'ds666_R1.0.0.tar.gz', 'ds666_R1.0.1.tar.gz', 'ds666_R2.0.0.tar.gz', '.datalad/crawl/statuses/incoming.json', '.datalad/crawl/versions/incoming.json' } eq_(set(all_files), target_files) # check that -beh was committed in 2nd commit in incoming, not the first one assert_not_in('ds666-beh_R1.0.1.tar.gz', repo.get_files(commits_l['incoming'][-1])) assert_in('ds666-beh_R1.0.1.tar.gz', repo.get_files(commits_l['incoming'][0])) # rerun pipeline -- make sure we are on the same in all branches! with chpwd(outd): out = run_pipeline(pipeline) eq_(len(out), 1) commits_hexsha_ = {b: list(repo.get_branch_commits(b, value='hexsha')) for b in branches} eq_(commits_hexsha, commits_hexsha_) # i.e. nothing new # actually we do manage to add_git 1 (README) since it is generated committed directly to git # BUT now fixed -- if not committed (was the same), should be marked as skipped # Nothing was committed so stats leaked all the way up eq_(out[0]['datalad_stats'], ActivityStats(files=5, skipped=5, urls=5)) eq_(out[0]['datalad_stats'], out[0]['datalad_stats'].get_total()) # rerun pipeline when new content is available # add new revision, rerun pipeline and check that stuff was processed/added correctly add_to_index(index_html, content='Raw data on AWS version 2.0.0') with chpwd(outd): out = run_pipeline(pipeline) all_files_updated = sorted(find_files('.')) eq_(len(out), 1) assert_not_equal(out[0]['datalad_stats'].get_total(), ActivityStats()) # there is no overlays ATM, so behav would be gone since no 2.0.0 for it! target_files.remove('./sub-1/beh/responses.tsv') eq_(set(all_files_updated), target_files) # new instance so it re-reads git stuff etc # repo = AnnexRepo(outd, create=False) # to be used in the checks commits_ = {b: list(repo.get_branch_commits(b)) for b in branches} commits_hexsha_ = {b: list(repo.get_branch_commits(b, value='hexsha')) for b in branches} commits_l_ = {b: list(repo.get_branch_commits(b, limit='left-only')) for b in branches} assert_not_equal(commits_hexsha, commits_hexsha_) eq_(out[0]['datalad_stats'], ActivityStats()) # commit happened so stats were consumed # numbers seems to be right total_stats = out[0]['datalad_stats'].get_total() # but for some reason downloaded_size fluctuates.... why? probably archiving...? total_stats.downloaded_size = 0 eq_(total_stats, ActivityStats(files=8, skipped=5, downloaded=1, renamed=1, urls=6, add_annex=2, # add_git=1, # README versions=['2.0.0'], merges=[['incoming', 'incoming-processed']])) check_dropall_get(repo) # Let's see if pipeline would remove files we stopped tracking remove_from_index(index_html, '') with chpwd(outd): with swallow_logs(new_level=logging.WARNING) as cml: out = run_pipeline(pipeline) # since files get removed in incoming, but repreprocessed completely # incomming-processed and merged into master -- new commits will come # They shouldn't have any difference but still should be new commits assert_in("There is already a tag 2.0.0 in the repository", cml.out) eq_(len(out), 1) incoming_files = repo.get_files('incoming') target_incoming_files.remove('ds666_R1.0.0.tar.gz') eq_(set(incoming_files), target_incoming_files) commits_hexsha_removed = {b: list(repo.get_branch_commits(b, value='hexsha')) for b in branches} # our 'statuses' database should have recorded the change thus got a diff # which propagated through all branches for b in 'master', 'incoming-processed': # with non persistent DB we had no changes # eq_(repo.repo.branches[b].commit.diff(commits_hexsha_[b][0]), []) eq_(repo.repo.branches[b].commit.diff(commits_hexsha_[b][0])[0].a_path, '.datalad/crawl/statuses/incoming.json') dincoming = repo.repo.branches['incoming'].commit.diff(commits_hexsha_['incoming'][0]) eq_(len(dincoming), 2) # 2 diff objects -- 1 file removed, 1 statuses updated eq_(set([d.a_path for d in dincoming]), {'.datalad/crawl/statuses/incoming.json', 'ds666_R1.0.0.tar.gz'}) # since it seems to diff "from current to the specified", it will be listed as new_file assert any(d.new_file for d in dincoming) eq_(out[0]['datalad_stats'].get_total().removed, 1) assert_not_equal(commits_hexsha_, commits_hexsha_removed) # we will check if a clone would be crawling just as good from datalad.api import crawl # make a brand new clone GitRepo(clonedir, outd) def _pipeline(*args, **kwargs): """Helper to mock openfmri.pipeline invocation so it looks at our 'server'""" kwargs = updated(kwargs, {'topurl': topurl, 'versioned_urls': False}) return ofpipeline(*args, **kwargs) with chpwd(clonedir), patch.object(openfmri, 'pipeline', _pipeline): output, stats = crawl() # we should be able to recrawl without doing anything ok_(stats, ActivityStats(files=6, skipped=6, urls=5)) test_openfmri_pipeline1.tags = ['integration'] @with_tree(tree={ 'ds666': { 'index.html': """ Release History Raw data on AWS version 1 %s """ % _PLUG_HERE, 'release_history.txt': '1.0.1 fixed\n1.0.0 whatever', 'ds666.tar.gz': {'ds666': {'sub-1': {'anat': {'sub-1_T1w.dat': "1.0.0"}}}}, # this one will get renamed, not added to index 'ds666_R2.0.0.tar.gz': {'ds666': {'sub-1': {'anat': {'sub-1_T1w.dat': "mighty load 2.0.0"}}}}, }}, archives_leading_dir=False ) @serve_path_via_http @with_tempfile def test_openfmri_pipeline2(ind, topurl, outd): # no versioned files -- should still work! ;) list(initiate_dataset( template="openfmri", dataset_name='dataladtest-ds666', path=outd, data_fields=['dataset'])({'dataset': 'ds666'})) with chpwd(outd): pipeline = ofpipeline('ds666', versioned_urls=False, topurl=topurl) out = run_pipeline(pipeline) eq_(len(out), 1) repo = AnnexRepo(outd, create=False) # to be used in the checks # Inspect the tree -- that we have all the branches branches = {'master', 'incoming', 'incoming-processed', 'git-annex'} eq_(set(repo.get_branches()), branches) # We do not have custom changes in master yet, so it just follows incoming-processed atm # eq_(repo.get_hexsha('master'), repo.get_hexsha('incoming-processed')) # Since we did initiate_dataset -- now we have separate master! assert_not_equal(repo.get_hexsha('master'), repo.get_hexsha('incoming-processed')) # and that one is different from incoming assert_not_equal(repo.get_hexsha('incoming'), repo.get_hexsha('incoming-processed')) # actually the tree should look quite neat with 1.0.0 tag having 1 parent in incoming # 1.0.1 having 1.0.0 and the 2nd commit in incoming as parents commits = {b: list(repo.get_branch_commits(b)) for b in branches} commits_hexsha = {b: list(repo.get_branch_commits(b, value='hexsha')) for b in branches} commits_l = {b: list(repo.get_branch_commits(b, limit='left-only')) for b in branches} eq_(len(commits['incoming']), 1) eq_(len(commits_l['incoming']), 1) eq_(len(commits['incoming-processed']), 2) eq_(len(commits_l['incoming-processed']), 2) # because original merge has only 1 parent - incoming # to avoid 'dataset init' commit create() needs save=False eq_(len(commits['master']), 6) # all commits out there, dataset init, crawler, init, incoming, incoming-processed, meta data aggregation, merge eq_(len(commits_l['master']), 4) # dataset init, init, meta data aggregation, merge # rerun pipeline -- make sure we are on the same in all branches! with chpwd(outd): out = run_pipeline(pipeline) eq_(len(out), 1) commits_hexsha_ = {b: list(repo.get_branch_commits(b, value='hexsha')) for b in branches} eq_(commits_hexsha, commits_hexsha_) # i.e. nothing new eq_(out[0]['datalad_stats'], ActivityStats(files=2, skipped=2, urls=2)) eq_(out[0]['datalad_stats'], out[0]['datalad_stats'].get_total()) os.rename(opj(ind, 'ds666', 'ds666_R2.0.0.tar.gz'), opj(ind, 'ds666', 'ds666.tar.gz')) with chpwd(outd): out = run_pipeline(pipeline) eq_(len(out), 1) eq_(out[0]['datalad_stats'], ActivityStats()) # was committed stats_total = out[0]['datalad_stats'].get_total() stats_total.downloaded_size = 0 eq_(stats_total, ActivityStats(files=4, overwritten=1, skipped=1, downloaded=1, merges=[['incoming', 'incoming-processed']], versions=['1.0.0'], renamed=1, urls=2, add_annex=2)) # in reality there is also 1.0.0+1 tag since file changed but no version suffix eq_([x.name for x in repo.repo.tags], ['1.0.0', '1.0.0+1']) check_dropall_get(repo) test_openfmri_pipeline2.tags = ['integration'] from ..openfmri_s3 import collection_pipeline, pipeline # TODO: RF to provide a generic/reusable test for this @with_tempfile(mkdir=True) def test_smoke_pipelines(d): # Just to verify that we can correctly establish the pipelines AnnexRepo(d, create=True) with chpwd(d): with swallow_logs(): for p in [pipeline('bogus'), collection_pipeline()]: ok_(len(p) > 1) datalad-0.4.1/datalad/crawler/pipelines/tests/test_openfmri_collection.py000066400000000000000000000054121301115270200267260ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import os from glob import glob from os.path import join as opj, exists from ...pipeline import run_pipeline, FinishPipeline from ...nodes.annex import Annexificator, initiate_dataset from ....support.stats import ActivityStats from ....support.gitrepo import GitRepo from ....utils import chpwd from ....tests.utils import with_tree from ....tests.utils import eq_, assert_not_equal, ok_, assert_raises from ....tests.utils import assert_in, assert_not_in from ....tests.utils import skip_if_no_module from ....tests.utils import with_tempfile from ....tests.utils import serve_path_via_http from ....tests.utils import skip_if_no_network from ....tests.utils import use_cassette from ....tests.utils import ok_file_has_content from ....tests.utils import ok_file_under_git from ....distribution.dataset import Dataset from ....distribution.dataset import Dataset from ....consts import CRAWLER_META_CONFIG_PATH from datalad.api import crawl from ..openfmri import superdataset_pipeline as ofcpipeline from logging import getLogger lgr = getLogger('datalad.crawl.tests') # if we decide to emulate change (e.g. new dataset added) _PLUG_HERE = '' @with_tree(tree={ 'index.html': """ ds001 ds002 %s """ % _PLUG_HERE, }, ) @serve_path_via_http @with_tempfile def test_openfmri_superdataset_pipeline1(ind, topurl, outd): list(initiate_dataset( template="openfmri", template_func="superdataset_pipeline", template_kwargs={'url': topurl}, path=outd, )()) with chpwd(outd): crawl() #pipeline = ofcpipeline(url=topurl) #out = run_pipeline(pipeline) #eq_(out, [{'datalad_stats': ActivityStats()}]) # TODO: replace below command with the one listing subdatasets subdatasets = ['ds000001', 'ds000002'] eq_(Dataset(outd).get_subdatasets(fulfilled=True), subdatasets) # Check that crawling configuration was created for every one of those for sub in subdatasets: repo = GitRepo(opj(outd, sub)) assert(not repo.dirty) assert(exists(opj(repo.path, CRAWLER_META_CONFIG_PATH))) # TODO: check that configuration for the crawler is up to the standard # Ideally should also crawl some fake datasets I guessdatalad-0.4.1/datalad/crawler/pipelines/tests/test_simple_with_archives.py000066400000000000000000000036161301115270200271100ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from os.path import join as opj from datalad.crawler.pipelines.tests.utils import _test_smoke_pipelines from ...nodes.annex import initiate_dataset from ....utils import chpwd from ....utils import _path_ from ....tests.utils import with_tree from ....tests.utils import eq_, assert_not_equal, ok_, assert_raises from ....tests.utils import with_tempfile from ....tests.utils import serve_path_via_http from ....tests.utils import ok_file_has_content from ....tests.utils import ok_file_under_git from ..simple_with_archives import pipeline from datalad.api import crawl from logging import getLogger lgr = getLogger('datalad.crawl.tests') def test_smoke_pipelines(): yield _test_smoke_pipelines, pipeline, ["random_url"] from .test_balsa import TEST_TREE1 # A little integration test @with_tree(tree=TEST_TREE1, archives_leading_dir=False) @serve_path_via_http @with_tempfile def test_simple1(ind, topurl, outd): list(initiate_dataset( template="simple_with_archives", dataset_name='test1', path=outd, add_fields={'url': topurl + 'study/show/WG33', 'a_href_match_': '.*download.*'} )({})) with chpwd(outd): out, stats = crawl() eq_(stats.add_annex, 3) ok_file_under_git(outd, 'file1.nii', annexed=True) ok_file_has_content(opj(outd, 'file1.nii'), 'content of file1.nii') ok_file_under_git(outd, _path_('dir1/file2.nii'), annexed=True) ok_file_has_content(opj(outd, 'dir1', 'file2.nii'), 'content of file2.nii') eq_(len(out), 1) datalad-0.4.1/datalad/crawler/pipelines/tests/utils.py000066400000000000000000000015521301115270200227760ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from ....support.annexrepo import AnnexRepo from ....utils import chpwd from ....utils import swallow_logs from ....tests.utils import ok_ from ....tests.utils import with_tempfile from logging import getLogger lgr = getLogger('datalad.crawl.tests') @with_tempfile(mkdir=True) def _test_smoke_pipelines(func, args, tmpdir): AnnexRepo(tmpdir, create=True) with chpwd(tmpdir): with swallow_logs(): for p in [func(*args)]: ok_(len(p) > 1) datalad-0.4.1/datalad/crawler/tests/000077500000000000000000000000001301115270200172715ustar00rootroot00000000000000datalad-0.4.1/datalad/crawler/tests/__init__.py000066400000000000000000000010661301115270200214050ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from datalad.tests.utils import skip_if_no_module skip_if_no_module('scrapy') from datalad.tests.utils import skip_if_scrapy_without_selector skip_if_scrapy_without_selector() datalad-0.4.1/datalad/crawler/tests/test_pipeline.py000066400000000000000000000231411301115270200225100ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from os.path import join as opj from datalad.tests.utils import skip_if_scrapy_without_selector skip_if_scrapy_without_selector() from ..nodes.crawl_url import crawl_url from ..nodes.matches import * from ..pipeline import run_pipeline, FinishPipeline from ..nodes.misc import Sink, assign, range_node, interrupt_if from ..nodes.annex import Annexificator from ..pipeline import load_pipeline_from_module from ...support.stats import ActivityStats from ...tests.utils import with_tree from ...tests.utils import eq_, ok_, assert_raises from ...tests.utils import assert_in from ...tests.utils import skip_if_no_module from ...tests.utils import with_tempfile from ...tests.utils import skip_if_no_network from ...tests.utils import use_cassette from logging import getLogger lgr = getLogger('datalad.crawl.tests') class AssertOrder(object): """Helper to verify that nodes executed in correct order Counter _call gets incremented with each invocation of the _call """ def __init__(self): self._call = 0 def __call__(self, numbers): if isinstance(numbers, int): numbers = {numbers} def _assert_order(data): self._call += 1 lgr.debug("#%d invocation of %s " % (self._call, self)) assert_in(self._call, numbers) yield data return _assert_order @with_tree(tree={ 'pipeline.py': 'pipeline = lambda: [1]', 'pipeline2.py': 'pipeline = lambda x: [2*x]', }) def test_load_pipeline_from_script(d): eq_(load_pipeline_from_module(opj(d, 'pipeline.py')), [1]) eq_(load_pipeline_from_module(opj(d, 'pipeline2.py'), kwargs=dict(x=2)), [4]) assert_raises(RuntimeError, load_pipeline_from_module, opj(d, 'unlikelytobethere.py')) DEFAULT_OUTPUT = [{'datalad_stats': ActivityStats()}] def _out(ld): """Adjust output entry to include default outputs as well """ outl = [] for d in ld: out = d.copy() outl.append(out) for k, v in DEFAULT_OUTPUT[0].items(): if k not in out: out[k] = v return outl def test_pipeline_linear_simple(): sink = Sink() pipeline = [ range_node(2, "out1"), range_node(3, "out2"), sink ] pipeline_output = run_pipeline(pipeline) eq_(pipeline_output, DEFAULT_OUTPUT) # by default 'input' is output and input is made empty dict if not provided eq_(sink.data, [{'out1': 0, 'out2': 0}, {'out1': 0, 'out2': 1}, {'out1': 0, 'out2': 2}, {'out1': 1, 'out2': 0}, {'out1': 1, 'out2': 1}, {'out1': 1, 'out2': 2}]) # if we extend pipeline with matching interrupt_if, the entire pipeline should # stop at that matching point, but otherwise there should be no crash etc sink.clean() pipeline_output = run_pipeline(pipeline + [interrupt_if({'out1': 0, 'out2': 1})]) eq_(pipeline_output, DEFAULT_OUTPUT) eq_(sink.data, [{'out1': 0, 'out2': 0}, {'out1': 0, 'out2': 1}]) def test_pipeline_unknown_opts(): assert_raises(ValueError, run_pipeline, [{'xxx': 1}]) def test_pipeline_linear_nested_order(): sink = Sink() sink2 = Sink() assert_order = AssertOrder() pipeline = [ assert_order(1), range_node(2, "out1"), assert_order({2, 5}), [ assert_order({3, 6}), range_node(3, "out2"), sink, ], assert_order({4, 7}), sink2 ] pipeline_output = run_pipeline(pipeline) def test_pipeline_linear_nested(): sink = Sink() sink2 = Sink() assert_order = AssertOrder() pipeline = [ range_node(2, "out1"), [ range_node(3, "out2"), sink, ], sink2 ] all_pairs = [{'out1': 0, 'out2': 0}, {'out1': 0, 'out2': 1}, {'out1': 0, 'out2': 2}, {'out1': 1, 'out2': 0}, {'out1': 1, 'out2': 1}, {'out1': 1, 'out2': 2}] pipeline_output = run_pipeline(pipeline) eq_(pipeline_output, DEFAULT_OUTPUT) eq_(sink.data, all_pairs) # and output is not seen outside of the nested pipeline eq_(sink2.data, [{'out1': 0}, {'out1': 1}]) # Let's make nested pipeline yield all sink.clean() sink2.clean() pipeline[1].insert(0, {'output': 'outputs'}) pipeline_output = run_pipeline(pipeline) eq_(pipeline_output, DEFAULT_OUTPUT) # by default no output produced eq_(sink.data, all_pairs) # and output was passed outside from the nested pipeline eq_(sink2.data, all_pairs) # Let's make it yield the last-output one sink2.clean() pipeline[1][0] = {'output': 'last-output'} pipeline_output = run_pipeline(pipeline) eq_(pipeline_output, DEFAULT_OUTPUT) # by default no output produced # only the last output from the nested pipeline appeared outside eq_(sink2.data, [{'out1': 0, 'out2': 2}, {'out1': 1, 'out2': 2}]) # Let's now add output to the top-most pipeline pipeline.insert(0, {'output': 'outputs'}) pipeline_output = run_pipeline(pipeline) eq_(pipeline_output, _out([{'out1': 0, 'out2': 2}, {'out1': 1, 'out2': 2}])) # and if we ask only for the last one pipeline[0] = {'output': 'last-output'} pipeline_output = run_pipeline(pipeline) eq_(pipeline_output, _out([{'out1': 1, 'out2': 2}])) def test_pipeline_recursive(): def less3(data): """a little helper which would not yield whenever input x>3""" if data['x'] < 3: yield updated(data, dict(x=data['x']+1)) pipeline = [ {'loop': True, 'output': 'outputs'}, less3, ] pipeline_output = run_pipeline(pipeline, dict(x=0)) eq_(pipeline_output, _out([{'x': 1}, {'x': 2}, {'x': 3}])) def test_pipeline_looping(): count = [0, 0] def count_threetimes(data): """helper to not yield anything if done it 3 times by now""" if count[0] >= 3: return count[0] += 1 for i in range(count[0]): yield updated(data, dict(somevar=(i, count[0]))) def add_count(data): count[1] += 1 yield updated(data, {'count': count[0]}) def passthrough(data): yield data pipeline_output = run_pipeline([{'loop': True}, count_threetimes], dict(x=0)) eq_(pipeline_output, _out([{'x': 0}])) eq_(count, [3, 0]) # and even if the node not yielding is note the first node pipeline_output = run_pipeline([{'loop': True}, passthrough, count_threetimes], dict(x=0)) eq_(pipeline_output, _out([{'x': 0}])) eq_(count, [3, 0]) count[0] = 0 # Let's rerun with explicit last-output, which would also affect output of this pipeline pipeline_output = run_pipeline([{'loop': True, 'output': 'last-output'}, count_threetimes], dict(x=0)) eq_(pipeline_output, _out([{'x': 0, 'somevar': (2, 3)}])) eq_(count, [3, 0]) # and if pipeline is composite, i.e. more than a single step, so we could make sure everything is called count[0] = 0 pipeline_output = run_pipeline([{'loop': True}, count_threetimes, add_count], dict(x=0)) eq_(pipeline_output, _out([{'x': 0}])) eq_(count, [3, 6]) count[0] = count[1] = 0 # Let's rerun with explicit last-output, which would also affect output of this pipeline pipeline_output = run_pipeline([{'loop': True, 'output': 'last-output'}, count_threetimes, add_count], dict(x=0)) eq_(pipeline_output, _out([{'x': 0, 'somevar': (2, 3), 'count': 3}])) eq_(count, [3, 6]) def test_pipeline_linear_top_isnested_pipeline(): # check if no generated data to reach the end node, it still gets executed was_called = [] pipeline = [ # range_node(1), [ range_node(1, "out2"), ], lambda d: was_called.append('yes') ] pipeline_output = run_pipeline(pipeline) eq_(was_called, ['yes']) def test_pipeline_updated_stats(): def n1(data): data['datalad_stats'].increment('add_git') yield data def n2(data): # doesn't care to maintain previous stats data = data.copy() data['datalad_stats'] = ActivityStats(files=2) data['out'] = 1 yield data pipeline_output = run_pipeline([{'output': 'outputs'}, n1, n2]) eq_(pipeline_output, [{'datalad_stats': ActivityStats(files=2, add_git=1), 'out': 1}]) def test_pipeline_dropped_stats(): def n1(data): data['datalad_stats'].increment('add_git') yield data def n2(data): # doesn't care to maintain previous stats yield {'out': 1} pipeline_output = run_pipeline([{'output': 'outputs'}, n1, n2]) eq_(pipeline_output, [{'datalad_stats': ActivityStats(add_git=1), 'out': 1}]) def test_pipeline_stats_persist(): # to test that we would get proper stats returned in various pipeline layouts def n1(data): data['datalad_stats'].increment('add_git') yield data def p(data): yield data def n2(data): # doesn't care to maintain previous stats data['datalad_stats'].increment('add_annex') yield data target_stats = ActivityStats(add_git=1, add_annex=1) def assert_pipeline(pipeline): eq_(run_pipeline(pipeline), [{'datalad_stats': target_stats}]) assert_pipeline([n1, n2]) assert_pipeline([n1, [n2]]) assert_pipeline([[n1], [n2]]) assert_pipeline([n1, [n2, p]]) assert_pipeline([[n1], n2]) assert_pipeline([[n1, p], n2]) datalad-0.4.1/datalad/customremotes/000077500000000000000000000000001301115270200174015ustar00rootroot00000000000000datalad-0.4.1/datalad/customremotes/__init__.py000066400000000000000000000007431301115270200215160ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Support of custom remotes (e.g. extraction from archives) """ __docformat__ = 'restructuredtext' datalad-0.4.1/datalad/customremotes/archives.py000066400000000000000000000365371301115270200215750ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Custom remote to support getting the load from archives present under annex""" __docformat__ = 'restructuredtext' import os from os.path import exists, join as opj from collections import OrderedDict from operator import itemgetter import logging lgr = logging.getLogger('datalad.customremotes.archive') lgr.log(5, "Importing datalad.customremotes.archive") from ..cmd import link_file_load from ..support.archives import ArchivesCache from ..support.network import URL from ..utils import getpwd from ..utils import unique from .base import AnnexCustomRemote from .main import main as super_main # TODO: RF functionality not specific to being a custom remote (loop etc) # into a separate class class ArchiveAnnexCustomRemote(AnnexCustomRemote): """Special custom remote allowing to obtain files from archives Archives should also be under annex control. """ CUSTOM_REMOTE_NAME = "archive" SUPPORTED_SCHEMES = (AnnexCustomRemote._get_custom_scheme(CUSTOM_REMOTE_NAME),) # Since we support only 1 scheme here URL_SCHEME = SUPPORTED_SCHEMES[0] URL_PREFIX = URL_SCHEME + ":" AVAILABILITY = "local" COST = 500 def __init__(self, persistent_cache=True, **kwargs): super(ArchiveAnnexCustomRemote, self).__init__(**kwargs) # annex requests load by KEY not but URL which it originally asked # about. So for a key we might get back multiple URLs and as a # heuristic let's use the most recently asked one self._last_url = None # for heuristic to choose among multiple URLs self._cache = ArchivesCache(self.path, persistent=persistent_cache) def stop(self, *args): """Stop communication with annex""" self._cache.clean() super(ArchiveAnnexCustomRemote, self).stop(*args) def get_file_url(self, archive_file=None, archive_key=None, file=None, size=None): """Given archive (file or a key) and a file -- compose URL for access Examples -------- dl+archive:SHA256E-s176--69...3e.tar.gz#path=1/d2/2d&size=123 when size of file within archive was known to be 123 dl+archive:SHA256E-s176--69...3e.tar.gz#path=1/d2/2d when size of file within archive was not provided Parameters ---------- size: int, optional Size of the file. If not provided, will simply be empty """ assert(file is not None) if archive_file is not None: if archive_key is not None: raise ValueError("Provide archive_file or archive_key - not both") archive_key = self.repo.get_file_key(archive_file) assert(archive_key is not None) attrs = OrderedDict() # looking forward for more if file: attrs['path'] = file.lstrip('/') if size is not None: attrs['size'] = size return str(URL(scheme=self.URL_SCHEME, path=archive_key, fragment=attrs)) @property def cache(self): return self._cache def _parse_url(self, url): """Parse url and return archive key, file within archive and additional attributes (such as size) """ url = URL(url) assert(url.scheme == self.URL_SCHEME) fdict = url.fragment_dict if 'path' not in fdict: # must be old-style key/path#size= assert '/' in url.path, "must be of key/path format" key, path = url.path.split('/', 1) else: key, path = url.path, fdict.pop('path') if 'size' in fdict: fdict['size'] = int(fdict['size']) return key, path, fdict def _gen_akey_afiles(self, key, sorted=False, unique_akeys=True): """Given a key, yield akey, afile pairs if `sorted`, then first those which have extracted version in local cache will be yielded Gets determined based on urls for datalad archives Made "generators all the way" as an exercise but also to delay any checks etc until really necessary. """ urls = self.get_URLS(key) akey_afiles = [ self._parse_url(url)[:2] # skip size for url in urls ] if unique_akeys: akey_afiles = unique(akey_afiles, key=itemgetter(0)) if not sorted: for pair in akey_afiles: yield pair return # Otherwise we will go through each one # multiple URLs are available so we need to figure out which one # would be most efficient to "deal with" akey_afile_paths = ( ((akey, afile), self.get_contentlocation( akey, absolute=True, verify_exists=False )) for akey, afile in akey_afiles ) # by default get_contentlocation would return empty result for a key # which is not available locally. But we could still have extracted archive # in the cache. So we need pretty much get first all possible and then # only remove those which aren't present locally. So verify_exists was added yielded = set() akey_afile_paths_ = [] # utilize cache to check which archives might already be present in the cache for akey_afile, akey_path in akey_afile_paths: if akey_path and self.cache[akey_path].is_extracted: yield akey_afile yielded.add(akey_afile) akey_afile_paths_.append((akey_afile, akey_path)) # replace generators with already collected ones into a list. # The idea that in many cases we don't even need to create a full list # and that initial single yield would be enough, thus we don't need to check # locations etc for every possible hit akey_afile_paths = akey_afile_paths_ # if not present in the cache -- check which are present # locally and choose that one to use, so it would get extracted for akey_afile, akey_path in akey_afile_paths: if akey_path and exists(akey_path): yielded.add(akey_afile) yield akey_afile # So no archive is present either in the cache or originally under annex # XXX some kind of a heuristic I guess is to use last_url ;-) if self._last_url and self._last_url in urls and (len(urls) == len(akey_afiles)): akey_afile, _ = akey_afile_paths[urls.index(self._last_url)] yielded.add(akey_afile) yield akey_afile for akey_afile, _ in akey_afile_paths: if akey_afile not in yielded: yield akey_afile # Protocol implementation def req_CHECKURL(self, url): """ Replies CHECKURL-CONTENTS Size|UNKNOWN Filename Indicates that the requested url has been verified to exist. The Size is the size in bytes, or use "UNKNOWN" if the size could not be determined. The Filename can be empty (in which case a default is used), or can specify a filename that is suggested to be used for this url. CHECKURL-MULTI Url Size|UNKNOWN Filename ... Indicates that the requested url has been verified to exist, and contains multiple files, which can each be accessed using their own url. Note that since a list is returned, neither the Url nor the Filename can contain spaces. CHECKURL-FAILURE Indicates that the requested url could not be accessed. """ # TODO: what about those MULTI and list to be returned? # should we return all filenames or keys within archive? # might be way too many? # only if just archive portion of url is given or the one pointing # to specific file? lgr.debug("Current directory: %s, url: %s" % (os.getcwd(), url)) akey, afile, attrs = self._parse_url(url) size = attrs.get('size', None) # But reply that present only if archive is present # TODO: this would throw exception if not present, so this statement is kinda bogus akey_path = self.get_contentlocation(akey, absolute=True) if akey_path: # Extract via cache only if size is not yet known if size is None: # if for testing we want to force getting the archive extracted # _ = self.cache.assure_extracted(self._get_key_path(akey)) # TEMP efile = self.cache[akey_path].get_extracted_filename(afile) if exists(efile): size = os.stat(efile).st_size if size is None: size = 'UNKNOWN' # FIXME: providing filename causes annex to not even talk to ask # upon drop :-/ self.send("CHECKURL-CONTENTS", size) # , basename(afile)) # so it was a good successful one -- record self._last_url = url else: # TODO: theoretically we should first check if key is available from # any remote to know if file is available self.send("CHECKURL-FAILURE") def req_CHECKPRESENT(self, key): """Check if copy is available TODO: just proxy the call to annex for underlying tarball Replies CHECKPRESENT-SUCCESS Key Indicates that a key has been positively verified to be present in the remote. CHECKPRESENT-FAILURE Key Indicates that a key has been positively verified to not be present in the remote. CHECKPRESENT-UNKNOWN Key ErrorMsg Indicates that it is not currently possible to verify if the key is present in the remote. (Perhaps the remote cannot be contacted.) """ # TODO: so we need to maintain mapping from urls to keys. Then # we could even store the filename within archive # Otherwise it is unrealistic to even require to recompute key if we # knew the backend etc lgr.debug("VERIFYING key %s" % key) # The same content could be available from multiple locations within the same # archive, so let's not ask it twice since here we don't care about "afile" for akey, _ in self._gen_akey_afiles(key, unique_akeys=True): if self.get_contentlocation(akey) or self.repo.is_available(akey, batch=True, key=True): self.send("CHECKPRESENT-SUCCESS", key) return self.send("CHECKPRESENT-UNKNOWN", key) def req_REMOVE(self, key): """ REMOVE-SUCCESS Key Indicates the key has been removed from the remote. May be returned if the remote didn't have the key at the point removal was requested REMOVE-FAILURE Key ErrorMsg Indicates that the key was unable to be removed from the remote. """ self.send("REMOVE-FAILURE", key, "Removal from file archives is not supported") return # # TODO: proxy query to the underlying tarball under annex that if # # tarball was removed (not available at all) -- report success, # # otherwise failure (current the only one) # akey, afile = self._get_akey_afile(key) # if False: # # TODO: proxy, checking present of local tarball is not sufficient # # not exists(self.get_key_path(key)): # self.send("REMOVE-SUCCESS", akey) # else: # self.send("REMOVE-FAILURE", akey, # "Removal from file archives is not supported") def req_WHEREIS(self, key): """ WHEREIS-SUCCESS String Indicates a location of a key. Typically an url, the string can be anything that it makes sense to display to the user about content stored in the special remote. WHEREIS-FAILURE Indicates that no location is known for a key. """ self.send("WHEREIS-FAILURE") # although more logical is to report back success, it leads to imho more confusing # duplication. See # http://git-annex.branchable.com/design/external_special_remote_protocol/#comment-3f9588f6a972ae566347b6f467b53b54 # try: # key, file = self._get_akey_afile(key) # self.send("WHEREIS-SUCCESS", "file %s within archive %s" % (file, key)) # except ValueError: # self.send("WHEREIS-FAILURE") def _transfer(self, cmd, key, path): akeys_tried = [] # the same file could come from multiple files within the same archive # So far it doesn't make sense to "try all" of them since if one fails # it means the others would fail too, so it makes sense to immediately # prune the list so we keep only the ones from unique akeys. # May be whenever we support extraction directly from the tarballs # we should go through all and choose the one easiest to get or smth. for akey, afile in self._gen_akey_afiles(key, sorted=True, unique_akeys=True): akeys_tried.append(akey) try: akey_fpath = self.get_contentlocation(akey) if not akey_fpath: # TODO: make it more stringent? # Command could have fail to run if key was not present locally yet # Thus retrieve the key using annex # TODO: we need to report user somehow about this happening and progress on the download self.runner(["git-annex", "get", "--key", akey], cwd=self.path, expect_stderr=True) akey_fpath = self.get_contentlocation(akey) if not akey_fpath: raise RuntimeError("We were reported to fetch it alright but now can't get its location. Check logic") akey_path = opj(self.repo.path, akey_fpath) assert exists(akey_path), "Key file %s is not present" % akey_path # Extract that bloody file from the bloody archive # TODO: implement/use caching, for now a simple one # actually patool doesn't support extraction of a single file # https://github.com/wummel/patool/issues/20 # so pwd = getpwd() lgr.debug("Getting file {afile} from {akey_path} while PWD={pwd}".format(**locals())) apath = self.cache[akey_path].get_extracted_file(afile) link_file_load(apath, path) self.send('TRANSFER-SUCCESS', cmd, key) return except Exception as exc: # from celery.contrib import rdb # rdb.set_trace() from datalad.dochelpers import exc_str exc_ = exc_str(exc) self.debug("Failed to fetch {akey} containing {key}: {exc_}".format(**locals())) continue self.error("Failed to fetch any archive containing {key}. Tried: {akeys}".format(**locals())) def main(): """cmdline entry point""" super_main(backend="archive") lgr.log(5, "Done importing datalad.customremotes.archive") datalad-0.4.1/datalad/customremotes/base.py000066400000000000000000000457261301115270200207030ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Base classes to custom git-annex remotes (e.g. extraction from archives)""" from __future__ import absolute_import __docformat__ = 'restructuredtext' import errno import os import sys from os.path import exists, join as opj, realpath, dirname, lexists from six.moves import range from six.moves.urllib.parse import urlparse import logging lgr = logging.getLogger('datalad.customremotes') lgr.log(5, "Importing datalad.customremotes.main") from ..ui import ui from ..support.protocol import ProtocolInterface from ..support.cache import DictCache from ..cmdline.helpers import get_repo_instance URI_PREFIX = "dl" SUPPORTED_PROTOCOL = 1 DEFAULT_COST = 100 DEFAULT_AVAILABILITY = "local" from datalad.ui.progressbars import ProgressBarBase class AnnexRemoteQuit(Exception): pass class AnnexExchangeProtocol(ProtocolInterface): """A little helper to protocol interactions of custom remote with annex """ HEADER = r"""#!/bin/bash set -e # Gets a VALUE response and stores it in $RET report () { echo "$@" >&2 } recv () { read resp #resp=${resp%\n} target="$@" if [ "$resp" != "$target" ]; then report "! exp $target" report " got $resp" else report "+ got $resp" fi } send () { echo "$@" report "sent $@" } """ def __init__(self, repopath, custom_remote_name=None): super(AnnexExchangeProtocol, self).__init__() self.repopath = repopath self.custom_remote_name = custom_remote_name self._file = None self._initiated = False def initiate(self): if self._initiated: return self._initiated = True d = opj(self.repopath, '.git', 'bin') if not exists(d): os.makedirs(d) suf = '-' + self.custom_remote_name.rstrip(':') if self.custom_remote_name else '' self._file = _file = opj(d, 'git-annex-remote-datalad' + suf) if exists(_file): lgr.debug("Commenting out previous entries") # comment out all the past entries with open(_file) as f: entries = f.readlines() for i in range(len(self.HEADER.split(os.linesep)), len(entries)): e = entries[i] if e.startswith('recv ') or e.startswith('send '): entries[i] = '#' + e with open(_file, 'w') as f: f.write(''.join(entries)) return # nothing else to be done lgr.debug("Initiating protocoling." "cd %s; vim %s" % (realpath(self.repopath), _file[len(self.repopath) + 1:])) with open(_file, 'a') as f: f.write(self.HEADER) os.chmod(_file, 0o755) def write_section(self, cmd): self.initiate() with open(self._file, 'a') as f: f.write('%s### %s%s' % (os.linesep, cmd, os.linesep)) lgr.debug("New section in the protocol: " "cd %s; PATH=%s:$PATH %s" % (realpath(self.repopath), dirname(self._file), cmd)) def write_entries(self, entries): self.initiate() with open(self._file, 'a') as f: f.write(os.linesep.join(entries + [''])) def __iadd__(self, entry): self.initiate() with open(self._file, 'a') as f: f.write(entry + os.linesep) return self def start_section(self, cmd): self._sections.append({'command': cmd}) self.write_section(cmd) return len(self._sections) - 1 def end_section(self, id_, exception): # raise exception in case of invalid id_ for consistency: self._sections.__getitem__(id_) def add_section(self, cmd, exception): self.start_section(cmd) @property def records_callables(self): return False @property def records_ext_commands(self): return True @property def do_execute_ext_commands(self): return True @property def do_execute_callables(self): return True class AnnexCustomRemote(object): """Base class to provide custom special remotes for git-annex Implements git-annex special custom remotes protocol described at http://git-annex.branchable.com/design/external_special_remote_protocol/ """ # Must be defined in subclasses. There is no classlevel properties, so leaving as this for now CUSTOM_REMOTE_NAME = None # if None -- no additional custom remote name # SUPPORTED_SCHEMES = () COST = DEFAULT_COST AVAILABILITY = DEFAULT_AVAILABILITY def __init__(self, path=None, cost=None): # , availability=DEFAULT_AVAILABILITY): """ Parameters ---------- path : string, optional Path to the repository for which this custom remote is serving. Usually this class is instantiated by a script which runs already within that directory, so the default is to point to current directory, i.e. '.' """ # TODO: probably we shouldn't have runner here but rather delegate # to AnnexRepo's functionality from ..support.annexrepo import AnnexRepo from ..cmd import GitRunner self.runner = GitRunner() # Custom remotes correspond to annex via stdin/stdout self.fin = sys.stdin self.fout = sys.stdout self.repo = get_repo_instance(class_=AnnexRepo) \ if not path \ else AnnexRepo(path, create=False, init=False) self.path = self.repo.path self._progress = 0 # transmission to be reported back if available if cost is None: cost = self.COST self.cost = cost #self.availability = availability.upper() assert(self.AVAILABILITY.upper() in ("LOCAL", "GLOBAL")) # To signal either we are in the loop and e.g. could correspond to annex self._in_the_loop = False self._protocol = \ AnnexExchangeProtocol(self.path, self.CUSTOM_REMOTE_NAME) \ if os.environ.get('DATALAD_TESTS_PROTOCOLREMOTE') else None self._contentlocations = DictCache(size_limit=100) # TODO: config ? # instruct annex backend UI to use this remote if ui.backend == 'annex': ui.set_specialremote(self) @classmethod def _get_custom_scheme(cls, prefix): """Helper to generate custom datalad URL prefixes """ # prefix which will be used in all URLs supported by this custom remote # https://tools.ietf.org/html/rfc2718 dictates "URL Schemes" standard # 2.1.2 suggests that we do use // since all of our URLs will define # some hierarchical structure. But actually since we might encode # additional information (such as size) into the URL, it will not be # strictly conforming it. Thus we will not use // return "%s+%s" % (URI_PREFIX, prefix) # if .PREFIX else '') # Helpers functionality def get_contentlocation(self, key, absolute=False, verify_exists=True): """Return (relative to top or absolute) path to the file containing the key This is a wrapper around AnnexRepo.get_contentlocation which provides caching of the result (we are asking the location for the same archive key often) """ if key not in self._contentlocations: fpath = self.repo.get_contentlocation(key, batch=True) if fpath: # shouldn't store empty ones self._contentlocations[key] = fpath else: fpath = self._contentlocations[key] # but verify that it exists if verify_exists and not lexists(opj(self.path, fpath)): # prune from cache del self._contentlocations[key] fpath = '' if absolute and fpath: return opj(self.path, fpath) else: return fpath # # Communication with git-annex # def send(self, *args): """Send a message to git-annex Parameters ---------- `*args`: list of strings arguments to be joined by a space and passed to git-annex """ msg = " ".join(map(str, args)) if not self._in_the_loop: lgr.debug("We are not yet in the loop, thus should not send to annex" " anything. Got: %s" % msg.encode()) return try: self.heavydebug("Sending %r" % msg) self.fout.write(msg + "\n") # .encode()) self.fout.flush() if self._protocol is not None: self._protocol += "send %s" % msg except IOError as exc: lgr.debug("Failed to send due to %s" % str(exc)) if exc.errno == errno.EPIPE: self.stop() else: raise exc def send_unsupported(self): self.send("UNSUPPORTED-REQUEST") def read(self, req=None, n=1): """Read a message from git-annex Parameters ---------- req : string, optional Expected request - first msg of the response n : int Number of response elements after first msg """ # TODO: should we strip or should we not? verify how annex would deal # with filenames starting/ending with spaces - encoded? # Split right away l = self.fin.readline().rstrip(os.linesep) if self._protocol is not None: self._protocol += "recv %s" % l msg = l.split(None, n) if req and (req != msg[0]): # verify correct response was given self.error("Expected %r, got %r. Ignoring" % (req, msg[0])) return None self.heavydebug("Received %r" % (msg,)) return msg # TODO: see if we could adjust the "originating" file:line, because # otherwise they are all reported from main.py:117 etc def heavydebug(self, msg, *args, **kwargs): lgr.log(4, msg, *args, **kwargs) # Since protocol allows for some messaging back, let's duplicate to lgr def debug(self, msg): lgr.debug(msg) self.send("DEBUG", msg) def error(self, msg, annex_err="ERROR"): lgr.error(msg) self.send(annex_err, msg) def progress(self, bytes): bytes = int(bytes) if self._progress != bytes: self.send("PROGRESS", bytes) def main(self): """Interface to the command line tool""" try: self._in_the_loop = True self._loop() except AnnexRemoteQuit: pass # no harm except KeyboardInterrupt: self.stop("Interrupted by user") except Exception as e: self.stop(str(e)) finally: self._in_the_loop = False def stop(self, msg=None): lgr.debug("Stopping communications of %s%s" % (self, ": %s" % msg if msg else "")) raise AnnexRemoteQuit(msg) def _loop(self): """The main loop """ self.send("VERSION", SUPPORTED_PROTOCOL) while True: l = self.read(n=-1) if l is not None and not l: # empty line: exit self.stop() return req, req_load = l[0], l[1:] method = getattr(self, "req_%s" % req, None) if not method: self.error("We have no support for %s request, part of %s response" % (req, l)) self.send("UNSUPPORTED-REQUEST") continue try: method(*req_load) except Exception as e: self.error("Problem processing %r with parameters %r: %r" % (req, req_load, e)) from traceback import format_exc lgr.error("Caught exception detail: %s" % format_exc()) def req_INITREMOTE(self, *args): """Initialize this remote. Provides high level abstraction. Specific implementation should go to _initialize """ try: self._initremote(*args) except Exception as e: self.error("Failed to initialize %s due to %s" % (self, e), "INITREMOTE-FAILURE") else: self.send("INITREMOTE-SUCCESS") def req_PREPARE(self, *args): """Prepare "to deliver". Provides high level abstraction Specific implementation should go to _prepare """ try: self._prepare(*args) except Exception as e: self.error("Failed to prepare %s due to %s" % (self, e), "PREPARE-FAILURE") else: self.send("PREPARE-SUCCESS") def req_GETCOST(self): self.send("COST", self.cost) def req_GETAVAILABILITY(self): self.send("AVAILABILITY", self.AVAILABILITY.upper()) def req_CLAIMURL(self, url): scheme = urlparse(url).scheme if scheme in self.SUPPORTED_SCHEMES: self.debug("Claiming url %r" % url) self.send("CLAIMURL-SUCCESS") else: self.debug("Not claiming url %s" % url) self.send("CLAIMURL-FAILURE") # TODO: we should unify what to be overriden and some will provide CHECKURL def req_TRANSFER(self, cmd, key, file): if cmd in ("RETRIEVE",): lgr.debug("%s key %s into/from %s" % (cmd, key, file)) # was INFO level self._transfer(cmd, key, file) else: self.error("Retrieved unsupported for TRANSFER command %s" % cmd) self.send_unsupported() # Specific implementations to be provided in derived classes when necessary def req_CHECKURL(self, url): """ The remote replies with one of CHECKURL-FAILURE, CHECKURL-CONTENTS, or CHECKURL-MULTI. CHECKURL-CONTENTS Size|UNKNOWN Filename Indicates that the requested url has been verified to exist. The Size is the size in bytes, or use "UNKNOWN" if the size could not be determined. The Filename can be empty (in which case a default is used), or can specify a filename that is suggested to be used for this url. CHECKURL-MULTI Url Size|UNKNOWN Filename ... Indicates that the requested url has been verified to exist, and contains multiple files, which can each be accessed using their own url. Note that since a list is returned, neither the Url nor the Filename can contain spaces. CHECKURL-FAILURE Indicates that the requested url could not be accessed. """ self.send_unsupported() def req_CHECKPRESENT(self, key): """ CHECKPRESENT-SUCCESS Key Indicates that a key has been positively verified to be present in the remote. CHECKPRESENT-FAILURE Key Indicates that a key has been positively verified to not be present in the remote. CHECKPRESENT-UNKNOWN Key ErrorMsg Indicates that it is not currently possible to verify if the key is present in the remote. (Perhaps the remote cannot be contacted.) """ raise NotImplementedError() def req_REMOVE(self, key): """ REMOVE-SUCCESS Key Indicates the key has been removed from the remote. May be returned if the remote didn't have the key at the point removal was requested. REMOVE-FAILURE Key ErrorMsg Indicates that the key was unable to be removed from the remote. """ raise NotImplementedError() def req_WHEREIS(self, key): """Added in 5.20150812-17-g6bc46e3 provide any information about ways to access the content of a key stored in it, such as eg, public urls. This will be displayed to the user by eg, git annex whereis. The remote replies with WHEREIS-SUCCESS or WHEREIS-FAILURE. Note that users expect git annex whereis to run fast, without eg, network access. This is not needed when SETURIPRESENT is used, since such uris are automatically displayed by git annex whereis. WHEREIS-SUCCESS String Indicates a location of a key. Typically an url, the string can be anything that it makes sense to display to the user about content stored in the special remote. WHEREIS-FAILURE Indicates that no location is known for a key. """ raise NotImplementedError() def _transfer(self, cmd, key, file): raise NotImplementedError() def _initremote(self, *args): """Custom initialization of the special custom remote.""" pass def _prepare(self, *args): """Prepare special custom remote. To be overridden""" pass # some requests we can send out def get_DIRHASH(self, key, full=False): """Gets a two level hash associated with a Key. Parameters ---------- full: bool, optional If True, would spit out full DIRHASH path, i.e. with a KEY/ directory Something like "abc/def". This is always the same for any given Key, so can be used for eg, creating hash directory structures to store Keys in. """ self.send("DIRHASH", key) val = self.read("VALUE", 1)[1] if full: return opj(self.path, val, key) else: return val def get_URLS(self, key): """Gets URL(s) associated with a Key. """ urls = [] for scheme in self.SUPPORTED_SCHEMES: scheme_ = scheme + ":" self.send("GETURLS", key, scheme_) while True: url = self.read("VALUE", 1)[1:] if url: assert(len(url) == 1) urls.append(url[0]) else: break self.heavydebug("Got %d URL(s) for key %s: %s", len(urls), key, urls) if not urls: raise ValueError("Did not get any URLs for %s which we support" % key) return urls def _get_key_path(self, key): """Return path to the KEY file """ # TODO: should actually be implemented by AnnexRepo # Command is available in annex >= 20140410 (out, err) = \ self.runner(['git-annex', 'contentlocation', key], cwd=self.path) # TODO: it would exit with non-0 if key is not present locally. # we need to catch and throw our exception return opj(self.path, out.rstrip(os.linesep)) # TODO: test on annex'es generated with those new options e.g.-c annex.tune.objecthash1=true #def get_GETCONFIG SETCONFIG SETCREDS GETCREDS GETUUID GETGITDIR SETWANTED GETWANTED #SETSTATE GETSTATE SETURLPRESENT SETURLMISSING lgr.log(5, "Done importing datalad.customremotes.main") datalad-0.4.1/datalad/customremotes/datalad.py000066400000000000000000000154011301115270200213460ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Universal custom remote to support anything our downloaders support""" __docformat__ = 'restructuredtext' import logging lgr = logging.getLogger('datalad.customremotes.datalad') from ..utils import swallow_logs from .base import AnnexCustomRemote from ..dochelpers import exc_str from ..downloaders.providers import Providers from ..downloaders.base import TargetFileAbsent from .main import main as super_main class DataladAnnexCustomRemote(AnnexCustomRemote): """Special custom remote allowing to obtain files from archives Archives should also be under annex control. """ SUPPORTED_SCHEMES = ('http', 'https', 's3') AVAILABILITY = "global" def __init__(self, persistent_cache=True, **kwargs): super(DataladAnnexCustomRemote, self).__init__(**kwargs) # annex requests load by KEY not but URL which it originally asked # about. So for a key we might get back multiple URLs and as a # heuristic let's use the most recently asked one self._last_url = None # for heuristic to choose among multiple URLs self._providers = Providers.from_config_files() # # Helper methods # Protocol implementation def req_CHECKURL(self, url): """ Replies CHECKURL-CONTENTS Size|UNKNOWN Filename Indicates that the requested url has been verified to exist. The Size is the size in bytes, or use "UNKNOWN" if the size could not be determined. The Filename can be empty (in which case a default is used), or can specify a filename that is suggested to be used for this url. CHECKURL-MULTI Url Size|UNKNOWN Filename ... Indicates that the requested url has been verified to exist, and contains multiple files, which can each be accessed using their own url. Note that since a list is returned, neither the Url nor the Filename can contain spaces. CHECKURL-FAILURE Indicates that the requested url could not be accessed. """ try: with swallow_logs(): status = self._providers.get_status(url) size = str(status.size) if status.size is not None else 'UNKNOWN' resp = ["CHECKURL-CONTENTS", size] \ + ([status.filename] if status.filename else []) except Exception as exc: self.debug("Failed to check url %s: %s" % (url, exc_str(exc))) resp = ["CHECKURL-FAILURE"] self.send(*resp) def req_CHECKPRESENT(self, key): """Check if copy is available Replies CHECKPRESENT-SUCCESS Key Indicates that a key has been positively verified to be present in the remote. CHECKPRESENT-FAILURE Key Indicates that a key has been positively verified to not be present in the remote. CHECKPRESENT-UNKNOWN Key ErrorMsg Indicates that it is not currently possible to verify if the key is present in the remote. (Perhaps the remote cannot be contacted.) """ lgr.debug("VERIFYING key %s" % key) resp = None for url in self.get_URLS(key): # somewhat duplicate of CHECKURL try: with swallow_logs(): status = self._providers.get_status(url) if status: # TODO: anything specific to check??? resp = "CHECKPRESENT-SUCCESS" break # TODO: for CHECKPRESENT-FAILURE we somehow need to figure out that # we can connect to that server but that specific url is N/A, # probably check the connection etc except TargetFileAbsent as exc: self.debug("Target url %s file seems to be missing: %s" % (url, exc_str(exc))) if not resp: # if it is already marked as UNKNOWN -- let it stay that way # but if not -- we might as well say that we can no longer access it resp = "CHECKPRESENT-FAILURE" except Exception as exc: resp = "CHECKPRESENT-UNKNOWN" self.debug("Failed to check status of url %s: %s" % (url, exc_str(exc))) if resp is None: resp = "CHECKPRESENT-UNKNOWN" self.send(resp, key) def req_REMOVE(self, key): """ REMOVE-SUCCESS Key Indicates the key has been removed from the remote. May be returned if the remote didn't have the key at the point removal was requested REMOVE-FAILURE Key ErrorMsg Indicates that the key was unable to be removed from the remote. """ self.send("REMOVE-FAILURE", key, "Removal of content from urls is not possible") def req_WHEREIS(self, key): """ WHEREIS-SUCCESS String Indicates a location of a key. Typically an url, the string can be anything that it makes sense to display to the user about content stored in the special remote. WHEREIS-FAILURE Indicates that no location is known for a key. """ # All that information is stored in annex itself, we can't complement anything self.send("WHEREIS-FAILURE") def _transfer(self, cmd, key, path): # TODO: We might want that one to be a generator so we do not bother requesting # all possible urls at once from annex. urls = self.get_URLS(key) if self._last_url in urls: # place it first among candidates... some kind of a heuristic urls.pop(self._last_url) urls = [self._last_url] + urls # TODO: priorities etc depending on previous experience or settings for url in urls: try: downloaded_path = self._providers.download( url, path=path, overwrite=True ) lgr.info("Successfully downloaded %s into %s" % (url, downloaded_path)) self.send('TRANSFER-SUCCESS', cmd, key) return except Exception as exc: self.debug("Failed to download url %s for key %s: %s" % (url, key, exc_str(exc))) self.send('TRANSFER-FAILURE', cmd, key, "Failed to download from any of %d locations" % len(urls)) def main(): """cmdline entry point""" super_main(backend="datalad") datalad-0.4.1/datalad/customremotes/main.py000066400000000000000000000112041301115270200206750ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """""" __docformat__ = 'restructuredtext' import argparse import os from .. import __doc__ as m__doc__, __version__ as m__version__ from ..log import lgr from ..cmdline import helpers from ..cmdline.main import _license_info from ..utils import setup_exceptionhook from ..ui import ui backends = ['archive'] def setup_parser(backend): suffix, desc = { 'datalad': ( '', "download content from various URLs (http{,s}, s3, etc) possibly " "requiring authentication or custom access mechanisms using " "DataLad's downloaders"), 'archive': ( '-archive', "extract content from archives (.tar{,.gz}, .zip, etc) which are " "in turn managed by git-annex. See `datalad add-archive-content` " "command") }[backend] # setup cmdline args parser # main parser # TODO: should be encapsulated for resharing with the main datalad's parser = argparse.ArgumentParser( fromfile_prefix_chars='@', # usage="%(prog)s ...", description="%s\n\n" % m__doc__ + "git-annex-remote-datalad%s is a git-annex custom special remote to %s" % (suffix, desc), epilog='"DataLad\'s git-annex very special remote"', formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False, ) # common options helpers.parser_add_common_opt(parser, 'help') helpers.parser_add_common_opt(parser, 'log_level') helpers.parser_add_common_opt( parser, 'version', version='datalad %s\n\n%s' % (m__version__, _license_info())) if __debug__: parser.add_argument( '--dbg', action='store_true', dest='common_debug', help="Catch exceptions and fall into debugger upon exception") parser.add_argument( "command", nargs="?", help="Command to run. TODO: list per each custom remote" ) return parser def _main(args, backend=None): """Unprotected portion""" # TODO: For now we have only one, but we might need to actually become aware # of multiple and provide multiple helpers/symlinks # OR consider implementing ultimate handler of all dl+ URLs within a # single beast -- probably wouldn't fly since we do might need different # initializations etc. assert(backend is not None) if backend == 'archive': from .archives import ArchiveAnnexCustomRemote remote = ArchiveAnnexCustomRemote() elif backend == 'datalad': from .datalad import DataladAnnexCustomRemote remote = DataladAnnexCustomRemote() else: raise ValueError("I don't know anything about %r backend. " "Known are: %s" % backends) # TODO: handle args -- not used ATM # Generic commands to support: # get-uri-prefix # get-uri [options] with [options] being backend specific # e.g. for archive --key KEY(archive) --file FILE command = args.command if command == 'get-uri-prefix': print(remote.url_prefix) elif args.command is None: # If no command - run the special remote if 'DATALAD_TESTS_USECASSETTE' in os.environ: # optionally feeding it a cassette, used by tests from ..support.vcr_ import use_cassette with use_cassette(os.environ['DATALAD_TESTS_USECASSETTE']): remote.main() else: remote.main() else: raise ValueError("Unknown command %s" % command) def main(args=None, backend=None): import sys # TODO: redirect lgr output to stderr if it is stdout and not "forced" # by env variable... # parser = setup_parser(backend=backend) # parse cmd args args = parser.parse_args(args) ui.set_backend('annex') # stdin/stdout will be used for interactions with annex if args.common_debug: # So we could see/stop clearly at the point of failure setup_exceptionhook() _main(args, backend) else: # Otherwise - guard and only log the summary. Postmortem is not # as convenient if being caught in this ultimate except try: _main(args, backend) except Exception as exc: lgr.error('%s (%s)' % (str(exc), exc.__class__.__name__)) sys.exit(1) datalad-0.4.1/datalad/customremotes/tests/000077500000000000000000000000001301115270200205435ustar00rootroot00000000000000datalad-0.4.1/datalad/customremotes/tests/__init__.py000066400000000000000000000021301301115270200226500ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import os from ...cmd import Runner from ..base import AnnexExchangeProtocol def _get_custom_runner(d): """A helper returning a Runner to be used in the tests for custom special remotes """ # We could just propagate current environ I guess to versatile our testing env = os.environ.copy() env.update({'DATALAD_LOG_TARGET': d + '_custom.log'}) if os.environ.get('DATALAD_LOG_LEVEL'): env['DATALAD_LOG_LEVEL'] = os.environ.get('DATALAD_LOG_LEVEL') if os.environ.get('DATALAD_TESTS_PROTOCOLREMOTE'): # TODO config tests.customremotes.protocol protocol = AnnexExchangeProtocol(d, 'archive') else: protocol = None return Runner(cwd=d, env=env, protocol=protocol) datalad-0.4.1/datalad/customremotes/tests/test_archives.py000066400000000000000000000174041301115270200237660ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for customremotes archives providing dl+archive URLs handling""" from ..archives import ArchiveAnnexCustomRemote from ..base import AnnexExchangeProtocol from ...support.annexrepo import AnnexRepo from ...consts import ARCHIVES_SPECIAL_REMOTE from ...tests.utils import * from ...cmd import Runner, GitRunner from ...utils import _path_ from . import _get_custom_runner # both files will have the same content # fn_inarchive_obscure = 'test.dat' # fn_extracted_obscure = 'test2.dat' fn_inarchive_obscure = get_most_obscure_supported_name() fn_archive_obscure = fn_inarchive_obscure.replace('a', 'b') + '.tar.gz' fn_extracted_obscure = fn_inarchive_obscure.replace('a', 'z') #import line_profiler #prof = line_profiler.LineProfiler() # TODO: with_tree ATM for archives creates this nested top directory # matching archive name, so it will be a/d/test.dat ... we don't want that probably @with_tree( tree=(('a.tar.gz', {'d': {fn_inarchive_obscure: '123'}}), ('simple.txt', '123'), (fn_archive_obscure, (('d', ((fn_inarchive_obscure, '123'),)),)), (fn_extracted_obscure, '123'))) @with_tempfile() def check_basic_scenario(fn_archive, fn_extracted, direct, d, d2): annex = AnnexRepo(d, runner=_get_custom_runner(d), direct=direct) annex.init_remote( ARCHIVES_SPECIAL_REMOTE, ['encryption=none', 'type=external', 'externaltype=%s' % ARCHIVES_SPECIAL_REMOTE, 'autoenable=true' ]) # We want two maximally obscure names, which are also different assert(fn_extracted != fn_inarchive_obscure) annex.add(fn_archive, commit=True, msg="Added tarball") annex.add(fn_extracted, commit=True, msg="Added the load file") # Operations with archive remote URL annexcr = ArchiveAnnexCustomRemote(path=d) # few quick tests for get_file_url eq_(annexcr.get_file_url(archive_key="xyz", file="a.dat"), "dl+archive:xyz#path=a.dat") eq_(annexcr.get_file_url(archive_key="xyz", file="a.dat", size=999), "dl+archive:xyz#path=a.dat&size=999") # see https://github.com/datalad/datalad/issues/441#issuecomment-223376906 # old style eq_(annexcr._parse_url("dl+archive:xyz/a.dat#size=999"), ("xyz", "a.dat", {'size': 999})) eq_(annexcr._parse_url("dl+archive:xyz/a.dat"), ("xyz", "a.dat", {})) # old format without size # new style eq_(annexcr._parse_url("dl+archive:xyz#path=a.dat&size=999"), ("xyz", "a.dat", {'size': 999})) eq_(annexcr._parse_url("dl+archive:xyz#path=a.dat"), ("xyz", "a.dat", {})) # old format without size file_url = annexcr.get_file_url( archive_file=fn_archive, file=fn_archive.replace('.tar.gz', '') + '/d/'+fn_inarchive_obscure) annex.add_url_to_file(fn_extracted, file_url, ['--relaxed']) annex.drop(fn_extracted) list_of_remotes = annex.whereis(fn_extracted, output='descriptions') in_('[%s]' % ARCHIVES_SPECIAL_REMOTE, list_of_remotes) assert_false(annex.file_has_content(fn_extracted)) annex.get(fn_extracted) assert_true(annex.file_has_content(fn_extracted)) annex.rm_url(fn_extracted, file_url) with swallow_logs(new_level=logging.WARN) as cm: assert_raises(RuntimeError, annex.drop, fn_extracted) in_("git-annex: drop: 1 failed", cm.out) annex.add_url_to_file(fn_extracted, file_url) annex.drop(fn_extracted) annex.get(fn_extracted) annex.drop(fn_extracted) # so we don't get from this one next # Let's create a clone and verify chain of getting file through the tarball cloned_annex = AnnexRepo(d2, d, runner=_get_custom_runner(d2), direct=direct) # we still need to enable manually atm that special remote for archives # cloned_annex.enable_remote('annexed-archives') assert_false(cloned_annex.file_has_content(fn_archive)) assert_false(cloned_annex.file_has_content(fn_extracted)) cloned_annex.get(fn_extracted) assert_true(cloned_annex.file_has_content(fn_extracted)) # as a result it would also fetch tarball assert_true(cloned_annex.file_has_content(fn_archive)) # Check if protocol was collected if os.environ.get('DATALAD_TESTS_PROTOCOLREMOTE'): assert_is_instance(annex.cmd_call_wrapper.protocol, AnnexExchangeProtocol) protocol_file = _path_(annex.path, '.git/bin/git-annex-remote-datalad-archive') ok_file_has_content(protocol_file, "VERSION 1", re_=True, match=False) ok_file_has_content(protocol_file, "GETAVAILABILITY", re_=True, match=False) ok_file_has_content(protocol_file, "#!/bin/bash", re_=True, match=False) else: assert_false(isinstance(annex.cmd_call_wrapper.protocol, AnnexExchangeProtocol)) # verify that we can drop if original archive gets dropped but available online: # -- done as part of the test_add_archive_content.py # verify that we can't drop a file if archive key was dropped and online archive was removed or changed size! ;) @with_tree( tree={'a.tar.gz': {'d': {fn_inarchive_obscure: '123'}}} ) def test_annex_get_from_subdir(topdir): from datalad.api import add_archive_content annex = AnnexRepo(topdir, init=True) annex.add('a.tar.gz', commit=True) add_archive_content('a.tar.gz', annex=annex, delete=True) fpath = opj(topdir, 'a', 'd', fn_inarchive_obscure) with chpwd(opj(topdir, 'a', 'd')): runner = Runner() runner(['git', 'annex', 'drop', fn_inarchive_obscure]) # run git annex drop assert_false(annex.file_has_content(fpath)) # and verify if file deleted from directory runner(['git', 'annex', 'get', fn_inarchive_obscure]) # run git annex get assert_true(annex.file_has_content(fpath)) # and verify if file got into directory def test_get_git_environ_adjusted(): gitrunner = GitRunner() env = {"GIT_DIR": "../../.git", "GIT_WORK_TREE": "../../", "TEST_VAR": "Exists"} # test conversion of relevant env vars from relative_path to correct absolute_path adj_env = gitrunner.get_git_environ_adjusted(env) assert_equal(adj_env["GIT_DIR"], abspath(env["GIT_DIR"])) assert_equal(adj_env["GIT_WORK_TREE"], abspath(env["GIT_WORK_TREE"])) # test if other environment variables passed to function returned unaltered assert_equal(adj_env["TEST_VAR"], env["TEST_VAR"]) # test import of sys_env if no environment passed to function sys_env = gitrunner.get_git_environ_adjusted() assert_equal(sys_env["PWD"], os.environ.get("PWD")) def test_basic_scenario(): yield check_basic_scenario, 'a.tar.gz', 'simple.txt', False if not on_windows: yield check_basic_scenario, 'a.tar.gz', 'simple.txt', True #yield check_basic_scenario, 'a.tar.gz', fn_extracted_obscure, False #yield check_basic_scenario, fn_archive_obscure, 'simple.txt', False yield check_basic_scenario, fn_archive_obscure, fn_extracted_obscure, False def test_no_rdflib_loaded(): # rely on rdflib polluting stdout to see that it is not loaded whenever we load this remote # since that adds 300ms delay for no immediate use from ...cmd import Runner runner = Runner() with swallow_outputs() as cmo: runner.run([sys.executable, '-c', 'import datalad.customremotes.archives, sys; print([k for k in sys.modules if k.startswith("rdflib")])'], log_stdout=False, log_stderr=False) # print cmo.out assert_not_in("rdflib", cmo.out) assert_not_in("rdflib", cmo.err) datalad-0.4.1/datalad/customremotes/tests/test_datalad.py000066400000000000000000000055741301115270200235610ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for the universal datalad's annex customremote""" from ...support.annexrepo import AnnexRepo from ...consts import DATALAD_SPECIAL_REMOTE from ...tests.utils import * from . import _get_custom_runner from ...support.exceptions import CommandError from ...downloaders.tests.utils import get_test_providers @with_tempfile() @skip_if_no_network def check_basic_scenario(direct, url, d): annex = AnnexRepo(d, runner=_get_custom_runner(d), direct=direct) annex.init_remote( DATALAD_SPECIAL_REMOTE, ['encryption=none', 'type=external', 'externaltype=%s' % DATALAD_SPECIAL_REMOTE, 'autoenable=true']) # TODO skip if no boto or no credentials get_test_providers(url) # so to skip if unknown creds # Let's try to add some file which we should have access to with swallow_outputs() as cmo: annex.add_urls([url]) annex.commit("committing") whereis1 = annex.whereis('3versions_allversioned.txt', output='full') eq_(len(whereis1), 2) # here and datalad annex.drop('3versions_allversioned.txt') if PY2: pass # stopped appearing within the test TODO #assert_in('100%', cmo.err) # we do provide our progress indicator else: pass # TODO: not sure what happened but started to fail for me on my laptop under tox whereis2 = annex.whereis('3versions_allversioned.txt', output='full') eq_(len(whereis2), 1) # datalad # if we provide some bogus address which we can't access, we shouldn't pollute output with swallow_outputs() as cmo, swallow_logs() as cml: with assert_raises(CommandError) as cme: annex.add_urls([url + '_bogus']) # assert_equal(cml.out, '') err, out = cmo.err, cmo.out assert_equal(out, '') assert_in('addurl: 1 failed', err) # and there should be nothing more # unfortunately with_tree etc decorators aren't generators friendly thus # this little adapters to test both on local and s3 urls @with_tree(tree={'3versions-allversioned.txt': "somefile"}) @serve_path_via_http def check_basic_scenario_local_url(direct, p, local_url): check_basic_scenario(direct, "%s3versions-allversioned.txt" % local_url) def check_basic_scenario_s3(direct): check_basic_scenario(direct, 's3://datalad-test0-versioned/3versions-allversioned.txt') def test_basic_scenario(): for test in check_basic_scenario_local_url, :#check_basic_scenario_s3: yield test, False if not on_windows: yield test, True datalad-0.4.1/datalad/distribution/000077500000000000000000000000001301115270200172075ustar00rootroot00000000000000datalad-0.4.1/datalad/distribution/__init__.py000066400000000000000000000007621301115270200213250ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Datalad distribution components: dataset, metadata handling, ... """ __docformat__ = 'restructuredtext' datalad-0.4.1/datalad/distribution/add.py000066400000000000000000000422041301115270200203130ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for adding dataset components """ import logging from os import curdir from os.path import isdir from os.path import join as opj from os.path import relpath from collections import defaultdict from datalad.interface.base import Interface from datalad.interface.common_opts import recursion_flag from datalad.interface.common_opts import recursion_limit from datalad.interface.common_opts import nosave_opt from datalad.interface.common_opts import git_opts from datalad.interface.common_opts import annex_opts from datalad.interface.common_opts import annex_add_opts from datalad.interface.common_opts import if_dirty_opt from datalad.interface.common_opts import jobs_opt from datalad.interface.utils import handle_dirty_dataset from datalad.interface.save import Save from datalad.support.constraints import EnsureStr from datalad.support.constraints import EnsureNone from datalad.support.param import Parameter from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.network import is_datalad_compat_ri from datalad.utils import assure_list from datalad.utils import with_pathsep as _with_sep from .dataset import EnsureDataset from .dataset import datasetmethod from .dataset import Dataset from .dataset import resolve_path from .dataset import require_dataset from .install import _get_git_url_from_source __docformat__ = 'restructuredtext' lgr = logging.getLogger('datalad.distribution.add') class Add(Interface): """Add files/directories to an existing dataset. Typically, files and directories to be added to a dataset would be placed into a directory of a dataset, and subsequently this command can be used to register this new content with the dataset. With recursion enabled, files will be added to their respective subdatasets as well. Alternatively, a source location can be given to indicate where to obtain data from. If no `path` argument is provided in this case, the content will be obtained from the source location and a default local name, derived from the source location will be generated. Alternatively, an explicit `path` can be given to override the default. If more than one `path` argument and a source location are provided, the `path` arguments will be sequentially used to complete the source URL/path (be means of concatenation), and an attempt is made to obtain data from those locations. || REFLOW >> By default all files are added to the dataset's :term:`annex`, i.e. only their content identity and availability information is tracked with Git. This results in lightweight datasets. If desired, the [PY: `to_git` PY][CMD: --to-git CMD] flag can be used to tell datalad to inject files directly into Git. While this is not recommended for binary data or large files, it can be used for source code and meta-data to be able to benefit from Git's track and merge capabilities. Files checked directly into Git are always and unconditionally available immediately after installation of a dataset. << REFLOW || .. note:: Power-user info: This command uses :command:`git annex add`, :command:`git annex addurl`, or :command:`git add` to incorporate new dataset content. """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), metavar='PATH', doc="""specify the dataset to perform the add operation on. If no dataset is given, an attempt is made to identify the dataset based on the current working directory and/or the `path` given""", constraints=EnsureDataset() | EnsureNone()), path=Parameter( args=("path",), metavar='PATH', doc="""path/name of the component to be added. The component must either exist on the filesystem already, or a `source` has to be provided.""", nargs="*", constraints=EnsureStr() | EnsureNone()), source=Parameter( args=("-s", "--source",), metavar='URL/PATH', doc="url or local path of the to be added component's source", action="append", constraints=EnsureStr() | EnsureNone()), to_git=Parameter( args=("--to-git",), action='store_true', doc="""flag whether to add data directly to Git, instead of tracking data identity only. Usually this is not desired, as it inflates dataset sizes and impacts flexibility of data transport"""), recursive=recursion_flag, recursion_limit=recursion_limit, save=nosave_opt, if_dirty=if_dirty_opt, git_opts=git_opts, annex_opts=annex_opts, annex_add_opts=annex_add_opts, jobs=jobs_opt ) @staticmethod @datasetmethod(name='add') def __call__( path=None, source=None, dataset=None, to_git=False, save=True, recursive=False, recursion_limit=None, if_dirty='ignore', git_opts=None, annex_opts=None, annex_add_opts=None, jobs=None): # parameter constraints: if not path and not source: raise InsufficientArgumentsError("insufficient information for " "adding: requires at least a path " "or a source.") # When called from cmdline `path` and `source` will be a list even if # there is only one item. # Make sure we deal with the same when called via python API: # always yields list; empty if None path = assure_list(path) source = assure_list(source) # TODO: Q: are the list operations in the following 3 blocks (resolving # paths, sources and datasets) guaranteed to be stable # regarding order? # resolve path(s): # TODO: RF: resolve_path => datalad.utils => more general (repos => normalize paths) resolved_paths = [resolve_path(p, dataset) for p in path] # must come after resolve_path()!! # resolve dataset: dataset = require_dataset(dataset, check_installed=True, purpose='adding') handle_dirty_dataset(dataset, if_dirty) # resolve source(s): resolved_sources = [] for s in source: if not is_datalad_compat_ri(s): raise ValueError("invalid source parameter: %s" % s) resolved_sources.append(_get_git_url_from_source(s)) # find (sub-)datasets to add things to (and fail on invalid paths): if recursive: # 1. Find the (sub-)datasets containing the given path(s): # Note, that `get_containing_subdataset` raises if `p` is # outside `dataset`, but it returns `dataset`, if `p` is inside # a subdataset not included by `recursion_limit`. In the latter # case, the git calls will fail instead. # We could check for this right here and fail early, but this # would lead to the need to discover the entire hierarchy no # matter if actually required. resolved_datasets = [dataset.get_containing_subdataset( p, recursion_limit=recursion_limit) for p in resolved_paths] # 2. Find implicit subdatasets to call add on: # If there are directories in resolved_paths (Note, # that this includes '.' and '..'), check for subdatasets # beneath them. These should be called recursively with '.'. # Therefore add the subdatasets to resolved_datasets and # corresponding '.' to resolved_paths, in order to generate the # correct call. for p in resolved_paths: if isdir(p): for subds_path in \ dataset.get_subdatasets(absolute=True, recursive=True, recursion_limit=recursion_limit): if subds_path.startswith(_with_sep(p)): resolved_datasets.append(Dataset(subds_path)) resolved_paths.append(curdir) else: # if not recursive, try to add everything to dataset itself: resolved_datasets = [dataset for i in range(len(resolved_paths))] # we need a resolved dataset per path: assert len(resolved_paths) == len(resolved_datasets) # sort parameters for actual git/git-annex calls: # (dataset, path, source) from six.moves import zip_longest param_tuples = list(zip_longest(resolved_datasets, resolved_paths, resolved_sources)) # possible None-datasets in `param_tuples` were filled in by zip_longest # and need to be replaced by `dataset`: param_tuples = [(d if d is not None else dataset, p, s) for d, p, s in param_tuples] calls = {d.path: { # list of paths to 'git-add': 'g_add': [], # list of paths to 'git-annex-add': 'a_add': [], # list of sources to 'git-annex-addurl': 'addurl_s': [], # list of (path, source) to # 'git-annex-addurl --file': 'addurl_f': [] } for d in [i for i, p, s in param_tuples]} for ds, p, s in param_tuples: # it should not happen, that `path` as well as `source` are None: assert p or s if not s: # we have a path only # Do not try to add to annex whenever there is no annex if to_git or not isinstance(ds.repo, AnnexRepo): calls[ds.path]['g_add'].append(p) else: calls[ds.path]['a_add'].append(p) elif not p: # we have a source only if to_git: raise NotImplementedError("Can't add a remote source " "directly to git.") calls[ds.path]['addurl_s'].append(s) else: # we have a path and a source if to_git: raise NotImplementedError("Can't add a remote source " "directly to git.") calls[ds.path]['addurl_f'].append((p, s)) # now do the actual add operations: # TODO: implement git/git-annex/git-annex-add options datasets_return_values = defaultdict(list) for dspath in calls: ds = Dataset(dspath) return_values = datasets_return_values[dspath] lgr.info("Processing dataset %s ..." % ds) # check every (sub-)dataset for annex once, since we can't add or # addurl anything, if there is no annex: # TODO: Q: Alternatively, just call git-annex-init if there's no # annex yet and we have an annex-add/annex-addurl request? _is_annex = isinstance(ds.repo, AnnexRepo) if calls[ds.path]['g_add']: lgr.debug("Adding %s to git", calls[dspath]['g_add']) added = ds.repo.add(calls[dspath]['g_add'], git=True, git_options=git_opts) return_values.extend(added) if calls[ds.path]['a_add']: if _is_annex: lgr.debug("Adding %s to annex", calls[dspath]['a_add']) return_values.extend( ds.repo.add(calls[dspath]['a_add'], git=False, jobs=jobs, git_options=git_opts, annex_options=annex_opts, options=annex_add_opts ) ) else: lgr.debug("{0} is no annex. Skip 'annex-add' for " "files {1}".format(ds, calls[dspath]['a_add'])) return_values.extend( [{'file': f, 'success': False, 'note': "no annex at %s" % ds.path} for f in calls[dspath]['a_add']] ) # TODO: AnnexRepo.add_urls' return value doesn't contain the created # file name but the url if calls[ds.path]['addurl_s']: if _is_annex: lgr.debug("Adding urls %s to annex", calls[dspath]['addurl_s']) return_values.extend( ds.repo.add_urls(calls[ds.path]['addurl_s'], options=annex_add_opts, # TODO: extra parameter for addurl? git_options=git_opts, annex_options=annex_opts, jobs=jobs, ) ) else: lgr.debug("{0} is no annex. Skip 'annex-addurl' for " "files {1}".format(ds, calls[dspath]['addurl_s'])) return_values.extend( [{'file': f, 'success': False, 'note': "no annex at %s" % ds.path} for f in calls[dspath]['addurl_s']] ) if calls[ds.path]['addurl_f']: if _is_annex: for f, u in calls[ds.path]['addurl_f']: lgr.debug("Adding urls %s to files in annex", calls[dspath]['addurl_f']) return_values.append( ds.repo.add_url_to_file(f, u, options=annex_add_opts, # TODO: see above git_options=git_opts, annex_options=annex_opts, batch=True)) else: lgr.debug("{0} is no annex. Skip 'annex-addurl' for " "files {1}".format(ds, calls[dspath]['addurl_f'])) return_values.extend( [{'file': f, 'success': False, 'note': "no annex at %s" % ds.path} for f in calls[dspath]['addurl_f']] ) return_values = None # to avoid mis-use # XXX or we could return entire datasets_return_values, could be useful # that way. But then should be unified with the rest of commands, e.g. # get etc return_values_flat = [] for dspath, return_values in datasets_return_values.items(): if save and len(return_values): # we got something added -> save # everything we care about at this point should be staged already Save.__call__( message='[DATALAD] added content', dataset=ds, auto_add_changes=False, recursive=False) # TODO: you feels that this is some common logic we already have somewhere dsrelpath = relpath(dspath, dataset.path) if dsrelpath != curdir: # we need ot adjust 'file' entry in each record for return_value in return_values: if 'file' in return_value: return_value['file'] = opj(dsrelpath, return_value['file']) return_values_flat.append(return_value) else: return_values_flat.extend(return_values) return return_values_flat @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui from os import linesep if res is None: res = [] if not isinstance(res, list): res = [res] if not len(res): ui.message("Nothing was added") return msg = linesep.join([ "{suc} {path}".format( suc="Added" if item.get('success', False) else "Failed to add. (%s)" % item.get('note', 'unknown reason'), path=item.get('file')) for item in res]) ui.message(msg) datalad-0.4.1/datalad/distribution/add_sibling.py000066400000000000000000000301141301115270200220170ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for adding a sibling to a dataset """ __docformat__ = 'restructuredtext' import logging from collections import OrderedDict from os.path import join as opj, basename from datalad.utils import assure_list from datalad.dochelpers import exc_str from datalad.support.param import Parameter from datalad.support.constraints import EnsureStr, EnsureNone, EnsureBool from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from datalad.support.network import RI from datalad.support.network import URL from ..interface.base import Interface from datalad.interface.common_opts import recursion_flag from datalad.interface.common_opts import as_common_datasrc from datalad.interface.common_opts import publish_depends from datalad.distribution.dataset import EnsureDataset, Dataset, \ datasetmethod, require_dataset from datalad.support.exceptions import CommandError lgr = logging.getLogger('datalad.distribution.add_publication_target') class AddSibling(Interface): """Add a sibling to a dataset. """ _params_ = dict( # TODO: Somehow the replacement of '_' and '-' is buggy on # positional arguments dataset=Parameter( args=("--dataset", "-d",), doc="""specify the dataset to add the sibling to. If no dataset is given, an attempt is made to identify the dataset based on the current working directory""", constraints=EnsureDataset() | EnsureNone()), name=Parameter( args=('name',), doc="""name of the sibling to be added. If RECURSIVE is set, the same name will be used to address the subdatasets' siblings""", constraints=EnsureStr() | EnsureNone()), url=Parameter( args=('url',), doc="""the URL of or path to the dataset sibling named by `name`. If you want to recursively add siblings, it is expected, that you pass a template for building the URLs of the siblings of all (sub)datasets by using placeholders.\n List of currently available placeholders:\n %%NAME\tthe name of the dataset, where slashes are replaced by dashes.\nThis option is ignored if there is already a configured sibling dataset under the name given by `name`""", constraints=EnsureStr() | EnsureNone(), nargs="?"), pushurl=Parameter( args=('--pushurl',), doc="""in case the `url` cannot be used to publish to the dataset sibling, this option specifies a URL to be used instead.\nIf no `url` is given, `pushurl` serves as `url` as well. This option is ignored if there is already a configured sibling dataset under the name given by `name`""", constraints=EnsureStr() | EnsureNone()), recursive=recursion_flag, fetch=Parameter( args=("--fetch",), action="store_true", doc="""fetch the sibling after adding"""), force=Parameter( args=("--force", "-f",), action="store_true", doc="""if sibling `name` exists already, force to (re-)configure its URLs""",), as_common_datasrc=as_common_datasrc, publish_depends=publish_depends, ) @staticmethod @datasetmethod(name='add_sibling') def __call__(name=None, url=None, dataset=None, pushurl=None, recursive=False, fetch=False, force=False, as_common_datasrc=None, publish_depends=None): # TODO: Detect malformed URL and fail? # XXX possibly fail if fetch is False and as_common_datasrc # not yet sure if that is an error if name is None or (url is None and pushurl is None): raise ValueError("""insufficient information to add a sibling (needs at least a dataset, a name and an URL).""") if url is None: url = pushurl ds = require_dataset(dataset, check_installed=True, purpose='sibling addition') assert(ds.repo is not None) ds_basename = basename(ds.path) repos = OrderedDict() repos[ds_basename] = {'repo': ds.repo} if recursive: for subds_name in ds.get_subdatasets(recursive=True): subds_path = opj(ds.path, subds_name) subds = Dataset(subds_path) lgr.debug("Adding sub-dataset %s for adding a sibling", subds_path) if not subds.is_installed(): lgr.info("Skipping adding sibling for %s since it " "is not installed", subds) continue repos[ds_basename + '/' + subds_name] = { # repos[subds_name] = { 'repo': GitRepo(subds_path, create=False) } # Note: This is copied from create_sibling # as it is the same logic as for its target_dir. # TODO: centralize and generalize template symbol handling # TODO: Check pushurl for template symbols too. Probably raise if only # one of them uses such symbols replicate_local_structure = "%NAME" not in url for repo_name in repos: repo = repos[repo_name] if not replicate_local_structure: repo['url'] = url.replace("%NAME", repo_name.replace("/", "-")) if pushurl: repo['pushurl'] = pushurl.replace("%NAME", repo_name.replace("/", "-")) else: repo['url'] = url if pushurl: repo['pushurl'] = pushurl if repo_name != ds_basename: repo['url'] = _urljoin(repo['url'], repo_name[len(ds_basename) + 1:]) if pushurl: repo['pushurl'] = _urljoin(repo['pushurl'], repo_name[len(ds_basename) + 1:]) # define config var name for potential publication dependencies depvar = 'remote.{}.datalad-publish-depends'.format(name) # collect existing remotes: already_existing = list() conflicting = list() for repo_name in repos: repoinfo = repos[repo_name] repo = repoinfo['repo'] if name in repo.get_remotes(): already_existing.append(repo_name) lgr.debug("Remote '{0}' already exists " "in '{1}'.""".format(name, repo_name)) existing_url = repo.get_remote_url(name) existing_pushurl = \ repo.get_remote_url(name, push=True) if (not existing_url or repoinfo['url'].rstrip('/') != existing_url.rstrip('/')) \ or (pushurl and existing_pushurl and repoinfo['pushurl'].rstrip('/') != existing_pushurl.rstrip('/')) \ or (pushurl and not existing_pushurl) \ or (publish_depends and set(ds.config.get(depvar, [])) != set(publish_depends)): conflicting.append(repo_name) if not force and conflicting: raise RuntimeError("Sibling '{0}' already exists with conflicting" " URL for {1} dataset(s). {2}".format( name, len(conflicting), conflicting)) successfully_added = list() for repo_name in repos: repoinfo = repos[repo_name] repo = repoinfo['repo'] if repo_name in already_existing: if repo_name not in conflicting: lgr.debug("Skipping {0}. Nothing to do.".format(repo_name)) continue # rewrite url repo.set_remote_url(name, repoinfo['url']) else: # add the remote repo.add_remote(name, repoinfo['url']) if pushurl: repo.set_remote_url(name, repoinfo['pushurl'], push=True) if fetch: # fetch the remote so we are up to date lgr.debug("Fetching sibling %s of %s", name, repo_name) repo.fetch(name) if publish_depends: if depvar in ds.config: # config vars are incremental, so make sure we start from # scratch ds.config.unset(depvar, where='local', reload=False) for d in assure_list(publish_depends): lgr.info( 'Configure additional publication dependency on "%s"', d) ds.config.add(depvar, d, where='local', reload=False) ds.config.reload() assert isinstance(repo, GitRepo) # just against silly code if isinstance(repo, AnnexRepo): # we need to check if added sibling an annex, and try to enable it # another part of the fix for #463 and #432 try: if not ds.config.obtain( 'remote.{}.annex-ignore'.format(name), default=False, valtype=EnsureBool(), store=False): repo.enable_remote(name) except CommandError as exc: lgr.info("Failed to enable annex remote %s, " "could be a pure git" % name) lgr.debug("Exception was: %s" % exc_str(exc)) if as_common_datasrc: ri = RI(repoinfo['url']) if isinstance(ri, URL) and ri.scheme in ('http', 'https'): # XXX what if there is already a special remote # of this name? Above check for remotes ignores special # remotes. we need to `git annex dead REMOTE` on reconfigure # before we can init a new one # XXX except it is not enough # make special remote of type=git (see #335) repo._run_annex_command( 'initremote', annex_options=[ as_common_datasrc, 'type=git', 'location={}'.format(repoinfo['url']), 'autoenable=true']) else: lgr.warning( 'Not configuring "%s" as a common data source, ' 'URL protocol is not http or https', name) successfully_added.append(repo_name) return successfully_added @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui if res is None: res = [] if not isinstance(res, list): res = [res] if not len(res): ui.message("No sibling was added") return items= '\n'.join(map(str, res)) msg = "Added sibling to {ds}:\n{items}".format( ds='{} datasets'.format(len(res)) if len(res) > 1 else 'one dataset', items=items) ui.message(msg) # TODO: RF nicely, test, make clear how different from urljoin etc def _urljoin(base, url): return base + url if (base.endswith('/') or url.startswith('/')) else base + '/' + url datalad-0.4.1/datalad/distribution/create.py000066400000000000000000000277211301115270200210350ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for dataset creation """ import logging import uuid from os import listdir from os.path import isdir, realpath, relpath from datalad.interface.base import Interface from datalad.interface.save import Save from datalad.interface.common_opts import git_opts from datalad.interface.common_opts import annex_opts from datalad.interface.common_opts import annex_init_opts from datalad.interface.common_opts import dataset_description from datalad.interface.common_opts import nosave_opt from datalad.interface.common_opts import if_dirty_opt from datalad.interface.common_opts import shared_access_opt from datalad.interface.utils import handle_dirty_dataset from datalad.support.constraints import EnsureStr from datalad.support.constraints import EnsureNone from datalad.support.constraints import EnsureDType from datalad.support.param import Parameter from datalad.support.annexrepo import AnnexRepo from datalad.support.gitrepo import GitRepo from datalad.utils import getpwd from datalad.utils import with_pathsep from .dataset import Dataset from .dataset import datasetmethod from .dataset import EnsureDataset from .dataset import resolve_path __docformat__ = 'restructuredtext' lgr = logging.getLogger('datalad.distribution.create') class Create(Interface): """Create a new dataset from scratch. This command initializes a new :term:`dataset` at a given location, or the current directory. The new dataset can optionally be registered in an existing :term:`superdataset` (the new dataset's path needs to be located within the superdataset for that, and the superdataset needs to be given explicitly). It is recommended to provide a brief description to label the dataset's nature *and* location, e.g. "Michael's music on black laptop". This helps humans to identify data locations in distributed scenarios. By default an identifier comprised of user and machine name, plus path will be generated. Plain Git repositories can be created via the [PY: `no_annex` PY][CMD: --no-annex CMD] flag. However, the result will not be a full dataset, and, consequently, not all features are supported (e.g. a description). || REFLOW >> To create a local version of a remote dataset use the :func:`~datalad.api.install` command instead. << REFLOW || .. note:: Power-user info: This command uses :command:`git init`, and :command:`git annex init` to prepare the new dataset. Registering to a superdataset is performed via a :command:`git submodule add` operation in the discovered superdataset. """ _params_ = dict( path=Parameter( args=("path",), metavar='PATH', doc="""path where the dataset shall be created, directories will be created as necessary. If no location is provided, a dataset will be created in the current working directory. Either way the command will error if the target directory is not empty. Use `force` to create a dataset in a non-empty directory.""", nargs='?', # put dataset 2nd to avoid useless conversion constraints=EnsureStr() | EnsureDataset() | EnsureNone()), dataset=Parameter( args=("-d", "--dataset"), metavar='PATH', doc="""specify the dataset to perform the create operation on. If a dataset is give, a new subdataset will be created in it.""", constraints=EnsureDataset() | EnsureNone()), force=Parameter( args=("-f", "--force",), doc="""enforce creation of a dataset in a non-empty directory""", action='store_true'), description=dataset_description, no_annex=Parameter( args=("--no-annex",), doc="""if set, a plain Git repository will be created without any annex""", action='store_true'), save=nosave_opt, if_dirty=if_dirty_opt, annex_version=Parameter( args=("--annex-version",), doc="""select a particular annex repository version. The list of supported versions depends on the available git-annex version. This should be left untouched, unless you know what you are doing""", constraints=EnsureDType(int) | EnsureNone()), annex_backend=Parameter( args=("--annex-backend",), constraints=EnsureStr() | EnsureNone(), # not listing choices here on purpose to avoid future bugs doc="""set default hashing backend used by the new dataset. For a list of supported backends see the git-annex documentation. The default is optimized for maximum compatibility of datasets across platforms (especially those with limited path lengths)""", nargs=1), native_metadata_type=Parameter( args=('--native-metadata-type',), metavar='LABEL', action='append', constraints=EnsureStr() | EnsureNone(), doc="""Metadata type label. Must match the name of the respective parser implementation in Datalad (e.g. "bids").[CMD: This option can be given multiple times CMD]"""), shared_access=shared_access_opt, git_opts=git_opts, annex_opts=annex_opts, annex_init_opts=annex_init_opts, ) @staticmethod @datasetmethod(name='create') def __call__( path=None, force=False, description=None, dataset=None, no_annex=False, save=True, annex_version=None, annex_backend='MD5E', native_metadata_type=None, if_dirty='save-before', shared_access=None, git_opts=None, annex_opts=None, annex_init_opts=None): # two major cases # 1. we got a `dataset` -> we either want to create it (path is None), # or another dataset in it (path is not None) # 2. we got no dataset -> we want to create a fresh dataset at the # desired location, either at `path` or PWD # sanity check first if git_opts: lgr.warning( "`git_opts` argument is presently ignored, please complain!") if no_annex: if description: raise ValueError("Incompatible arguments: cannot specify " "description for annex repo and declaring " "no annex repo.") if annex_opts: raise ValueError("Incompatible arguments: cannot specify " "options for annex and declaring no " "annex repo.") if annex_init_opts: raise ValueError("Incompatible arguments: cannot specify " "options for annex init and declaring no " "annex repo.") if not isinstance(force, bool): raise ValueError("force should be bool, got %r. Did you mean to provide a 'path'?" % force) # straight from input arg, no messing around before this if path is None: if dataset is None: # nothing given explicity, assume create fresh right here path = getpwd() else: # no path, but dataset -> create that dataset path = dataset.path else: # resolve the path against a potential dataset path = resolve_path(path, ds=dataset) # we know that we need to create a dataset at `path` assert(path is not None) if git_opts is None: git_opts = {} if shared_access: # configure `git --shared` value git_opts['shared'] = shared_access # check for sane subdataset path real_targetpath = with_pathsep(realpath(path)) # realpath OK if dataset is not None: # make sure we get to an expected state if dataset.is_installed(): handle_dirty_dataset(dataset, if_dirty) if not real_targetpath.startswith( # realpath OK with_pathsep(realpath(dataset.path))): # realpath OK raise ValueError("path {} outside {}".format(path, dataset)) # important to use the given Dataset object to avoid spurious ID # changes with not-yet-materialized Datasets tbds = dataset if dataset is not None and dataset.path == path else Dataset(path) # don't create in non-empty directory without `force`: if isdir(tbds.path) and listdir(tbds.path) != [] and not force: raise ValueError("Cannot create dataset in directory %s " "(not empty). Use option 'force' in order to " "ignore this and enforce creation." % tbds.path) if no_annex: lgr.info("Creating a new git repo at %s", tbds.path) GitRepo( tbds.path, url=None, create=True, git_opts=git_opts) else: # always come with annex when created from scratch lgr.info("Creating a new annex repo at %s", tbds.path) AnnexRepo( tbds.path, url=None, create=True, backend=annex_backend, version=annex_version, description=description, git_opts=git_opts, annex_opts=annex_opts, annex_init_opts=annex_init_opts) if native_metadata_type is not None: if not isinstance(native_metadata_type, list): native_metadata_type = [native_metadata_type] for nt in native_metadata_type: tbds.config.add('datalad.metadata.nativetype', nt) # record an ID for this repo for the afterlife # to be able to track siblings and children id_var = 'datalad.dataset.id' if id_var in tbds.config: # make sure we reset this variable completely, in case of a re-create tbds.config.unset(id_var, where='dataset') tbds.config.add( id_var, tbds.id if tbds.id is not None else uuid.uuid1().urn.split(':')[-1], where='dataset') # save everthing tbds.repo.add('.datalad', git=True) if save: Save.__call__( message='[DATALAD] new dataset', dataset=tbds, auto_add_changes=False, recursive=False) if dataset is not None and dataset.path != tbds.path: # we created a dataset in another dataset # -> make submodule from datalad.distribution.utils import _install_subds_inplace subdsrelpath = relpath(realpath(tbds.path), realpath(dataset.path)) # realpath OK _install_subds_inplace(ds=dataset, path=tbds.path, relativepath=subdsrelpath) # this will have staged the changes in the superdataset already if save: Save.__call__( message='[DATALAD] added subdataset', dataset=dataset, auto_add_changes=False, recursive=False) return tbds @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui if res is None: ui.message("Nothing was created") elif isinstance(res, Dataset): ui.message("Created dataset at %s." % res.path) datalad-0.4.1/datalad/distribution/create_sibling.py000066400000000000000000000522711301115270200225420ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for creation of publication target via SSH """ __docformat__ = 'restructuredtext' import logging import datalad from os.path import join as opj, abspath, basename, relpath, normpath, dirname from distutils.version import LooseVersion from glob import glob from datalad.support.network import RI, URL, SSHRI from datalad.support.gitrepo import GitRepo from datalad.support.param import Parameter from datalad.dochelpers import exc_str from datalad.support.constraints import EnsureStr, EnsureNone, EnsureBool from datalad.support.constraints import EnsureChoice from datalad.support.annexrepo import AnnexRepo from ..interface.base import Interface from datalad.interface.common_opts import recursion_flag from datalad.interface.common_opts import as_common_datasrc from datalad.distribution.dataset import EnsureDataset, Dataset, \ datasetmethod, require_dataset from datalad.cmd import CommandError from datalad.utils import not_supported_on_windows, getpwd from .add_sibling import AddSibling from datalad import ssh_manager from datalad.utils import make_tempfile from datalad.consts import WEB_HTML_DIR, WEB_META_LOG from datalad.consts import TIMESTAMP_FMT from datalad.utils import _path_ lgr = logging.getLogger('datalad.distribution.create_sibling') class CreateSibling(Interface): """Create dataset(s)'s sibling (e.g., on a web server). Those (empty) datasets can then serve as a target for the `publish` command. """ _params_ = dict( # TODO: Figure out, whether (and when) to use `sshurl` as push url dataset=Parameter( args=("--dataset", "-d",), doc="""specify the dataset to create the publication target for. If no dataset is given, an attempt is made to identify the dataset based on the current working directory""", constraints=EnsureDataset() | EnsureNone()), sshurl=Parameter( args=("sshurl",), metavar='SSHURL', doc="""Login information for the target server. This can be given as a URL (ssh://host/path) or SSH-style (user@host:path). Unless overridden, this also serves the future dataset's access URL and path on the server.""", constraints=EnsureStr()), target=Parameter( args=('target',), metavar='TARGETNAME', doc="""sibling name to create for this publication target. If `recursive` is set, the same name will be used to label all the subdatasets' siblings. Note, this is just a convenience option, siblings can also be added at a later point in time. When creation target datasets fails, no siblings are added""", constraints=EnsureStr() | EnsureNone(), nargs="?"), target_dir=Parameter( args=('--target-dir',), metavar='PATH', doc="""path to the directory *on the server* where the dataset shall be created. By default the SSH access URL is used to identify this directory. If a relative path is provided here, it is interpreted as being relative to the user's home directory on the server.\n Additional features are relevant for recursive processing of datasets with subdatasets. By default, the local dataset structure is replicated on the server. However, it is possible to provide a template for generating different target directory names for all (sub)datasets. Templates can contain certain placeholder that are substituted for each (sub)dataset. For example: "/mydirectory/dataset-%%NAME".\nSupported placeholders:\n %%NAME - the name of the datasets, with any slashes replaced by dashes\n""", constraints=EnsureStr() | EnsureNone()), target_url=Parameter( args=('--target-url',), metavar='URL', doc=""""public" access URL of the to-be-created target dataset(s) (default: `sshurl`). Accessibility of this URL determines the access permissions of potential consumers of the dataset. As with `target_dir`, templates (same set of placeholders) are supported. Also, if specified, it is provided as the annex description\n""", constraints=EnsureStr() | EnsureNone()), target_pushurl=Parameter( args=('--target-pushurl',), metavar='URL', doc="""In case the `target_url` cannot be used to publish to the dataset, this option specifies an alternative URL for this purpose. As with `target_url`, templates (same set of placeholders) are supported.\n""", constraints=EnsureStr() | EnsureNone()), recursive=recursion_flag, existing=Parameter( args=("--existing",), constraints=EnsureChoice('skip', 'replace', 'error', 'reconfigure'), metavar='MODE', doc="""action to perform, if target directory exists already. Dataset is skipped if 'skip'. 'replace' forces to (re-)init the dataset, and to (re-)configure the dataset sibling, i.e. its URL(s), in case it already exists. 'reconfigure' updates metadata of the dataset sibling. 'error' causes an exception to be raised.""",), shared=Parameter( args=("--shared",), metavar='false|true|umask|group|all|world|everybody|0xxx', doc="""if given, configures the access permissions on the server for multi-users (this could include access by a webserver!). Possible values for this option are identical to those of `git init --shared` and are described in its documentation.""", constraints=EnsureStr() | EnsureBool()), ui=Parameter( args=("--ui",), metavar='false|true|html_filename', doc="""publish a web interface for the dataset with an optional user-specified name for the html at publication target. defaults to `index.html` at dataset root""", constraints=EnsureBool() | EnsureStr()), as_common_datasrc=as_common_datasrc, ) @staticmethod @datasetmethod(name='create_sibling') def __call__(sshurl, target=None, target_dir=None, target_url=None, target_pushurl=None, dataset=None, recursive=False, existing='error', shared=False, ui=False, as_common_datasrc=None): if sshurl is None: raise ValueError("""insufficient information for target creation (needs at least a dataset and a SSH URL).""") if target is None and (target_url is not None or target_pushurl is not None): raise ValueError("""insufficient information for adding the target as a sibling (needs at least a name)""") # shortcut ds = require_dataset(dataset, check_installed=True, purpose='creating a sibling') assert(ds is not None and sshurl is not None and ds.repo is not None) # determine target parameters: sshri = RI(sshurl) if not isinstance(sshri, SSHRI) \ and not (isinstance(sshri, URL) and sshri.scheme == 'ssh'): raise ValueError("Unsupported SSH URL: '{0}', use ssh://host/path or host:path syntax".format(sshurl)) if target_dir is None: if sshri.path: target_dir = sshri.path else: target_dir = '.' # TODO: centralize and generalize template symbol handling replicate_local_structure = False if "%NAME" not in target_dir: replicate_local_structure = True # collect datasets to use: datasets = dict() datasets[basename(ds.path)] = ds if recursive: for subds in ds.get_subdatasets(recursive=True): sub_path = opj(ds.path, subds) # TODO: when enhancing Dataset/*Repo classes and therefore # adapt to moved code, make proper distinction between name and # path of a submodule, which are technically different. This # probably will become important on windows as well as whenever # we want to allow for moved worktrees. datasets[basename(ds.path) + '/' + subds] = \ Dataset(sub_path) # request ssh connection: not_supported_on_windows("TODO") lgr.info("Connecting ...") ssh = ssh_manager.get_connection(sshurl) ssh.open() # flag to check if at dataset_root at_root = True # loop over all datasets, ordered from top to bottom to make test # below valid (existing directories would cause the machinery to halt) # But we need to run post-update hook in depth-first fashion, so # would only collect first and then run (see gh #790) remote_repos_to_run_hook_for = [] for current_dspath in \ sorted(datasets.keys(), key=lambda x: x.count('/')): current_ds = datasets[current_dspath] if not current_ds.is_installed(): lgr.info("Skipping %s since not installed locally", current_dspath) continue if not replicate_local_structure: path = target_dir.replace("%NAME", current_dspath.replace("/", "-")) else: # TODO: opj depends on local platform, not the remote one. # check how to deal with it. Does windows ssh server accept # posix paths? vice versa? Should planned SSH class provide # tools for this issue? path = normpath(opj(target_dir, relpath(datasets[current_dspath].path, start=ds.path))) lgr.info("Creating target dataset {0} at {1}".format(current_dspath, path)) # Must be set to True only if exists and existing='reconfigure' # otherwise we might skip actions if we say existing='reconfigure' # but it did not even exist before only_reconfigure = False if path != '.': # check if target exists # TODO: Is this condition valid for != '.' only? path_exists = True try: out, err = ssh(["ls", path]) except CommandError as e: if "No such file or directory" in e.stderr and \ path in e.stderr: path_exists = False else: raise # It's an unexpected failure here if path_exists: if existing == 'error': raise RuntimeError("Target directory %s already exists." % path) elif existing == 'skip': continue elif existing == 'replace': ssh(["chmod", "+r+w", "-R", path]) # enable write permissions to allow removing dir ssh(["rm", "-rf", path]) # remove target at path path_exists = False # if we succeeded in removing it elif existing == 'reconfigure': only_reconfigure = True else: raise ValueError("Do not know how to handle existing=%s" % repr(existing)) if not path_exists: try: ssh(["mkdir", "-p", path]) except CommandError as e: lgr.error("Remotely creating target directory failed at " "%s.\nError: %s" % (path, exc_str(e))) continue # don't (re-)initialize dataset if existing == reconfigure if not only_reconfigure: # init git and possibly annex repo if not CreateSibling.init_remote_repo( path, ssh, shared, datasets[current_dspath], description=target_url): continue # check git version on remote end lgr.info("Adjusting remote git configuration") remote_git_version = CreateSibling.get_remote_git_version(ssh) if remote_git_version and remote_git_version >= "2.4": # allow for pushing to checked out branch try: ssh(["git", "-C", path] + ["config", "receive.denyCurrentBranch", "updateInstead"]) except CommandError as e: lgr.error("git config failed at remote location %s.\n" "You will not be able to push to checked out " "branch. Error: %s", path, exc_str(e)) else: lgr.error("Git version >= 2.4 needed to configure remote." " Version detected on server: %s\nSkipping configuration" " of receive.denyCurrentBranch - you will not be able to" " publish updates to this repository. Upgrade your git" " and run with --existing=reconfigure" % remote_git_version) # enable metadata refresh on dataset updates to publication server lgr.info("Enabling git post-update hook ...") try: CreateSibling.create_postupdate_hook( path, ssh, datasets[current_dspath]) except CommandError as e: lgr.error("Failed to add json creation command to post update " "hook.\nError: %s" % exc_str(e)) # publish web-interface to root dataset on publication server if at_root and ui: lgr.info("Uploading web interface to %s" % path) at_root = False try: CreateSibling.upload_web_interface(path, ssh, shared, ui) except CommandError as e: lgr.error("Failed to push web interface to the remote " "datalad repository.\nError: %s" % exc_str(e)) remote_repos_to_run_hook_for.append(path) # in reverse order would be depth first lgr.debug("Running post-update hooks in all created siblings") for path in remote_repos_to_run_hook_for[::-1]: # Trigger the hook try: ssh( ["cd '" + _path_(path, ".git") + "' && hooks/post-update"], wrap_args=False # we wrapped here manually ) except CommandError as e: lgr.error("Failed to run post-update hook under path %s. " "Error: %s" % (path, exc_str(e))) if target: # add the sibling(s): lgr.debug("Adding the siblings") if target_url is None: target_url = sshurl if target_pushurl is None: target_pushurl = sshurl AddSibling()(dataset=ds, name=target, url=target_url, pushurl=target_pushurl, recursive=recursive, fetch=True, force=existing in {'replace'}, as_common_datasrc=as_common_datasrc) # TODO: Return value!? # => [(Dataset, fetch_url)] @staticmethod def init_remote_repo(path, ssh, shared, dataset, description=None): cmd = ["git", "-C", path, "init"] if shared: cmd.append("--shared=%s" % shared) try: ssh(cmd) except CommandError as e: lgr.error("Initialization of remote git repository failed at %s." "\nError: %s\nSkipping ..." % (path, exc_str(e))) return False if isinstance(dataset.repo, AnnexRepo): # init remote git annex repo (part fix of #463) try: ssh( ["git", "-C", path, "annex", "init"] + ([description] if description else []) ) except CommandError as e: lgr.error("Initialization of remote git annex repository failed at %s." "\nError: %s\nSkipping ..." % (path, exc_str(e))) return False return True @staticmethod def get_remote_git_version(ssh): try: # options to disable all auto so we don't trigger them while testing # for absent changes out, err = ssh(["git"] + ["version"]) assert out.strip().startswith("git version") git_version = out.strip().split()[2] lgr.debug("Detected git version on server: %s" % git_version) return LooseVersion(git_version) except CommandError as e: lgr.warning( "Failed to determine git version on remote.\n" "Error: {0}\nTrying to configure anyway " "...".format(exc_str(e))) return None @staticmethod def create_postupdate_hook(path, ssh, dataset): # location of post-update hook file, logs folder on remote target hooks_remote_dir = opj(path, '.git', 'hooks') hook_remote_target = opj(hooks_remote_dir, 'post-update') # post-update hook should create its log directory if doesn't exist logs_remote_dir = opj(path, WEB_META_LOG) make_log_dir = 'mkdir -p "{}"'.format(logs_remote_dir) # create json command for current dataset json_command = r''' mkdir -p {}; ( which datalad > /dev/null \ && ( cd ..; GIT_DIR=$PWD/.git datalad ls -a --json file '{}'; ) \ || echo "no datalad found - skipping generation of indexes for web frontend"; \ ) &> "{}/{}" '''.format(logs_remote_dir, str(path), logs_remote_dir, 'datalad-publish-hook-$(date +%s).log' % TIMESTAMP_FMT) # collate content for post_update hook hook_content = '\n'.join(['#!/bin/bash', 'git update-server-info', make_log_dir, json_command]) with make_tempfile(content=hook_content) as tempf: # create post_update hook script ssh.copy(tempf, hook_remote_target) # upload hook to dataset ssh(['chmod', '+x', hook_remote_target]) # and make it executable @staticmethod def upload_web_interface(path, ssh, shared, ui): # path to web interface resources on local webui_local = opj(dirname(datalad.__file__), 'resources', 'website') # local html to dataset html_local = opj(webui_local, "index.html") # name and location of web-interface html on target html_targetname = {True: ui, False: "index.html"}[isinstance(ui, str)] html_target = opj(path, html_targetname) # upload ui html to target ssh.copy(html_local, html_target) # upload assets to the dataset webresources_local = opj(webui_local, 'assets') webresources_remote = opj(path, WEB_HTML_DIR) ssh(['mkdir', '-p', webresources_remote]) ssh.copy(webresources_local, webresources_remote, recursive=True) # minimize and upload js assets for js_file in glob(opj(webresources_local, 'js', '*.js')): with open(js_file) as asset: try: from jsmin import jsmin minified = jsmin(asset.read()) # minify asset except ImportError: lgr.warning( "Will not minify web interface javascript, no jsmin available") minified = asset.read() # no minify available with make_tempfile(content=minified) as tempf: # write minified to tempfile js_name = js_file.split('/')[-1] ssh.copy(tempf, opj(webresources_remote, 'assets', 'js', js_name)) # and upload js # explicitly make web+metadata dir of dataset world-readable, if shared set to 'all' mode = None if shared in (True, 'true', 'all', 'world', 'everybody'): mode = 'a+rX' elif shared == 'group': mode = 'g+rX' elif str(shared).startswith('0'): mode = shared if mode: ssh(['chmod', mode, '-R', dirname(webresources_remote), opj(path, 'index.html')]) datalad-0.4.1/datalad/distribution/create_sibling_github.py000066400000000000000000000337021301115270200241020ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for creating a publication target on GitHub """ __docformat__ = 'restructuredtext' import logging import re from os.path import join as opj from datalad import cfg from datalad.interface.common_opts import recursion_flag, recursion_limit from datalad.interface.common_opts import publish_depends from datalad.downloaders.credentials import UserPassword from datalad.dochelpers import exc_str from datalad.utils import assure_list from datalad.support.param import Parameter from datalad.support.constraints import EnsureStr, EnsureNone from datalad.support.constraints import EnsureChoice from datalad.support.exceptions import MissingExternalDependency from ..interface.base import Interface from datalad.distribution.dataset import EnsureDataset, datasetmethod, \ require_dataset, Dataset from .add_sibling import AddSibling lgr = logging.getLogger('datalad.distribution.create_sibling_github') def _get_github_entity(gh, cred, github_user, github_passwd, github_organization): # figure out authentication if not (github_user and github_passwd): # access to the system secrets if github_user: # check that they keystore knows about this user if github_user != cred.get('user', github_user): # there is a mismatch, we need to ask creds = cred.enter_new() github_user = creds['user'] github_passwd = creds['password'] # if a user is provided, go with it, don't even ask any store if github_user is None and not cred.is_known: # let's figure out authentication if github_user is None: # check if there is an oauth token from # https://github.com/sociomantic/git-hub github_user = cfg.get('hub.oauthtoken', None) if github_user is None: # still nothing, ask if necessary creds = cred() github_user = creds['user'] github_passwd = creds['password'] if not github_user: raise gh.BadCredentialsException(403, 'no user specified') # this will always succeed, but it might later throw an exception # if the credentials were wrong # XXX make sure to wipe out known credentials if that happens authed_gh = gh.Github( github_user, password=github_passwd) try: if github_organization: try: entity = authed_gh.get_organization(github_organization) except gh.UnknownObjectException as e: raise ValueError('unknown organization "{}" [{}]'.format( github_organization, exc_str(e))) else: entity = authed_gh.get_user() except gh.BadCredentialsException as e: # things blew up, wipe out cred store, if anything is in it if cred.is_known: cred.delete() raise e return entity def _make_github_repos( gh, github_user, github_passwd, github_organization, rinfo, existing, access_protocol, dryrun): cred = UserPassword('github', 'https://github.com/login') # determine the entity under which to create the repos entity = _get_github_entity( gh, cred, github_user, github_passwd, github_organization) res = [] for ds, reponame in rinfo: try: access_url, existed = _make_github_repo( gh, entity, reponame, existing, access_protocol, dryrun) res.append((ds, access_url, existed)) except gh.BadCredentialsException as e: # things blew up, wipe out cred store, if anything is in it if cred.is_known: cred.delete() raise e return res def _make_github_repo(gh, entity, reponame, existing, access_protocol, dryrun): repo = None try: repo = entity.get_repo(reponame) except gh.GithubException as e: if e.status != 404: # this is not a not found message, raise raise e lgr.debug( 'To be created repository "%s" does not yet exist on Github', reponame) if repo is not None: if existing in ('skip', 'reconfigure'): access_url = getattr(repo, '{}_url'.format(access_protocol)) return access_url, existing == 'skip' elif existing == 'error': msg = 'repository "{}" already exists on Github'.format(reponame) if dryrun: lgr.error(msg) else: raise ValueError(msg) else: RuntimeError('to must not happen') if repo is None and not dryrun: try: repo = entity.create_repo( reponame, # TODO description='', # TODO homepage='', # TODO private=False, has_issues=False, has_wiki=False, has_downloads=False, auto_init=False) except gh.GithubException as e: msg = "Github {}: {}".format( e.data.get('message', 'unknown'), ', '.join([err.get('message') for err in e.data.get('errors', []) if 'message' in err])) raise RuntimeError(msg) if repo is None and not dryrun: raise RuntimeError( 'something went wrong, we got no Github repository') if dryrun: return '{}:github/.../{}'.format(access_protocol, reponame), False else: # report URL for given access protocol return getattr(repo, '{}_url'.format(access_protocol)), False # presently only implemented method to turn subdataset paths into Github # compliant repository name suffixes template_fx = lambda x: re.sub(r'\s+', '_', re.sub(r'[/\\]+', '-', x)) class CreateSiblingGithub(Interface): """Create dataset sibling on Github. A repository can be created under a user's Github account, or any organization a user is a member of (given appropriate permissions). Recursive sibling creation for subdatasets is supported. A dataset hierarchy is represented as a flat list of Github repositories. Github cannot host dataset content. However, in combination with other data sources (and siblings), publishing a dataset to Github can facilitate distribution and exchange, while still allowing any dataset consumer to obtain actual data content from alternative sources. For Github authentication user credentials can be given as arguments. Alternatively, they are obtained interactively or queried from the systems credential store. Lastly, an *oauth* token stored in the Git configuration under variable *hub.oauthtoken* will be used automatically. Such a token can be obtained, for example, using the commandline Github interface (https://github.com/sociomantic/git-hub) by running: :kbd:`git hub setup`. """ _params_ = dict( dataset=Parameter( args=("--dataset", "-d",), doc="""specify the dataset to create the publication target for. If no dataset is given, an attempt is made to identify the dataset based on the current working directory""", constraints=EnsureDataset() | EnsureNone()), reponame=Parameter( args=('reponame',), metavar='REPONAME', doc="""Github repository name. When operating recursively, a suffix will be appended to this name for each subdataset""", constraints=EnsureStr()), recursive=recursion_flag, recursion_limit=recursion_limit, sibling_name=Parameter( args=('--sibling-name',), metavar='NAME', doc="""name to represent the Github repository in the local dataset installation""", constraints=EnsureStr()), existing=Parameter( args=("--existing",), constraints=EnsureChoice('skip', 'error', 'reconfigure'), metavar='MODE', doc="""desired behavior when already existing or configured siblings are discovered. 'skip': ignore; 'error': fail immediately; 'reconfigure': use the existing repository and reconfigure the local dataset to use it as a sibling""",), github_user=Parameter( args=('--github-user',), constraints=EnsureStr() | EnsureNone(), metavar='NAME', doc="""Github user name"""), github_passwd=Parameter( args=('--github-passwd',), constraints=EnsureStr() | EnsureNone(), metavar='PASSWORD', doc="""Github user password"""), github_organization=Parameter( args=('--github-organization',), constraints=EnsureStr() | EnsureNone(), metavar='NAME', doc="""If provided, the repository will be created under this Github organization. The respective Github user needs appropriate permissions."""), access_protocol=Parameter( args=("--access-protocol",), constraints=EnsureChoice('git', 'ssh'), doc="""Which access protocol/URL to configure for the sibling"""), publish_depends=publish_depends, dryrun=Parameter( args=("--dryrun",), action="store_true", doc="""If this flag is set, no communication with Github is performed, and no repositories will be created. Instead would-be repository names are reported for all relevant datasets """), ) @staticmethod @datasetmethod(name='create_sibling_github') def __call__( reponame, dataset=None, recursive=False, recursion_limit=None, sibling_name='github', existing='error', github_user=None, github_passwd=None, github_organization=None, access_protocol='git', publish_depends=None, dryrun=False): try: # this is an absolute leaf package, import locally to avoid # unecessary dependencies import github as gh except ImportError: raise MissingExternalDependency( 'PyGitHub', msg='GitHub-related functionality is unavailable without this package') # what to operate on ds = require_dataset( dataset, check_installed=True, purpose='create Github sibling') # gather datasets and essential info # dataset instance and mountpoint relative to the top toprocess = [(ds, '')] if recursive: for d in ds.get_subdatasets( fulfilled=None, # we want to report on missing dataset in here absolute=False, recursive=recursive, recursion_limit=recursion_limit): sub = Dataset(opj(ds.path, d)) if not sub.is_installed(): lgr.info('Ignoring unavailable subdataset %s', sub) continue toprocess.append((sub, d)) # check for existing remote configuration filtered = [] for d, mp in toprocess: if sibling_name in d.repo.get_remotes(): if existing == 'error': msg = '{} already had a configured sibling "{}"'.format( d, sibling_name) if dryrun: lgr.error(msg) else: raise ValueError(msg) elif existing == 'skip': continue gh_reponame = '{}{}{}'.format( reponame, '-' if mp else '', template_fx(mp)) filtered.append((d, gh_reponame)) if not filtered: # all skipped return [] # actually make it happen on Github rinfo = _make_github_repos( gh, github_user, github_passwd, github_organization, filtered, existing, access_protocol, dryrun) # lastly configure the local datasets for d, url, existed in rinfo: if not dryrun: # first make sure that annex doesn't touch this one # but respect any existing config ignore_var = 'remote.{}.annex-ignore'.format(sibling_name) if not ignore_var in d.config: d.config.add(ignore_var, 'true', where='local') AddSibling()( dataset=d, name=sibling_name, url=url, recursive=False, # TODO fetch=True, maybe only if one existed already force=existing in {'reconfigure'}, publish_depends=publish_depends) # TODO let submodule URLs point to Github (optional) return rinfo @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui res = assure_list(res) if args.dryrun: ui.message('DRYRUN -- Anticipated results:') if not len(res): ui.message("Nothing done") else: for d, url, existed in res: ui.message( "'{}'{} configured as sibling '{}' for {}".format( url, " (existing repository)" if existed else '', args.sibling_name, d)) datalad-0.4.1/datalad/distribution/create_test_dataset.py000066400000000000000000000143441301115270200235760ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """A Helper to initiate arbitrarily small/large test meta-dataset """ __docformat__ = 'numpy' import random import logging import tempfile from datalad.utils import get_tempfile_kwargs import os from os.path import join as opj, exists, isabs, abspath from datalad.distribution.dataset import Dataset from datalad.support.param import Parameter from datalad.support.constraints import EnsureStr, EnsureNone, EnsureInt from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from datalad.interface.base import Interface lgr = logging.getLogger('datalad.distribution.tests') def _parse_spec(spec): out = [] # will return a list of tuples (min, max) for each layer for ilevel, level in enumerate(spec.split('/')): minmax = level.split('-') if len(minmax) == 1: # only abs number specified minmax = int(minmax[0]) min_, max_ = (minmax, minmax) elif len(minmax) == 2: min_, max_ = minmax if not min_: # might be omitted entirely min_ = 0 if not max_: raise ValueError("Specify max number at level %d. Full spec was: %s" % (ilevel, spec)) min_ = int(min_) max_ = int(max_) else: raise ValueError("Must have only min-max at level %d" % ilevel) out.append((min_, max_)) return out def _makeds(path, levels, ds=None): """Create a hierarchy of datasets Used recursively, with current invocation generating datasets for the first level, and delegating sub-levels to recursive invocation Parameters ---------- path : str Path to the top directory under which dataset will be created. If relative -- relative to current directory levels : list of list List of specifications for :func:`random.randint` call per each level. ds : Dataset, optional Super-dataset which would contain a new dataset (thus its path whould be a parent of path. Note that ds needs to be installed. Yields ------ str Path to the generated dataset(s) """ # we apparently can't import api functionality within api from datalad.api import install # To simplify managing all the file paths etc if not isabs(path): path = abspath(path) # make it a git (or annex??) repository... ok - let's do randomly one or another ;) RepoClass = GitRepo if random.randint(0, 1) else AnnexRepo lgr.info("Generating repo of class %s under %s", RepoClass, path) repo = RepoClass(path, create=True) # let's create some dummy file and add it to the beast fn = opj(path, "file%d.dat" % random.randint(1, 1000)) with open(fn, 'w') as f: f.write(fn) repo.add(fn, git=True, commit=True, msg="Added %s" % fn, _datalad_msg=True) if ds: assert ds.is_installed() rpath = os.path.relpath(path, ds.path) out = install( rpath, source=opj(os.curdir, rpath), dataset=ds ) ds.repo.commit("subdataset %s installed." % rpath, _datalad_msg=True) if not levels: return # make a dataset for that one since we want to add sub datasets ds_ = Dataset(path) level, levels_ = levels[0], levels[1:] nrepos = random.randint(*level) # how many subds to generate for irepo in range(nrepos): # we would like to have up to 2 leading dirs subds_path = opj(*(['d%i' % i for i in range(random.randint(0, 3))] + ['r%i' % irepo])) subds_fpath = opj(path, subds_path) yield subds_fpath # and all under for d in _makeds(subds_fpath, levels_, ds=ds_): yield d class CreateTestDataset(Interface): """Create test (meta-)dataset. """ _params_ = dict( path=Parameter( args=("path",), doc="path/name where to create (if specified, must not exist)", constraints=EnsureStr() | EnsureNone()), spec=Parameter( args=("--spec",), doc="""\ spec for hierarchy, defined as a min-max (min could be omitted to assume 0) defining how many (random number from min to max) of sub-datasets to generate at any given level of the hierarchy. Each level separated from each other with /. Example: 1-3/-2 would generate from 1 to 3 subdatasets at the top level, and up to two within those at the 2nd level """, constraints=EnsureStr() | EnsureNone()), seed=Parameter( args=("--seed",), doc="""seed for rng""", constraints=EnsureInt() | EnsureNone()), ) @staticmethod def __call__(path=None, spec=None, seed=None): if spec is None: spec = "10/1-3/-2" # 10 on top level, some random number from 1 to 3 at the 2nd, up to 2 on 3rd levels = _parse_spec(spec) if seed is not None: # TODO: if to be used within a bigger project we shouldn't seed main RNG random.seed(seed) if path is None: kw = get_tempfile_kwargs({}, prefix="ds") path = tempfile.mkdtemp(**kw) else: # so we don't override anything assert not exists(path) os.makedirs(path) # now we should just make it happen and return list of all the datasets return list(_makeds(path, levels)) @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui if res is None: res = [] if not len(res): ui.message("No repos were created... oops") return items = '\n'.join(map(str, res)) msg = "{n} installed {obj} available at\n{items}".format( obj='items are' if len(res) > 1 else 'item is', n=len(res), items=items) ui.message(msg) datalad-0.4.1/datalad/distribution/dataset.py000066400000000000000000000516531301115270200212200ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Implements class Dataset """ import logging from functools import wraps from os.path import abspath from os.path import commonprefix from os.path import curdir from os.path import exists from os.path import join as opj from os.path import normpath from os.path import pardir from os.path import realpath from os.path import relpath from os.path import sep from six import PY2 from six import string_types from datalad.config import ConfigManager from datalad.consts import LOCAL_CENTRAL_PATH from datalad.dochelpers import exc_str from datalad.support.annexrepo import AnnexRepo from datalad.support.constraints import Constraint from datalad.support.exceptions import NoDatasetArgumentFound from datalad.support.exceptions import PathOutsideRepositoryError from datalad.support.gitrepo import GitRepo from datalad.support.gitrepo import InvalidGitRepositoryError from datalad.support.gitrepo import NoSuchPathError from datalad.utils import getpwd from datalad.utils import optional_args, expandpath, is_explicit_path, \ with_pathsep from datalad.utils import swallow_logs lgr = logging.getLogger('datalad.dataset') lgr.log(5, "Importing dataset") # TODO: use the same piece for resolving paths against Git/AnnexRepo instances # (see normalize_path) def resolve_path(path, ds=None): """Resolve a path specification (against a Dataset location) Any explicit path (absolute or relative) is returned as an absolute path. In case of an explicit relative path, the current working directory is used as a reference. Any non-explicit relative path is resolved against as dataset location, i.e. considered relative to the location of the dataset. If no dataset is provided, the current working directory is used. Returns ------- Absolute path """ path = expandpath(path, force_absolute=False) # TODO: normpath?! if is_explicit_path(path): return abspath(path) # no dataset given, use CWD as reference # note: abspath would disregard symlink in CWD top_path = getpwd() \ if ds is None else ds.path if isinstance(ds, Dataset) else ds return normpath(opj(top_path, path)) class Dataset(object): __slots__ = ['_path', '_repo', '_id', '_cfg'] def __init__(self, path): # Custom handling for few special abbreviations path_ = path if path == '^': # get the topmost dataset from current location. Note that 'zsh' # might have its ideas on what to do with ^, so better use as -d^ path_ = Dataset(curdir).get_superdataset(topmost=True).path elif path == '///': # TODO: logic/UI on installing a central dataset could move here # from search? path_ = LOCAL_CENTRAL_PATH if path != path_: lgr.debug("Resolved dataset alias %r to path %r", path, path_) self._path = abspath(path_) self._repo = None self._id = None self._cfg = None def __repr__(self): return "" % self.path def __eq__(self, other): if not hasattr(other, 'path'): return False return realpath(self.path) == realpath(other.path) @property def path(self): """path to the dataset""" return self._path @property def repo(self): """Get an instance of the version control system/repo for this dataset, or None if there is none yet. If creating an instance of GitRepo is guaranteed to be really cheap this could also serve as a test whether a repo is present. Returns ------- GitRepo """ if self._repo is None: with swallow_logs(): for cls, ckw, kw in ( (AnnexRepo, {'allow_noninitialized': True}, {'init': False}), (GitRepo, {}, {}) ): if cls.is_valid_repo(self._path, **ckw): try: lgr.debug("Detected %s at %s", cls, self._path) self._repo = cls(self._path, create=False, **kw) break except (InvalidGitRepositoryError, NoSuchPathError) as exc: lgr.debug( "Oops -- guess on repo type was wrong?: %s", exc_str(exc)) pass # version problems come as RuntimeError: DO NOT CATCH! if self._repo is None: # Often .repo is requested to 'sense' if anything is installed # under, and if so -- to proceed forward. Thus log here only # at DEBUG level and if necessary "complaint upstairs" lgr.debug("Failed to detect a valid repo at %s" % self.path) elif not isinstance(self._repo, AnnexRepo): # repo was initially set to be self._repo but might become AnnexRepo # at a later moment, so check if it didn't happen if 'git-annex' in self._repo.get_branches(): # we acquired git-annex branch self._repo = AnnexRepo(self._repo.path, create=False) return self._repo @property def id(self): """Identifier of the dataset. This identifier is supposed to be unique across datasets, but identical for different versions of the same dataset (that have all been derived from the same original dataset repository). Returns ------- str This is either a stored UUID, or `None`. """ if self._id is None: # if we have one on record, stick to it! self._id = self.config.get('datalad.dataset.id', None) return self._id @property def config(self): """Get an instance of the parser for the persistent dataset configuration. Returns ------- ConfigManager """ if self._cfg is None: # associate with this dataset and read the entire config hierarchy self._cfg = ConfigManager(dataset=self, dataset_only=False) return self._cfg def register_sibling(self, name, url, publish_url=None, verify=None): """Register the location of a sibling dataset under a given name. Optionally, different URLs can be given for retrieving information from the sibling and for publishing information to it. This is a cheap operation that does not confirm that at the given location an actual sibling dataset is available, unless verify is set. The value "dataset" verifies, that at the given URL an accessible dataset is available and the value "sibling" furthermore verifies, that this dataset shares at least one commit with self. Parameters ---------- name url publish_url verify None | "dataset" | "sibling" """ repo = self.repo if verify is not None: raise NotImplementedError("TODO: verify not implemented yet") if name not in repo.get_remotes(): # Add remote repo.add_remote(name, url) if publish_url is not None: # set push url: repo._git_custom_command('', ["git", "remote", "set-url", "--push", name, publish_url]) lgr.info("Added remote '%s':\n %s (pull)\n%s (push)." % (name, url, publish_url if publish_url else url)) else: lgr.warning("Remote '%s' already exists. Ignore.") raise ValueError("'%s' already exists. Couldn't register sibling.") # TODO: RF: Dataset.get_subdatasets to return Dataset instances! (optional?) # weakref # singleton def get_subdatasets(self, pattern=None, fulfilled=None, absolute=False, recursive=False, recursion_limit=None): """Get names/paths of all known subdatasets (sorted depth-first) optionally matching a specific name pattern. Parameters ---------- pattern : None Not implemented fulfilled : None or bool If not None, return either only present or absent datasets. absolute : bool If True, absolute paths will be returned. recursive : bool If True, recurse into all subdatasets and report them too. recursion_limit: int or None If not None, set the number of subdataset levels to recurse into. Returns ------- list(Dataset paths) or None None is return if there is not repository instance yet. For an existing repository with no subdatasets an empty list is returned. """ if isinstance(recursion_limit, int) and (recursion_limit <= 0): return [] if pattern is not None: raise NotImplementedError repo = self.repo if repo is None: return [] # check whether we have anything in the repo. if not go home early if not repo.repo.head.is_valid(): return [] try: submodules = repo.get_submodules() except InvalidGitRepositoryError: # this happens when we access a repository with a submodule that # has no commits, hence doesn't appear in the index and # 'git submodule status' also doesn't list it return [] # filter if desired if fulfilled is None: submodules = [sm.path for sm in submodules] else: submodules = [sm.path for sm in submodules if sm.module_exists() == fulfilled] # expand list with child submodules. keep all paths relative to parent # and convert jointly at the end if recursive \ and (recursion_limit in (None, 'existing') or (isinstance(recursion_limit, int) and recursion_limit > 1)): rsm = [] for sm in submodules: sdspath = opj(self._path, sm) rsm.extend( [opj(sm, sdsh) for sdsh in Dataset(sdspath).get_subdatasets( pattern=pattern, fulfilled=fulfilled, absolute=False, recursive=recursive, recursion_limit=(recursion_limit - 1) if isinstance(recursion_limit, int) else recursion_limit)]) rsm.append(sm) submodules = rsm if absolute: return [opj(self._path, sm) for sm in submodules] else: return submodules # def get_file_handles(self, pattern=None, fulfilled=None): # """Get paths to all known file_handles, optionally matching a specific # name pattern. # # If fulfilled is True, only paths to fullfiled handles are returned, # if False, only paths to unfulfilled handles are returned. # # Parameters # ---------- # pattern: str # fulfilled: bool # # Returns # ------- # list of str # (paths) # """ # raise NotImplementedError("TODO") def recall_state(self, whereto): """Something that can be used to checkout a particular state (tag, commit) to "undo" a change or switch to a otherwise desired previous state. Parameters ---------- whereto: str """ if not self.is_installed(): raise RuntimeError( "cannot remember a state when a dataset is not yet installed") self.repo.checkout(whereto) def is_installed(self): """Returns whether a dataset is installed. A dataset is installed when a repository for it exists on the filesystem. Returns ------- bool """ # do early check manually if path exists to not even ask git at all exists_now = exists(self.path) was_once_installed = None if exists_now: was_once_installed = self.path is not None and \ self.repo is not None if not exists_now or \ (was_once_installed and not exists(self.repo.repo.git_dir)): # repo gone now, reset self._repo = None return False else: return was_once_installed def get_superdataset(self, datalad_only=False, topmost=False): """Get the dataset's superdataset Parameters ---------- datalad_only : bool, optional Either to consider only "datalad datasets" (with non-None id), or (if False, which is default) - any git repository topmost : bool, optional Return the topmost super-dataset. Might then be the current one. Returns ------- Dataset or None """ # TODO: return only if self is subdataset of the superdataset # (meaning: registered as submodule)? path = self.path sds_path = path if topmost else None while path: # normalize the path after adding .. so we guaranteed to not # follow into original directory if path itself is a symlink par_path = normpath(opj(path, pardir)) sds_path_ = GitRepo.get_toppath(par_path) if sds_path_ is None: # no more parents, use previous found break if datalad_only: # test if current git is actually a dataset? sds = Dataset(sds_path_) # can't use ATM since we just autogenerate and ID, see # https://github.com/datalad/datalad/issues/986 # if not sds.id: if not sds.config.get('datalad.dataset.id', None): break # That was a good candidate sds_path = sds_path_ path = par_path if not topmost: # no looping break if sds_path is None: # None was found return None # No postprocessing now should be necessary since get_toppath # tries its best to not resolve symlinks now return Dataset(sds_path) def get_containing_subdataset(self, path, recursion_limit=None): """Get the (sub-)dataset containing `path` Note: The "mount point" of a subdataset is classified as belonging to that respective subdataset. Parameters ---------- path : str Path to determine the containing (sub-)dataset for recursion_limit: int or None limit the subdatasets to take into account to the given number of hierarchy levels Returns ------- Dataset """ if recursion_limit is not None and (recursion_limit < 1): lgr.warning("recursion limit < 1 (%s) always results in self.", recursion_limit) return self if is_explicit_path(path): path = resolve_path(path, self) if not path.startswith(self.path): raise PathOutsideRepositoryError(file_=path, repo=self) path = relpath(path, self.path) candidates = [] # TODO: this one would follow all the sub-datasets, which might # be inefficient if e.g. there is lots of other sub-datasets already # installed but under another sub-dataset. There is a TODO 'pattern' # option which we could use I guess eventually for subds in self.get_subdatasets(recursive=True, #pattern= recursion_limit=recursion_limit, absolute=False): common = commonprefix((with_pathsep(subds), with_pathsep(path))) if common.endswith(sep) and common == with_pathsep(subds): candidates.append(common) if candidates: return Dataset(path=opj(self.path, max(candidates, key=len))) return self @optional_args def datasetmethod(f, name=None, dataset_argname='dataset'): """Decorator to bind functions to Dataset class. The decorated function is still directly callable and additionally serves as method `name` of class Dataset. To achieve this, the first positional argument is redirected to original keyword argument 'dataset_argname'. All other arguments stay in order (and keep their names, of course). That means, that the signature of the bound function is name(self, a, b) if the original signature is name(a, dataset, b) for example. The decorator has no effect on the actual function decorated with it. """ if not name: name = f.func_name if PY2 else f.__name__ @wraps(f) def apply_func(*args, **kwargs): """Wrapper function to assign arguments of the bound function to original function. Note ---- This wrapper is NOT returned by the decorator, but only used to bind the function `f` to the Dataset class. """ kwargs = kwargs.copy() from inspect import getargspec orig_pos = getargspec(f).args # If bound function is used with wrong signature (especially by # explicitly passing a dataset, let's raise a proper exception instead # of a 'list index out of range', that is not very telling to the user. if len(args) > len(orig_pos) or dataset_argname in kwargs: raise TypeError("{0}() takes at most {1} arguments ({2} given):" " {3}".format(name, len(orig_pos), len(args), ['self'] + [a for a in orig_pos if a != dataset_argname])) kwargs[dataset_argname] = args[0] ds_index = orig_pos.index(dataset_argname) for i in range(1, len(args)): if i <= ds_index: kwargs[orig_pos[i-1]] = args[i] elif i > ds_index: kwargs[orig_pos[i]] = args[i] return f(**kwargs) setattr(Dataset, name, apply_func) # So we could post-hoc later adjust the documentation string which is assigned # within .api apply_func.__orig_func__ = f return f # Note: Cannot be defined with constraints.py, since then dataset.py needs to # be imported from constraints.py, which needs to be imported from dataset.py # for another constraint class EnsureDataset(Constraint): def __call__(self, value): if isinstance(value, Dataset): return value elif isinstance(value, string_types): return Dataset(path=value) else: raise ValueError("Can't create Dataset from %s." % type(value)) # TODO: Proper description? Mentioning Dataset class doesn't make sense for # commandline doc! def short_description(self): return "Dataset" def long_description(self): return """Value must be a Dataset or a valid identifier of a Dataset (e.g. a path)""" def require_dataset(dataset, check_installed=True, purpose=None): """Helper function to resolve a dataset. This function tries to resolve a dataset given an input argument, or based on the process' working directory, if `None` is given. Parameters ---------- dataset : None or path or Dataset Some value identifying a dataset or `None`. In the latter case a dataset will be searched based on the process working directory. check_installed : bool, optional If True, an optional check whether the resolved dataset is properly installed will be performed. purpose : str, optional This string will be inserted in error messages to make them more informative. The pattern is "... dataset for ". Returns ------- Dataset Or raises an exception (InsufficientArgumentsError). """ if dataset is not None and not isinstance(dataset, Dataset): dataset = Dataset(dataset) if dataset is None: # possible scenario of cmdline calls dspath = GitRepo.get_toppath(getpwd()) if not dspath: raise NoDatasetArgumentFound("No dataset found") dataset = Dataset(dspath) assert(dataset is not None) lgr.debug("Resolved dataset{0}: {1}".format( ' for {}'.format(purpose) if purpose else '', dataset)) if check_installed and not dataset.is_installed(): raise ValueError("No installed dataset found at " "{0}.".format(dataset.path)) return dataset datalad-0.4.1/datalad/distribution/get.py000066400000000000000000000425531301115270200203510ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for getting dataset content """ import logging from itertools import chain from os import curdir from os.path import isdir from os.path import join as opj from os.path import relpath from os.path import lexists from os.path import dirname from datalad.interface.base import Interface from datalad.interface.common_opts import recursion_flag from datalad.interface.common_opts import git_opts from datalad.interface.common_opts import annex_opts from datalad.interface.common_opts import annex_get_opts from datalad.interface.common_opts import jobs_opt from datalad.interface.common_opts import reckless_opt from datalad.interface.common_opts import verbose from datalad.support.constraints import EnsureInt from datalad.support.constraints import EnsureChoice from datalad.support.constraints import EnsureStr from datalad.support.constraints import EnsureNone from datalad.support.param import Parameter from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.exceptions import IncompleteResultsError from datalad.dochelpers import single_or_plural from datalad.utils import assure_list from datalad.utils import with_pathsep as _with_sep # TODO: RF whenever merge conflict is not upon us from .dataset import Dataset from .dataset import EnsureDataset from .dataset import datasetmethod from .dataset import resolve_path from .utils import install_necessary_subdatasets from .utils import _recursive_install_subds_underneath __docformat__ = 'restructuredtext' lgr = logging.getLogger('datalad.distribution.get') def _sort_paths_into_datasets(paths, out=None, dir_lookup=None, recursive=False, recursion_limit=None): """Returns dict of `existing dataset path`: `directory` mappings Any paths that are not part of a dataset or ignored. """ # sort paths into the respective datasets if dir_lookup is None: dir_lookup = {} if out is None: out = {} # paths that don't exist (yet) unavailable_paths = [] for path in paths: if not lexists(path): # not there yet, impossible to say which ds it will actually # be in, if any unavailable_paths.append(path) continue # the path exists in some shape or form if isdir(path): # this could contain all types of additional content d = path else: # for everything else we are interested in the container d = dirname(path) if not d: d = curdir # this could be `None` if there is no git repo dspath = dir_lookup.get(d, GitRepo.get_toppath(d)) dir_lookup[d] = dspath if not dspath: lgr.warning("%s is not part of a dataset, ignored.", path) continue if isdir(path): ds = Dataset(dspath) # we need to doublecheck that this is not a subdataset mount # point, in whic case get_toppath() would point to the parent smpath = ds.get_containing_subdataset( path, recursion_limit=1).path if smpath != dspath: # fix entry dir_lookup[d] = smpath # submodule still needs to be obtained unavailable_paths.append(path) continue if recursive: # make sure we get everything relevant in all _checked out_ # subdatasets, obtaining of previously unavailable subdataset # else done elsewhere subs = ds.get_subdatasets(fulfilled=True, recursive=recursive, recursion_limit=recursion_limit) for sub in subs: subdspath = opj(dspath, sub) if subdspath.startswith(_with_sep(path)): # this subdatasets is underneath the search path # we want it all # be careful to not overwrite anything, in case # this subdataset has been processed before out[subdspath] = out.get(subdspath, [subdspath]) out[dspath] = out.get(dspath, []) + [path] return out, unavailable_paths, dir_lookup def _get(content_by_ds, refpath=None, source=None, jobs=None, get_data=True): """Loops through datasets and calls git-annex call where appropriate """ for ds_path in sorted(content_by_ds.keys()): cur_ds = Dataset(ds_path) content = content_by_ds[ds_path] results = [] if len(content) >= 1 and content[0] == curdir: # we hit a subdataset that just got installed few lines above, and was # requested specifically, as opposed to some of its content. results.append(cur_ds) if not get_data: lgr.debug( "Will not get any content in %s, as instructed.", cur_ds) yield results continue # needs to be an annex: found_an_annex = isinstance(cur_ds.repo, AnnexRepo) if not found_an_annex: lgr.debug("Found no annex at %s. Skipped.", cur_ds) if results: yield results continue lgr.info("Getting %i items of dataset %s ...", len(content), cur_ds) results.extend(cur_ds.repo.get( content, options=['--from=%s' % source] if source else [], jobs=jobs)) if refpath: # adapt relative paths reported by annex to be relative some # reference if ds_path != refpath: for lr in results: if isinstance(lr, dict): lr['file'] = relpath(opj(ds_path, lr['file']), refpath) yield results class Get(Interface): """Get any dataset content (files/directories/subdatasets). This command only operates on dataset content. To obtain a new independent dataset from some source use the `install` command. By default this command operates recursively within a dataset, but not across potential subdatasets, i.e. if a directory is provided, all files in the directory are obtained. Recursion into subdatasets is supported too. If enabled, relevant subdatasets are detected and installed in order to fulfill a request. Known data locations for each requested file are evaluated and data are obtained from some available location (according to git-annex configuration and possibly assigned remote priorities), unless a specific source is specified. .. note:: Power-user info: This command uses :command:`git annex get` to fulfill file handles. """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), metavar="PATH", doc="""specify the dataset to perform the add operation on, in which case `path` arguments are interpreted as being relative to this dataset. If no dataset is given, an attempt is made to identify a dataset for each input `path`""", constraints=EnsureDataset() | EnsureNone()), path=Parameter( args=("path",), metavar="PATH", doc="""path/name of the requested dataset component. The component must already be known to a dataset. To add new components to a dataset use the `add` command""", nargs="*", constraints=EnsureStr() | EnsureNone()), source=Parameter( args=("-s", "--source",), metavar="LABEL", doc="""label of the data source to be used to fulfill requests. This can be the name of a dataset :term:`sibling` or another known source""", constraints=EnsureStr() | EnsureNone()), recursive=recursion_flag, recursion_limit=Parameter( args=("--recursion-limit",), metavar="LEVELS", constraints=EnsureInt() | EnsureChoice('existing') | EnsureNone(), doc="""limit recursion into subdataset to the given number of levels. Alternatively, 'existing' will limit recursion to subdatasets that already existed on the filesystem at the start of processing, and prevent new subdatasets from being obtained recursively."""), get_data=Parameter( args=("-n", "--no-data",), dest='get_data', action='store_false', doc="""whether to obtain data for all file handles. If disabled, `get` operations are limited to dataset handles.[CMD: This option prevents data for file handles from being obtained CMD]"""), reckless=reckless_opt, git_opts=git_opts, annex_opts=annex_opts, annex_get_opts=annex_get_opts, jobs=jobs_opt, verbose=verbose) # Note: May be use 'git annex find --not --in here' to have a list of all # files to actually get and give kind of a progress in terms of number # files processed ... @staticmethod @datasetmethod(name='get') def __call__( path, source=None, dataset=None, recursive=False, recursion_limit=None, get_data=True, reckless=False, git_opts=None, annex_opts=None, annex_get_opts=None, jobs=None, verbose=False, # internal -- instead of returning 'get'ed items, return final # content_by_ds, unavailable_paths. To be used by the call from # Install.__call__ and done so to avoid creating another reusable # function which would need to duplicate all this heavy list of # kwargs _return_datasets=False ): dataset_path = dataset.path if isinstance(dataset, Dataset) else dataset path = assure_list(path) if not path: raise InsufficientArgumentsError( "`get` needs at least one path as argument") # IMPLEMENTATION CONCEPT: # # 1. turn all input paths into absolute paths # 2. Sort the world into existing handles and the rest # 3. Try locate missing handles (obtain subdatasets along the way) # 4. Expand into subdatasets with recursion enables (potentially # obtain even more subdatasets # 5. Shoot info of which handles to get in each subdataset to, # git-annex, once at the very end # resolve path(s): resolved_paths = [resolve_path(p, dataset) for p in path] if dataset: # guarantee absolute paths relative to any given dataset resolved_paths = [opj(dataset_path, p) for p in resolved_paths] lgr.debug('Resolved targets to get: %s', resolved_paths) # sort paths into the respective datasets content_by_ds, unavailable_paths, dir_lookup = \ _sort_paths_into_datasets(resolved_paths, recursive=recursive, recursion_limit=recursion_limit) lgr.debug( "Found %i existing dataset(s) to get content in " "and %d unavailable paths", len(content_by_ds), len(unavailable_paths) ) # IMPORTANT NOTE re `content_by_ds` # each key is a subdataset that we need to get something in # if the value[0] is the subdataset's path, we want all of it # if the value[0] == curdir, we just installed it as part of # resolving file handles and we did not say anything but "give # me the dataset handle" # explore the unknown for path in sorted(unavailable_paths): # how close can we get? dspath = GitRepo.get_toppath(path) if dspath is None: # nothing we can do for this path continue ds = Dataset(dspath) # must always yield a dataset -- we sorted out the ones outside # any dataset at the very top assert ds.is_installed() # now actually obtain whatever is necessary to get to this path containing_ds = install_necessary_subdatasets(ds, path, reckless) if containing_ds.path != ds.path: lgr.debug("Installed %s to fulfill request for content for " "path %s", containing_ds, path) # mark resulting dataset as auto-installed if containing_ds.path == path: # we had to get the entire dataset, not something within # mark that it just appeared content_by_ds[path] = [curdir] else: # we need to get content within content_by_ds[path] = [path] if recursive and not recursion_limit == 'existing': # obtain any subdatasets underneath the paths given inside the # subdatasets that we know already exist # unless we do not want recursion into not-yet-installed datasets for subdspath in sorted(content_by_ds.keys()): for content_path in content_by_ds[subdspath]: if not isdir(content_path): # a non-directory cannot have content underneath continue subds = Dataset(subdspath) lgr.info( "Obtaining %s %s recursively", subds, ("underneath %s" % content_path if subds.path != content_path else "")) cbysubds = _recursive_install_subds_underneath( subds, # `content_path` was explicitly given as input # we count recursions from the input, hence we # can start with the full number recursion_limit, reckless, # protect against magic marker misinterpretation # only relevant for _get, hence replace here start=content_path if content_path != curdir else None) # gets file content for all freshly installed subdatasets content_by_ds.update(cbysubds) ## we have now done everything we could to obtain whatever subdataset ## to get something on the file system for previously unavailable paths ## check and sort one last content_by_ds, unavailable_paths, dir_lookup = \ _sort_paths_into_datasets( unavailable_paths, out=content_by_ds, dir_lookup=dir_lookup, recursive=recursive, recursion_limit=recursion_limit) if unavailable_paths: lgr.warning('ignored non-existing paths: %s', unavailable_paths) # hand over to git-annex results = list(chain.from_iterable( _get(content_by_ds, refpath=dataset_path, source=source, jobs=jobs, get_data=get_data))) # ??? should we in _return_datasets case just return both content_by_ds # and unavailable_paths may be so we provide consistent across runs output # and then issue outside similar IncompleteResultsError? if unavailable_paths: # and likely other error flags if _return_datasets: results = sorted(set(content_by_ds).difference(unavailable_paths)) raise IncompleteResultsError(results, failed=unavailable_paths) else: return sorted(content_by_ds) if _return_datasets else results @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui from os import linesep if res is None: res = [] if not isinstance(res, list): res = [res] if not len(res): ui.message("Got nothing new") return # provide summary nsuccess = sum(item.get('success', False) if isinstance(item, dict) else True for item in res) nfailure = len(res) - nsuccess msg = "Tried to get %d %s." % ( len(res), single_or_plural("file", "files", len(res))) if nsuccess: msg += " Got %d. " % nsuccess if nfailure: msg += " Failed to get %d." % (nfailure,) ui.message(msg) # if just a few or less than initially explicitly requested if len(res) < 10 or args.verbose: msg = linesep.join([ "{path} ... {suc}".format( suc="ok." if isinstance(item, Dataset) or item.get('success', False) else "failed. (%s)" % item.get('note', 'unknown reason'), path=item.get('file') if isinstance(item, dict) else item.path) for item in res]) ui.message(msg) datalad-0.4.1/datalad/distribution/install.py000066400000000000000000000564651301115270200212470ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for dataset (component) installation """ import logging from os import curdir from os import linesep from os.path import relpath from os.path import pardir from datalad.interface.base import Interface from datalad.interface.common_opts import recursion_flag from datalad.interface.common_opts import recursion_limit from datalad.interface.common_opts import dataset_description from datalad.interface.common_opts import jobs_opt from datalad.interface.common_opts import git_opts from datalad.interface.common_opts import git_clone_opts from datalad.interface.common_opts import annex_opts from datalad.interface.common_opts import annex_init_opts from datalad.interface.common_opts import if_dirty_opt from datalad.interface.common_opts import nosave_opt from datalad.interface.common_opts import reckless_opt from datalad.interface.utils import handle_dirty_dataset from datalad.support.constraints import EnsureNone from datalad.support.constraints import EnsureStr from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.exceptions import InstallFailedError from datalad.support.exceptions import IncompleteResultsError from datalad.support.param import Parameter from datalad.support.network import RI from datalad.support.network import PathRI from datalad.support.network import is_datalad_compat_ri from datalad.support.network import get_local_file_url from datalad.utils import assure_list from datalad.dochelpers import exc_str from datalad.dochelpers import single_or_plural from .dataset import Dataset from .dataset import datasetmethod from .dataset import resolve_path from .dataset import require_dataset from .dataset import EnsureDataset from .get import Get from .utils import _get_git_url_from_source from .utils import _install_subds_from_flexible_source from .utils import _get_flexible_source_candidates from .utils import _get_tracking_source from .utils import _clone_from_any_source from .utils import _handle_possible_annex_dataset from .utils import _save_installed_datasets __docformat__ = 'restructuredtext' lgr = logging.getLogger('datalad.distribution.install') def _get_installationpath_from_url(url): """Returns a relative path derived from the trailing end of a URL This can be used to determine an installation path of a Dataset from a URL, analog to what `git clone` does. """ path = url.rstrip('/') if '/' in path: path = path.split('/') if path[-1] == '.git': path = path[-2] else: path = path[-1] if path.endswith('.git'): path = path[:-4] return path class Install(Interface): """Install a dataset from a (remote) source. This command creates a local :term:`sibling` of an existing dataset from a (remote) location identified via a URL or path. Optional recursion into potential subdatasets, and download of all referenced data is supported. The new dataset can be optionally registered in an existing :term:`superdataset` by identifying it via the `dataset` argument (the new dataset's path needs to be located within the superdataset for that). It is recommended to provide a brief description to label the dataset's nature *and* location, e.g. "Michael's music on black laptop". This helps humans to identify data locations in distributed scenarios. By default an identifier comprised of user and machine name, plus path will be generated. When only partial dataset content shall be obtained, it is recommended to use this command without the `get-data` flag, followed by a :func:`~datalad.api.get` operation to obtain the desired data. .. note:: Power-user info: This command uses :command:`git clone`, and :command:`git annex init` to prepare the dataset. Registering to a superdataset is performed via a :command:`git submodule add` operation in the discovered superdataset. """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), # TODO: this probably changes to install into the dataset (add_to_super) # and to install the thing 'just there' without operating 'on' a dataset. # Adapt doc. # MIH: `shouldn't this be the job of `add`? doc="""specify the dataset to perform the install operation on. If no dataset is given, an attempt is made to identify the dataset in a parent directory of the current working directory and/or the `path` given""", constraints=EnsureDataset() | EnsureNone()), path=Parameter( args=("path",), metavar='PATH', nargs="*", # doc: TODO doc="""path/name of the installation target. If no `path` is provided a destination path will be derived from a source URL similar to :command:`git clone`"""), source=Parameter( args=("-s", "--source"), metavar='SOURCE', doc="URL or local path of the installation source", constraints=EnsureStr() | EnsureNone()), get_data=Parameter( args=("-g", "--get-data",), doc="""if given, obtain all data content too""", action="store_true"), description=dataset_description, recursive=recursion_flag, recursion_limit=recursion_limit, if_dirty=if_dirty_opt, save=nosave_opt, reckless=reckless_opt, git_opts=git_opts, git_clone_opts=git_clone_opts, annex_opts=annex_opts, annex_init_opts=annex_init_opts, jobs=jobs_opt, ) @staticmethod @datasetmethod(name='install') def __call__( path=None, source=None, dataset=None, get_data=False, description=None, recursive=False, recursion_limit=None, if_dirty='save-before', save=True, reckless=False, git_opts=None, git_clone_opts=None, annex_opts=None, annex_init_opts=None, jobs=None): # normalize path argument to be equal when called from cmdline and # python and nothing was passed into `path` path = assure_list(path) if not source and not path: raise InsufficientArgumentsError( "Please provide at least a source or a path") ## Common kwargs to pass to underlying git/install calls. # They might need adjustments (e.g. for recursion_limit, but # otherwise would be applicable throughout # # There should have been more of common options! # since underneath get could do similar installs, but now they # have duplicated implementations which differ (e.g. get does not # annex init installed annexes) common_kwargs = dict( get_data=get_data, recursive=recursive, recursion_limit=recursion_limit, git_opts=git_opts, annex_opts=annex_opts, reckless=reckless, jobs=jobs, ) installed_items = [] failed_items = [] # did we explicitly get a dataset to install into? # if we got a dataset, path will be resolved against it. # Otherwise path will be resolved first. ds = None if dataset is not None: ds = require_dataset(dataset, check_installed=True, purpose='installation') handle_dirty_dataset(ds, if_dirty) # switch into scenario without --source: if source is None: # we need to collect URLs and paths to_install = [] to_get = [] for urlpath in path: ri = RI(urlpath) (to_get if isinstance(ri, PathRI) else to_install).append(urlpath) common_kwargs['dataset'] = dataset # first install, and then get for s in to_install: lgr.debug("Install passes into install source=%s", s) try: result = Install.__call__( source=s, description=description, if_dirty=if_dirty, save=save, git_clone_opts=git_clone_opts, annex_init_opts=annex_init_opts, **common_kwargs ) installed_items += assure_list(result) except Exception as exc: lgr.warning("Installation of %s has failed: %s", s, exc_str(exc)) failed_items.append(s) if to_get: lgr.debug("Install passes into get %d items", len(to_get)) # all commented out hint on inability to pass those options # into underlying install-related calls. # Also need to pass from get: # annex_get_opts try: installed_datasets = Get.__call__( to_get, # description=description, # if_dirty=if_dirty, # save=save, # git_clone_opts=git_clone_opts, # annex_init_opts=annex_init_opts _return_datasets=True, **common_kwargs ) except IncompleteResultsError as exc: exc_str_ = ': ' + exc_str(exc) if exc.results else '' lgr.warning("Some items failed to install: %s", exc_str_) installed_datasets = exc.results failed_items.extend(exc.failed) # compose content_by_ds into result for dspath in installed_datasets: ds_ = Dataset(dspath) if ds_.is_installed(): installed_items.append(ds_) else: lgr.warning("%s was not installed", ds_) return Install._handle_and_return_installed_items( ds, installed_items, failed_items, save) if source and path and len(path) > 1: raise ValueError( "install needs a single PATH when source is provided. " "Was given mutliple PATHs: %s" % str(path)) # parameter constraints: if not source: raise InsufficientArgumentsError( "a `source` is required for installation") # code below deals with a single path only path = path[0] if path else None if source == path: # even if they turn out to be identical after resolving symlinks # and more sophisticated witchcraft, it would still happily say # "it appears to be already installed", so we just catch an # obviously pointless input combination raise ValueError( "installation `source` and destination `path` are identical. " "If you are trying to add a subdataset simply use `save` %s".format( path)) # resolve the target location (if local) against the provided dataset # or CWD: if path is not None: # Should work out just fine for regular paths, so no additional # conditioning is necessary try: path_ri = RI(path) except Exception as e: raise ValueError( "invalid path argument {}: ({})".format(path, exc_str(e))) try: # Wouldn't work for SSHRI ATM, see TODO within SSHRI # yoh: path should be a local path, and mapping note within # SSHRI about mapping localhost:path to path is kinda # a peculiar use-case IMHO path = resolve_path(path_ri.localpath, dataset) # any `path` argument that point to something local now # resolved and is no longer a URL except ValueError: # URL doesn't point to a local something # so we have an actual URL in `path`. Since this is valid as a # single positional argument, `source` has to be None at this # point. if is_datalad_compat_ri(path) and source is None: # we have an actual URL -> this should be the source lgr.debug( "Single argument given to install, that doesn't seem to " "be a local path. " "Assuming the argument identifies a source location.") source = path path = None else: # `path` is neither a valid source nor a local path. # TODO: The only thing left is a known subdataset with a # name, that is not a path; Once we correctly distinguish # between path and name of a submodule, we need to consider # this. # For now: Just raise raise ValueError("Invalid path argument {0}".format(path)) # `path` resolved, if there was any. # Possibly do conversion from source into a git-friendly url # luckily GitRepo will undo any fancy file:/// url to make use of Git's # optimization for local clones.... source = _get_git_url_from_source(source) lgr.debug("Resolved source: {0}".format(source)) # TODO: we probably need to resolve source, if it is a local path; # expandpath, normpath, ... Where exactly is the point to do it? # derive target from source: if path is None: # we got nothing but a source. do something similar to git clone # and derive the path from the source and continue lgr.debug( "Neither dataset nor target installation path provided. " "Deriving destination path from given source %s", source) path = _get_installationpath_from_url(source) # since this is a relative `path`, resolve it: path = resolve_path(path, dataset) # there is no other way -- my intoxicated brain tells me assert(path is not None) lgr.debug("Resolved installation target: {0}".format(path)) destination_dataset = Dataset(path) if destination_dataset.is_installed(): # this should not be, check if this is an error, or a reinstall # from the same source # this is where we would have installed this from candidate_sources = _get_flexible_source_candidates( source, destination_dataset.path) # this is where it was installed from track_name, track_url = _get_tracking_source(destination_dataset) if track_url in candidate_sources or get_local_file_url(track_url): # TODO: this one breaks "promise" assumptions of the repeated # invocations of install. # yoh thinks that we actually should be the ones to run update # (without merge) after basic # check that it is clean and up-to-date with its super dataset # and if so, not return here but continue with errands (recursive # installation and get_data) so we could provide the same # result if we rerun the same install twice. lgr.info( "%s was already installed from %s. Use `update` to obtain " "latest updates, or `get` or `install` with a path, not URL, " "to (re)fetch data and / or subdatasets", destination_dataset, track_url) return destination_dataset else: raise ValueError("There is already a dataset installed at the " "destination: %s", destination_dataset) ########### # we should know everything necessary by now # actual installation starts ########### # FLOW GUIDE: # four cases: # 1. install into a dataset # 1.1. we install a known subdataset # => git submodule update --init # 1.2. we install an existing repo as a subdataset inplace # => git submodule add + magic # 1.3. we (recursively) try to install implicit subdatasets between # ds and path # 1.4. we install a new subdataset from an explicit source # => git submodule add # 2. we "just" install from an explicit source # => git clone if ds is not None: # FLOW GUIDE: 1. # express the destination path relative to the root of # the dataset relativepath = relpath(path, start=ds.path) if relativepath.startswith(pardir): raise ValueError("installation path outside dataset " "({0})".format(path)) lgr.debug("Resolved installation target relative to dataset " "{0}: {1}".format(ds, relativepath)) # FLOW_GUIDE 1.4. lgr.info("Installing subdataset from '{0}' at: {0}".format( source, relativepath)) destination_dataset = _install_subds_from_flexible_source( ds, relativepath, source, reckless) else: # FLOW GUIDE: 2. lgr.info("Installing dataset at {0} from {1}".format(path, source)) # Currently assuming there is nothing at the target to deal with # and rely on failures raising from the git call ... # We possibly need to consider /.git URL candidate_sources = _get_flexible_source_candidates(source) _clone_from_any_source(candidate_sources, destination_dataset.path) # FLOW GUIDE: All four cases done. if not destination_dataset.is_installed(): # XXX shouldn't we just fail!? (unless some explicit --skip-failing?) lgr.error("Installation failed.") return None _handle_possible_annex_dataset(destination_dataset, reckless) lgr.debug("Installation of %s done.", destination_dataset) if not destination_dataset.is_installed(): # log error and don't report as installed item, but don't raise, # since we might be in a process of recursive installation where # a lot of other datasets can still be installed successfully. lgr.error("Installation of {0} failed.".format(destination_dataset)) else: installed_items.append(destination_dataset) # we need to decrease the recursion limit, relative to # subdatasets now subds_recursion_limit = max(0, recursion_limit - 1) \ if isinstance(recursion_limit, int) \ else recursion_limit # Now, recursive calls: if recursive: if description: # yoh: why? especially if we somehow allow for templating them # with e.g. '%s' to catch the subdataset path lgr.warning("Description can't be assigned recursively.") subs = destination_dataset.get_subdatasets( # yes, it does make sense to combine no recursion with # recursion_limit: when the latter is 0 we get no subdatasets # reported, otherwise we always get the 1st-level subs recursive=False, recursion_limit=recursion_limit, absolute=False) if subs: lgr.debug("Obtaining subdatasets of %s: %s", destination_dataset, subs) kwargs = common_kwargs.copy() kwargs['recursion_limit'] = subds_recursion_limit rec_installed = Get.__call__( subs, # all at once dataset=destination_dataset, # TODO expose this # yoh: exactly! #annex_get_opts=annex_get_opts, **kwargs ) # TODO do we want to filter this so `install` only returns # the datasets? if isinstance(rec_installed, list): installed_items.extend(rec_installed) else: installed_items.append(rec_installed) if get_data: lgr.debug("Getting data of {0}".format(destination_dataset)) kwargs = common_kwargs.copy() kwargs['recursive'] = False destination_dataset.get(curdir, **kwargs) return Install._handle_and_return_installed_items( ds, installed_items, failed_items, save) @staticmethod def _handle_and_return_installed_items(ds, installed_items, failed_items, save): if save and ds is not None: _save_installed_datasets(ds, installed_items) if failed_items: msg = '' for act, l in (("succeeded", installed_items), ("failed", failed_items)): if not l: continue if msg: msg += ', and ' msg += "%s %s" % ( single_or_plural("dataset", "datasets", len(l), include_count=True), act) if ds: paths = [relpath(i.path, ds.path) if hasattr(i, 'path') else i if not i.startswith(ds.path) else relpath(i, ds.path) for i in l] else: paths = l msg += " (%s)" % (", ".join(map(str, paths))) msg += ' to install' # we were asked for multiple installations if installed_items or len(failed_items) > 1: raise IncompleteResultsError( results=installed_items, failed=failed_items, msg=msg) else: raise InstallFailedError(msg=msg) return installed_items[0] \ if len(installed_items) == 1 else installed_items @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui if res is None: res = [] if not isinstance(res, list): res = [res] if not len(res): ui.message("Nothing was installed") return items = '\n'.join(map(str, res)) msg = "{n} installed {obj} available at\n{items}".format( obj='items are' if len(res) > 1 else 'item is', n=len(res), items=items) ui.message(msg) datalad-0.4.1/datalad/distribution/move.py000066400000000000000000000021051301115270200205250ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for moving dataset content """ __docformat__ = 'restructuredtext' from os import curdir from os.path import join as opj, abspath, expanduser, expandvars, exists from datalad.support.param import Parameter from datalad.support.constraints import EnsureStr, EnsureNone from datalad.interface.base import Interface class Move(Interface): """Move dataset content.""" _params_ = dict( path=Parameter( args=('path',), nargs='?', doc="name of or path to the content to be updated", constraints=EnsureStr() | EnsureNone())) @staticmethod def __call__(path=curdir): raise NotImplementedError("TODO") datalad-0.4.1/datalad/distribution/publish.py000066400000000000000000000351651301115270200212410ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for dataset (component) publishing """ import logging from os.path import join as opj from six import string_types from datalad.interface.base import Interface from datalad.support.param import Parameter from datalad.support.constraints import EnsureStr from datalad.support.constraints import EnsureNone from datalad.support.constraints import EnsureChoice from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import InsufficientArgumentsError from datalad.dochelpers import exc_str from datalad.utils import assure_list from .dataset import EnsureDataset from .dataset import Dataset from .dataset import datasetmethod from .dataset import require_dataset __docformat__ = 'restructuredtext' lgr = logging.getLogger('datalad.distribution.publish') def _log_push_info(pi_list): from git.remote import PushInfo as PI if pi_list: for push_info in pi_list: if (push_info.flags & PI.ERROR) == PI.ERROR: lgr.error(push_info.summary) else: lgr.info(push_info.summary) else: lgr.warning("Nothing was pushed.") class Publish(Interface): """Publish a dataset (e.g. to a web server) This makes the current state of a dataset available to a sibling of the dataset. That sibling needs to be known to the dataset before. `create_publication_target` commands are meant to set up such a sibling, but generally you can publish to any sibling added to a dataset. Publishing may or may not include subdatasets. By default file handles are published without their actual content, but they can be published including the content, of course. """ # TODO: Figure out, how to tell about tracking branch/upstream # (and the respective remote) # - it is used, when no destination is given # - it is configured to be the given destination, if there was no # upstream set up before, so you can use just "datalad publish" next # time. # TODO: Doc! _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), metavar='DATASET', doc="""specify the dataset to publish. If no dataset is given, an attempt is made to identify the dataset based on the current working directory""", constraints=EnsureDataset() | EnsureNone()), to=Parameter( args=("--to",), metavar='DEST', doc="""sibling name identifying the publication target. If no destination is given an attempt is made to identify the target based on the dataset's configuration""", # TODO: See TODO at top of class! constraints=EnsureStr() | EnsureNone()), recursive=Parameter( args=("-r", "--recursive"), action="store_true", doc="recursively publish all subdatasets of the dataset. In order " "to recursively publish with all data, use '.' as `path` in " "combination with `recursive`"), since=Parameter( args=("--since",), constraints=EnsureStr() | EnsureNone(), doc="""When publishing dataset(s), specifies commit (treeish, tag, etc) from which to look for changes to decide either updated publishing is necessary for this and which children. If empty argument is provided, then we will always run publish command. By default, would take from the previously published to that remote/sibling state (for the current branch)"""), skip_failing=Parameter( args=("--skip-failing",), action="store_true", doc="skip failing sub-datasets (incombination with `recursive`) " "instead of failing altogether"), path=Parameter( args=("path",), metavar='PATH', doc="path(s), that may point to file handle(s) to publish including " "their actual content or to subdataset(s) to be published. If a " "file handle is published with its data, this implicitly means " "to also publish the (sub)dataset it belongs to. '.' as a path " "is treated in a special way in the sense, that it is passed " "to subdatasets in case `recursive` is also given.", constraints=EnsureStr() | EnsureNone(), nargs='*'), annex_copy_opts=Parameter( args=("--annex-copy-opts",), metavar='OPT_STR', # better name? doc="options passed to 'annex copy'", constraints=EnsureStr() | EnsureNone(),)) @staticmethod @datasetmethod(name='publish') def __call__(path=None, dataset=None, to=None, recursive=False, since=None, skip_failing=False, annex_copy_opts=None): ds = require_dataset(dataset, check_installed=True, purpose='publication') assert(ds.repo is not None) path = assure_list(path) # figure out, what to publish from what (sub)dataset: publish_this = False # whether to publish `ds` publish_files = [] # which files to publish by `ds` expl_subs = set() # subdatasets to publish explicitly publish_subs = dict() # collect what to publish from subdatasets if not path: # publish `ds` itself, if nothing else is given: publish_this = True else: for p in path: subdatasets = ds.get_subdatasets() if p in subdatasets: # p is a subdataset, that needs to be published itself expl_subs.add(p) else: try: d = ds.get_containing_subdataset(p) except ValueError as e: # p is not in ds => skip: lgr.warning(str(e) + " - Skipped.") continue if d == ds: # p needs to be published from ds publish_this = True publish_files.append(p) else: # p belongs to subds `d` if not publish_subs[d.path]: publish_subs[d.path] = dict() if not publish_subs[d.d.path]['files']: publish_subs[d.d.path]['files'] = list() publish_subs[d.path]['dataset'] = d publish_subs[d.path]['files'].append(p) if publish_this: # Note: we need an upstream remote, if there's none given. We could # wait for git push to complain, but we need to explicitly figure it # out for pushing annex branch anyway and we might as well fail # right here. track_remote, track_branch = ds.repo.get_tracking_branch() # keep `to` in case it's None for passing to recursive calls: dest_resolved = to if to is None: if track_remote: dest_resolved = track_remote else: # we have no remote given and no upstream => fail raise InsufficientArgumentsError( "No known default target for " "publication and none given.") subds_prev_hexsha = {} if recursive: all_subdatasets = ds.get_subdatasets(fulfilled=True) subds_to_consider = \ Publish._get_changed_datasets( ds.repo, all_subdatasets, dest_resolved, since=since) \ if publish_this \ else all_subdatasets # if we were returned a dict, we got subds_prev_hexsha if isinstance(subds_to_consider, dict): subds_prev_hexsha = subds_to_consider for subds_path in subds_to_consider: if path and '.' in path: # we explicitly are passing '.' to subdatasets in case of # `recursive`. Therefore these datasets are going into # `publish_subs`, instead of `expl_subs`: sub = Dataset(opj(ds.path, subds_path)) publish_subs[sub.path] = dict() publish_subs[sub.path]['dataset'] = sub publish_subs[sub.path]['files'] = ['.'] else: # we can recursively publish only, if there actually # is something expl_subs.add(subds_path) published, skipped = [], [] for dspath in sorted(expl_subs): # these datasets need to be pushed regardless of additional paths # pointing inside them # due to API, this may not happen when calling publish with paths, # therefore force it. # TODO: There might be a better solution to avoid two calls of # publish() on the very same Dataset instance ds_ = Dataset(opj(ds.path, dspath)) try: # we could take local diff for the subdataset # but may be we could just rely on internal logic within # subdataset to figure out what it needs to publish. # But we need to pass empty string one inside as is pkw = {} if since == '': pkw['since'] = since else: # pass previous state for that submodule if known pkw['since'] = subds_prev_hexsha.get(dspath, None) published_, skipped_ = ds_.publish(to=to, recursive=recursive, **pkw) published += published_ skipped += skipped_ except Exception as exc: if not skip_failing: raise lgr.warning("Skipped %s: %s", ds.path, exc_str(exc)) skipped += [ds_] for d in publish_subs: # recurse into subdatasets # TODO: need to fetch. see above publish_subs[d]['dataset'].repo.fetch(remote=to) published_, skipped_ = publish_subs[d]['dataset'].publish( to=to, path=publish_subs[d]['files'], recursive=recursive, annex_copy_opts=annex_copy_opts) published += published_ skipped += skipped_ if publish_this: # upstream branch needed for update (merge) and subsequent push, # in case there is no. # no tracking branch yet? set_upstream = track_branch is None # is `to` an already known remote? if dest_resolved not in ds.repo.get_remotes(): # unknown remote raise ValueError("No sibling '%s' found." % dest_resolved) lgr.info("Publishing {0} to {1}".format(ds, dest_resolved)) # we now know where to push to: _log_push_info(ds.repo.push(remote=dest_resolved, refspec=ds.repo.get_active_branch(), set_upstream=set_upstream)) # push annex branch: if isinstance(ds.repo, AnnexRepo): ds.repo.fetch(remote=dest_resolved) ds.repo.merge_annex(dest_resolved) _log_push_info(ds.repo.push(remote=dest_resolved, refspec="git-annex:git-annex")) # we need to fetch # TODO # Note: This is about a gitpython issue as well as something about # annex -> might mean, that we need to do it in case we pushed an annex branch only. # Apparently, we can annex copy new files only, after this fetch. Figure it out! ds.repo.fetch(remote=dest_resolved) published.append(ds) if publish_files or annex_copy_opts: if not isinstance(ds.repo, AnnexRepo): raise RuntimeError( "Cannot publish content of something, that is not " "part of an annex. ({0})".format(ds)) lgr.info("Publishing data of dataset {0} ...".format(ds)) published += ds.repo.copy_to(files=publish_files, remote=dest_resolved, options=annex_copy_opts) return published, skipped @staticmethod def result_renderer_cmdline(results, args): from datalad.ui import ui for res, res_label in zip(results, ('published', 'skipped')): if not res: if res_label == 'published': ui.message("Nothing was %s" % res_label) continue msg = "{n} {obj} {res_label}:\n".format( obj='items were' if len(res) > 1 else 'item was', n=len(res), res_label=res_label) for item in res: if isinstance(item, Dataset): msg += "Dataset: %s\n" % item.path else: msg += "File: %s\n" % item ui.message(msg) @staticmethod def _get_changed_datasets(repo, all_subdatasets, to, since=None): if since == '' or not all_subdatasets: # we are instructed to publish all return all_subdatasets if since is None: # default behavior - only updated since last update # so we figure out what was the last update # XXX here we assume one to one mapping of names from local branches # to the remote active_branch = repo.get_active_branch() since = '%s/%s' % (to, active_branch) if since not in repo.get_remote_branches(): # we did not publish it before - so everything must go return all_subdatasets lgr.debug("Checking diff since %s for %s" % (since, all_subdatasets)) diff = repo.repo.commit().diff(since, all_subdatasets) for d in diff: # not sure if it could even track renames of subdatasets # but let's "check" assert(d.a_path == d.b_path) return dict((d.b_path, d.b_blob.hexsha if d.b_blob else None) for d in diff) datalad-0.4.1/datalad/distribution/rewrite_urls.py000066400000000000000000000066301301115270200223140ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for modifying URLs of subdatasets """ __docformat__ = 'restructuredtext' import logging from os.path import join as opj from datalad.support.param import Parameter from datalad.support.constraints import EnsureStr, EnsureNone from datalad.support.gitrepo import GitRepo from datalad.interface.base import Interface from datalad.distribution.dataset import Dataset, EnsureDataset, \ datasetmethod, require_dataset lgr = logging.getLogger('datalad.distribution.rewrite_urls') def get_module_parser(repo): from git import GitConfigParser gitmodule_path = opj(repo.path, ".gitmodules") # TODO: What does constructor of GitConfigParser, in case file doesn't exist? #if exists(gitmodule_path): parser = GitConfigParser(gitmodule_path) parser.read() return parser class RewriteURLs(Interface): """Rewrite the URLs of sub-datasets of a dataset """ _params_ = dict( url=Parameter( args=("url",), doc="a template for building the URLs of the subdatasets " "List of currently available placeholders:\n" "%%NAME\tthe name of the subdataset, where slashes are replaced by " "dashes", constraints=EnsureStr()), dataset=Parameter( args=("-d", "--dataset",), doc="""specify the dataset to update. If no dataset is given, an attempt is made to identify the dataset based on the current working directory""", constraints=EnsureDataset() | EnsureNone()), recursive=Parameter( args=("-r", "--recursive"), action="store_true", doc="recursively modify all subdataset URLs of `dataset` "),) # TODO: User interaction. Allow for skipping and editing on a per # subdataset basis. Therefore some --mode option (see below). Additionally, # this leads to URL being optional, so no URL given means to # edit per subdataset # mode=Parameter( # args=("--mode",), # doc="", # constraints=EnsureChoice(["all", "ask"]),) @staticmethod @datasetmethod(name='rewrite_urls') def __call__(url, dataset=None, recursive=False): # shortcut ds = require_dataset( dataset, check_installed=True, purpose='modifying subdataset URLs') assert(ds.repo is not None) repos_to_update = [ds.repo] if recursive: repos_to_update += [GitRepo(opj(ds.path, sub_path)) for sub_path in ds.get_subdatasets(recursive=True)] for dataset_repo in repos_to_update: parser = get_module_parser(dataset_repo) for submodule_section in parser.sections(): submodule_name = submodule_section[11:-1] parser.set_value(submodule_section, "url", url.replace("%NAME", submodule_name.replace("/", "-"))) return # TODO: return value? datalad-0.4.1/datalad/distribution/tests/000077500000000000000000000000001301115270200203515ustar00rootroot00000000000000datalad-0.4.1/datalad/distribution/tests/__init__.py000066400000000000000000000007041301115270200224630ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Distribution tests """ __docformat__ = 'restructuredtext' datalad-0.4.1/datalad/distribution/tests/test_add.py000066400000000000000000000211571301115270200225200ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test add action """ from os import pardir from os.path import join as opj from datalad.api import create from datalad.api import add from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.exceptions import FileNotInRepositoryError from datalad.support.exceptions import CommandError from datalad.tests.utils import ok_ from datalad.tests.utils import eq_ from datalad.tests.utils import with_tempfile from datalad.tests.utils import with_tree from datalad.tests.utils import assert_raises from datalad.tests.utils import assert_false from datalad.tests.utils import assert_in from datalad.tests.utils import serve_path_via_http from datalad.utils import chpwd from datalad.utils import _path_ from ..dataset import Dataset @with_tempfile(mkdir=True) def test_add_insufficient_args(path): # no argument: assert_raises(InsufficientArgumentsError, add) # no `path`, no `source`: assert_raises(InsufficientArgumentsError, add, dataset=path) # not in a dataset, no dataset given: with chpwd(path): assert_raises(InsufficientArgumentsError, add, path="some") ds = Dataset(path) ds.create() assert_raises(FileNotInRepositoryError, ds.add, opj(pardir, 'path', 'outside')) tree_arg = dict(tree={'test.txt': 'some', 'test_annex.txt': 'some annex', 'test1.dat': 'test file 1', 'test2.dat': 'test file 2', 'dir': {'testindir': 'someother', 'testindir2': 'none'}}) @with_tree(**tree_arg) def test_add_files(path): ds = Dataset(path) ds.create(force=True) test_list_1 = ['test_annex.txt'] test_list_2 = ['test.txt'] test_list_3 = ['test1.dat', 'test2.dat'] test_list_4 = [opj('dir', 'testindir'), opj('dir', 'testindir2')] all_files = test_list_1 + test_list_2 + test_list_3 + test_list_4 unstaged = set(all_files) staged = set() for arg in [(test_list_1[0], False), (test_list_2[0], True), (test_list_3, False), (test_list_4, False)]: # special case 4: give the dir: if arg[0] == test_list_4: result = ds.add('dir', to_git=arg[1], save=False, if_dirty='ignore') else: result = ds.add(arg[0], to_git=arg[1], save=False, if_dirty='ignore') # TODO eq_(result, arg[0]) # added, but not committed: ok_(ds.repo.dirty) # get sets for comparison: annexed = set(ds.repo.get_annexed_files()) indexed = set(ds.repo.get_indexed_files()) # ignore the initial config file in index: indexed.remove('.datalad/config') if isinstance(arg[0], list): for x in arg[0]: unstaged.remove(x) staged.add(x) else: unstaged.remove(arg[0]) staged.add(arg[0]) # added, but nothing else was: eq_(staged, indexed) ok_(unstaged.isdisjoint(annexed)) ok_(unstaged.isdisjoint(indexed)) @with_tree(**tree_arg) def test_add_recursive(path): ds = Dataset(path) ds.create(force=True, save=False) ds.create('dir', force=True, if_dirty='ignore') ds.save("Submodule added.") # TODO: CommandError to something meaningful # fail without recursive: assert_raises(CommandError, ds.add, opj('dir', 'testindir'), recursive=False) # fail with recursion limit too low: assert_raises(CommandError, ds.add, opj('dir', 'testindir'), recursive=True, recursion_limit=0) # add while also instructing annex to add in parallel 2 jobs (smoke testing # for that effect ATM) added1 = ds.add(opj('dir', 'testindir'), recursive=True, jobs=2) # added to annex, so annex output record eq_(added1, [{'file': _path_('dir/testindir'), 'command': 'add', 'key': 'MD5E-s9--3f0f870d18d6ba60a79d9463ff3827ea', 'success': True}]) assert_in('testindir', Dataset(opj(path, 'dir')).repo.get_annexed_files()) added2 = ds.add(opj('dir', 'testindir2'), recursive=True, to_git=True) # added to git, so parsed git output record eq_(added2, [{'success': True, 'file': _path_('dir/testindir2')}]) assert_in('testindir2', Dataset(opj(path, 'dir')).repo.get_indexed_files()) # We used to fail to add to pure git repository, but now it should all be # just fine subds = ds.create('git-sub', no_annex=True) with open(opj(subds.path, 'somefile.txt'), "w") as f: f.write("bla bla") result = ds.add(opj('git-sub', 'somefile.txt'), recursive=True, to_git=False) eq_(result, [{'file': _path_('git-sub/somefile.txt'), 'success': True}]) @with_tree(**tree_arg) def test_relpath_add(path): ds = Dataset(path).create(force=True) with chpwd(opj(path, 'dir')): eq_(add('testindir', if_dirty='ignore')[0]['file'], opj('dir', 'testindir')) # and now add all add('..') # auto-save enabled assert_false(ds.repo.dirty) @with_tree(tree={'file1.txt': 'whatever 1', 'file2.txt': 'whatever 2', 'file3.txt': 'whatever 3', 'file4.txt': 'whatever 4', 'file5.txt': 'whatever 5', 'file6.txt': 'whatever 6', 'file7.txt': 'whatever 7'}) @serve_path_via_http @with_tempfile(mkdir=True) def test_add_source(path, url, ds_dir): from os import listdir from datalad.support.network import RI urls = [RI(url + f) for f in listdir(path)] ds = Dataset(ds_dir).create() eq_(len(ds.repo.get_annexed_files()), 0) # add a remote source to git => fail: assert_raises(NotImplementedError, ds.add, source=urls[0], to_git=True) # annex add a remote source: ds.add(source=urls[0]) eq_(len(ds.repo.get_annexed_files()), 1) # add two remote source an give local names: ds.add(path=['local1.dat', 'local2.dat'], source=urls[1:3]) annexed = ds.repo.get_annexed_files() eq_(len(annexed), 3) assert_in('local1.dat', annexed) assert_in('local2.dat', annexed) # add a second source for one of them ds.add(path='local1.dat', source=urls[3]) eq_(len(annexed), 3) whereis_dict = ds.repo.whereis('local1.dat', output='full') reg_urls = [whereis_dict[uuid]['urls'] for uuid in whereis_dict if not whereis_dict[uuid]['here']] eq_(len(reg_urls), 1) # one remote for 'local1.dat', that is not "here" eq_({str(urls[1]), str(urls[3])}, set(reg_urls[0])) # just to be sure compare to 'local2.dat': whereis_dict = ds.repo.whereis('local2.dat', output='full') reg_urls = [whereis_dict[uuid]['urls'] for uuid in whereis_dict if not whereis_dict[uuid]['here']] eq_(len(reg_urls), 1) # one remote for 'local2.dat', that is not "here" eq_([urls[2]], reg_urls[0]) # provide more paths than sources: # report failure on non-existing 'local4.dat': result = ds.add(path=['local3.dat', 'local4.dat'], source=urls[4]) ok_(all([r['success'] is False and r['note'] == 'not found' for r in result if r['file'] == 'local4.dat'])) with open(opj(ds.path, 'local4.dat'), 'w') as f: f.write('local4 content') ds.add(path=['local3.dat', 'local4.dat'], source=urls[4]) annexed = ds.repo.get_annexed_files() eq_(len(annexed), 5) assert_in('local3.dat', annexed) assert_in('local4.dat', annexed) # 'local3.dat' has a remote source whereis_dict = ds.repo.whereis('local3.dat', output='full') reg_urls = [whereis_dict[uuid]['urls'] for uuid in whereis_dict if not whereis_dict[uuid]['here']] eq_(len(reg_urls), 1) # one remote for 'local3.dat', that is not "here" eq_([urls[4]], reg_urls[0]) # 'local4.dat' has no remote source whereis_dict = ds.repo.whereis('local4.dat', output='full') reg_urls = [whereis_dict[uuid]['urls'] for uuid in whereis_dict if not whereis_dict[uuid]['here']] eq_(len(reg_urls), 0) # provide more sources than paths: ds.add('local5.dat', source=urls[5:]) annexed = ds.repo.get_annexed_files() assert_in('local5.dat', annexed) eq_(len(annexed), 5 + len(urls[5:])) # Note: local4.dat didn't come from an url, # but 'local1.dat' consumes two urls eq_(len(annexed), len(urls)) # all files annexed (-2 for '.git' and '.datalad'): eq_(len(annexed), len(listdir(ds.path)) - 2) datalad-0.4.1/datalad/distribution/tests/test_add_sibling.py000066400000000000000000000121621301115270200242230ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test adding sibling(s) to a dataset """ from os.path import join as opj, basename from datalad.api import install, add_sibling from datalad.support.gitrepo import GitRepo from datalad.tests.utils import with_tempfile, assert_in, with_testrepos from datalad.tests.utils import assert_raises from nose.tools import eq_, ok_ @with_testrepos('submodule_annex', flavors=['local']) @with_tempfile(mkdir=True) def test_add_sibling(origin, repo_path): # prepare src source = install(repo_path, source=origin, recursive=True)[0] # pollute config depvar = 'remote.test-remote.datalad-publish-depends' source.config.add(depvar, 'stupid', where='local') res = add_sibling(dataset=source, name="test-remote", url="http://some.remo.te/location", publish_depends=['r1', 'r2'], force=True) eq_(source.config.get(depvar, None), ('r1', 'r2')) eq_(res, [basename(source.path)]) assert_in("test-remote", source.repo.get_remotes()) eq_("http://some.remo.te/location", source.repo.get_remote_url("test-remote")) # doing it again doesn't do anything res = add_sibling(dataset=source, name="test-remote", url="http://some.remo.te/location") eq_(res, []) assert_in("test-remote", source.repo.get_remotes()) eq_("http://some.remo.te/location", source.repo.get_remote_url("test-remote")) # fail with conflicting url: with assert_raises(RuntimeError) as cm: add_sibling(dataset=source, name="test-remote", url="http://some.remo.te/location/elsewhere") assert_in("""'test-remote' already exists with conflicting URL""", str(cm.exception)) # don't fail with conflicting url, when using force: res = add_sibling(dataset=source, name="test-remote", url="http://some.remo.te/location/elsewhere", force=True) eq_(res, [basename(source.path)]) eq_("http://some.remo.te/location/elsewhere", source.repo.get_remote_url("test-remote")) # add a push url without force fails, since in a way the fetch url is the # configured push url, too, in that case: with assert_raises(RuntimeError) as cm: add_sibling(dataset=source, name="test-remote", url="http://some.remo.te/location/elsewhere", pushurl="ssh://push.it", force=False) assert_in("""'test-remote' already exists with conflicting URL""", str(cm.exception)) # add push url (force): res = add_sibling(dataset=source, name="test-remote", url="http://some.remo.te/location/elsewhere", pushurl="ssh://push.it", force=True) eq_(res, [basename(source.path)]) eq_("http://some.remo.te/location/elsewhere", source.repo.get_remote_url("test-remote")) eq_("ssh://push.it", source.repo.get_remote_url("test-remote", push=True)) # recursively: res = add_sibling(dataset=source, name="test-remote", url="http://some.remo.te/location/%NAME", pushurl="ssh://push.it/%NAME", recursive=True, force=True) eq_(set(res), {basename(source.path), opj(basename(source.path), "subm 1"), opj(basename(source.path), "subm 2")}) for repo in [source.repo, GitRepo(opj(source.path, "subm 1")), GitRepo(opj(source.path, "subm 2"))]: assert_in("test-remote", repo.get_remotes()) url = repo.get_remote_url("test-remote") pushurl = repo.get_remote_url("test-remote", push=True) ok_(url.startswith("http://some.remo.te/location/" + basename(source.path))) ok_(url.endswith(basename(repo.path))) ok_(pushurl.startswith("ssh://push.it/" + basename(source.path))) ok_(pushurl.endswith(basename(repo.path))) # recursively without template: res = add_sibling(dataset=source, name="test-remote-2", url="http://some.remo.te/location", pushurl="ssh://push.it/", recursive=True, force=True) eq_(set(res), {basename(source.path), opj(basename(source.path), "subm 1"), opj(basename(source.path), "subm 2")}) for repo in [source.repo, GitRepo(opj(source.path, "subm 1")), GitRepo(opj(source.path, "subm 2"))]: assert_in("test-remote-2", repo.get_remotes()) url = repo.get_remote_url("test-remote-2") pushurl = repo.get_remote_url("test-remote-2", push=True) ok_(url.startswith("http://some.remo.te/location")) ok_(pushurl.startswith("ssh://push.it/")) if repo != source.repo: ok_(url.endswith('/' + basename(repo.path))) ok_(pushurl.endswith(basename(repo.path))) datalad-0.4.1/datalad/distribution/tests/test_create.py000066400000000000000000000153521301115270200232330ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test create action """ import os from os.path import join as opj from ..dataset import Dataset from datalad.api import create from datalad.api import uninstall from datalad.utils import chpwd from datalad.utils import rmtree from datalad.cmd import Runner from datalad.support.exceptions import CommandError from datalad.tests.utils import with_tempfile from datalad.tests.utils import eq_ from datalad.tests.utils import ok_ from datalad.tests.utils import assert_not_in from datalad.tests.utils import assert_in from datalad.tests.utils import assert_raises from datalad.tests.utils import assert_equal from datalad.tests.utils import ok_clean_git from datalad.tests.utils import with_tree _dataset_hierarchy_template = { 'origin': { 'file1': '', 'sub': { 'file2': 'file2', 'subsub': { 'file3': 'file3'}}}} @with_tempfile(mkdir=True) @with_tempfile def test_create_raises(path, outside_path): ds = Dataset(path) # incompatible arguments (annex only): assert_raises(ValueError, ds.create, no_annex=True, description='some') assert_raises(ValueError, ds.create, no_annex=True, annex_opts=['some']) assert_raises(ValueError, ds.create, no_annex=True, annex_init_opts=['some']) with open(opj(path, "somefile.tst"), 'w') as f: f.write("some") # non-empty without `force`: assert_raises(ValueError, ds.create, force=False) # non-empty with `force`: ds.create(force=True) # create sub outside of super: assert_raises(ValueError, ds.create, outside_path) # create a sub: ds.create('sub') # fail when doing it again without `force`: assert_raises(ValueError, ds.create, 'sub') @with_tempfile @with_tempfile def test_create_curdir(path, path2): with chpwd(path, mkdir=True): create() ds = Dataset(path) ok_(ds.is_installed()) ok_clean_git(ds.path, annex=True) with chpwd(path2, mkdir=True): create(no_annex=True) ds = Dataset(path2) ok_(ds.is_installed()) ok_clean_git(ds.path, annex=False) @with_tempfile def test_create(path): ds = Dataset(path) ds.create(description="funny", native_metadata_type=['bim', 'bam', 'bum'], shared_access='world') ok_(ds.is_installed()) ok_clean_git(ds.path, annex=True) # check default backend eq_(ds.config.get("annex.backends"), 'MD5E') eq_(ds.config.get("core.sharedrepository"), '2') runner = Runner() # check description in `info` cmd = ['git-annex', 'info'] cmlout = runner.run(cmd, cwd=path) assert_in('funny [here]', cmlout[0]) # check datset ID eq_(ds.config.get_value('datalad.dataset', 'id'), ds.id) assert_equal( ds.config.get_value('datalad.metadata', 'nativetype'), ('bim', 'bam', 'bum')) @with_tempfile def test_create_sub(path): ds = Dataset(path) ds.create() # 1. create sub and add to super: subds = ds.create("some/what/deeper") ok_(isinstance(subds, Dataset)) ok_(subds.is_installed()) ok_clean_git(subds.path, annex=True) # subdataset is known to superdataset: assert_in("some/what/deeper", ds.get_subdatasets()) # and was committed: ok_clean_git(ds.path) # subds finds superdataset ok_(subds.get_superdataset() == ds) # 2. create sub without adding to super: subds2 = Dataset(opj(path, "someother")).create() ok_(isinstance(subds2, Dataset)) ok_(subds2.is_installed()) ok_clean_git(subds2.path, annex=True) # unknown to superdataset: assert_not_in("someother", ds.get_subdatasets()) # 3. create sub via super: subds3 = ds.create("third", no_annex=True) ok_(isinstance(subds3, Dataset)) ok_(subds3.is_installed()) ok_clean_git(subds3.path, annex=False) assert_in("third", ds.get_subdatasets()) @with_tree(tree=_dataset_hierarchy_template) def test_create_subdataset_hierarchy_from_top(path): # how it would look like to overlay a subdataset hierarchy onto # an existing directory tree ds = Dataset(opj(path, 'origin')).create(force=True) ok_(ds.is_installed()) # the following create() calls need to ignore the dirty state # of the parent, otherwise they would auto-save it and turn # everything into one big dataset subds = ds.create('sub', force=True, if_dirty='ignore') ok_(subds.is_installed()) subsubds = subds.create('subsub', force=True, if_dirty='ignore') ok_(subsubds.is_installed()) ds.save(recursive=True, auto_add_changes=True) ok_clean_git(ds.path) ok_(ds.id != subds.id != subsubds.id) @with_tempfile def test_nested_create(path): # to document some more organic usage pattern ds = Dataset(path).create() ok_clean_git(ds.path) lvl2relpath = opj('lvl1', 'lvl2') lvl2path = opj(ds.path, lvl2relpath) os.makedirs(lvl2path) os.makedirs(opj(ds.path, 'lvl1', 'empty')) with open(opj(lvl2path, 'file'), 'w') as f: f.write('some') ok_(ds.save(auto_add_changes=True)) # later create subdataset in a fresh dir subds1 = ds.create(opj('lvl1', 'subds')) ok_clean_git(ds.path) eq_(ds.get_subdatasets(), [opj('lvl1', 'subds')]) # later create subdataset in an existing empty dir subds2 = ds.create(opj('lvl1', 'empty')) ok_clean_git(ds.path) # later try to wrap existing content into a new subdataset # but that won't work assert_raises(ValueError, ds.create, lvl2relpath) # even with force, as to do this properly complicated surgery would need to # take place assert_raises(CommandError, ds.create, lvl2relpath, force=True) # only way to make it work is to unannex the content upfront ds.repo._run_annex_command('unannex', annex_options=[opj(lvl2relpath, 'file')]) # nothing to save, git-annex commits the unannex itself ok_(not ds.save()) # still nothing without force # "err='lvl1/lvl2' already exists in the index" assert_raises(ValueError, ds.create, lvl2relpath) # XXX even force doesn't help, because (I assume) GitPython doesn't update # its representation of the Git index properly assert_raises(CommandError, ds.create, lvl2relpath, force=True) # it is not GitPython's state that is at fault here, test with fresh # dataset isnstance ds = Dataset(ds.path) assert_raises(CommandError, ds.create, lvl2relpath, force=True) # it seems we are at fault here rmtree(opj(lvl2path, '.git')) assert_raises(CommandError, ds.repo.add_submodule, lvl2relpath) # despite the failure: assert_in(lvl2relpath, ds.get_subdatasets()) datalad-0.4.1/datalad/distribution/tests/test_create_github.py000066400000000000000000000047531301115270200246000ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test create publication target github""" from os.path import join as opj # this must with with and without pygithub from datalad.api import create_sibling_github from datalad.api import Dataset from datalad.support.exceptions import MissingExternalDependency from datalad.tests.utils import with_tempfile from nose.tools import assert_raises, assert_in, assert_true, assert_false, \ assert_not_in, assert_equal from nose import SkipTest try: import github as gh except ImportError: # make sure that the command complains too assert_raises(MissingExternalDependency, create_sibling_github, 'some') raise SkipTest @with_tempfile def test_invalid_call(path): # no dataset assert_raises(ValueError, create_sibling_github, 'bogus', dataset=path) ds = Dataset(path).create() # no user assert_raises(gh.BadCredentialsException, ds.create_sibling_github, 'bogus', github_user='') @with_tempfile def test_dont_trip_over_missing_subds(path): ds1 = Dataset(opj(path, 'ds1')).create() ds2 = Dataset(opj(path, 'ds2')).create() subds2 = ds1.install(source=ds2.path, path='subds2') assert_true(subds2.is_installed()) assert_in('subds2', ds1.get_subdatasets()) subds2.uninstall(remove_handles=True, remove_history=True) assert_in('subds2', ds1.get_subdatasets()) assert_false(subds2.is_installed()) # this will deinit the submodule ds1.save(files=['subds2']) # see if it wants to talk to github (and fail), or if it trips over something # before assert_raises(gh.BadCredentialsException, ds1.create_sibling_github, 'bogus', recursive=True, github_user='') # inject remote config prior run assert_not_in('github', ds1.repo.get_remotes()) # fail on existing ds1.repo.add_remote('github', 'http://nothere') assert_raises(ValueError, ds1.create_sibling_github, 'bogus', recursive=True, github_user='') # talk to github when existing is OK assert_raises(gh.BadCredentialsException, ds1.create_sibling_github, 'bogus', recursive=True, github_user='', existing='reconfigure') # return happy emptiness when all is skipped assert_equal(ds1.create_sibling_github('bogus', recursive=True, github_user='', existing='skip'), []) datalad-0.4.1/datalad/distribution/tests/test_create_test_dataset.py000066400000000000000000000032231301115270200257710ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test create testdataset helpers """ from glob import glob from os.path import join as opj from datalad.tests.utils import with_tempfile from datalad.tests.utils import assert_raises from datalad.tests.utils import ok_ from datalad.tests.utils import ok_clean_git from datalad.utils import swallow_logs from datalad.utils import chpwd from datalad.distribution.create_test_dataset import _parse_spec from nose.tools import eq_ @with_tempfile(mkdir=True) def test_create(outdir): from datalad.api import create assert_raises(ValueError, create, outdir, description='Precious data', no_annex=True) def test_parse_spec(): eq_(_parse_spec('0/3/-1'), [(0, 0), (3, 3), (0, 1)]) eq_(_parse_spec('4-10'), [(4, 10)]) def test_create_test_dataset(): # rudimentary smoke test from datalad.api import create_test_dataset with swallow_logs(): dss = create_test_dataset(spec='2/1-2') ok_(4 <= len(dss) <= 6) # at least four - two on top level, 1 in each for ds in dss: ok_clean_git(ds, annex=False) # soem of them are annex but we just don't check ok_(len(glob(opj(ds, 'file*')))) @with_tempfile(mkdir=True) def test_create_test_dataset_new_relpath(topdir): from datalad.api import create_test_dataset with swallow_logs(), chpwd(topdir): dss = create_test_dataset('testds', spec='1') eq_(len(dss), 1) datalad-0.4.1/datalad/distribution/tests/test_dataset.py000066400000000000000000000266361301115270200234240ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test Dataset class """ import os import shutil from os.path import join as opj, abspath, normpath from ..dataset import Dataset, EnsureDataset, resolve_path, require_dataset from datalad.api import create from datalad.api import install from datalad.api import get from datalad.consts import LOCAL_CENTRAL_PATH from datalad.utils import chpwd, getpwd, rmtree from datalad.utils import _path_ from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from nose.tools import ok_, eq_, assert_false, assert_equal, assert_true from datalad.tests.utils import with_tempfile, assert_in, with_tree, with_testrepos from datalad.tests.utils import assert_cwd_unchanged from datalad.tests.utils import assert_raises from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.exceptions import PathOutsideRepositoryError def test_EnsureDataset(): c = EnsureDataset() # fails with anything else than a string or a Dataset: assert_raises(ValueError, c, 1) assert_raises(ValueError, c, ['a', 'list']) assert_raises(ValueError, c, (1, 2, 3)) assert_raises(ValueError, c, {"what": "ever"}) # returns Dataset, when string or Dataset passed res = c(opj("some", "path")) ok_(isinstance(res, Dataset)) ok_(isinstance(c(res), Dataset)) ok_(c(res) is res) # Note: Ensuring that string is valid path is not # part of the constraint itself, so not explicitly tested here. @assert_cwd_unchanged @with_tempfile(mkdir=True) def test_resolve_path(somedir): abs_path = abspath(somedir) # just to be sure rel_path = "some" expl_path_cur = opj(os.curdir, rel_path) expl_path_par = opj(os.pardir, rel_path) eq_(resolve_path(abs_path), abs_path) current = getpwd() # no Dataset => resolve using cwd: eq_(resolve_path(abs_path), abs_path) eq_(resolve_path(rel_path), opj(current, rel_path)) eq_(resolve_path(expl_path_cur), normpath(opj(current, expl_path_cur))) eq_(resolve_path(expl_path_par), normpath(opj(current, expl_path_par))) # now use a Dataset as reference: ds = Dataset(abs_path) eq_(resolve_path(abs_path, ds), abs_path) eq_(resolve_path(rel_path, ds), opj(abs_path, rel_path)) eq_(resolve_path(expl_path_cur, ds), normpath(opj(current, expl_path_cur))) eq_(resolve_path(expl_path_par, ds), normpath(opj(current, expl_path_par))) # TODO: test remember/recall more extensive? @with_testrepos(flavors=['local', 'local-url']) @with_tempfile(mkdir=True) def test_register_sibling(remote, path): AnnexRepo(path) ds = Dataset(path) ds.register_sibling('my_sibling', remote) assert_in('my_sibling', ds.repo.get_remotes()) eq_(ds.repo.get_remote_url('my_sibling'), remote) ds.register_sibling('my_other_sibling', remote, publish_url='http://fake.pushurl.com') assert_in('my_other_sibling', ds.repo.get_remotes()) eq_(ds.repo.get_remote_url('my_other_sibling'), remote) # TODO: GitRepo method for push-url! # TODO: Validation! @with_testrepos('.*nested_submodule.*', flavors=['local']) def test_get_subdatasets(path): ds = Dataset(path) eq_(ds.get_subdatasets(), ['sub dataset1']) eq_(ds.get_subdatasets(recursive=True), [ 'sub dataset1/sub sub dataset1/subm 1', 'sub dataset1/sub sub dataset1/subm 2', 'sub dataset1/sub sub dataset1', 'sub dataset1/subm 1', 'sub dataset1/subm 2', 'sub dataset1' ]) eq_(ds.get_subdatasets(recursive=True, recursion_limit=0), []) eq_(ds.get_subdatasets(recursive=True, recursion_limit=1), ['sub dataset1']) eq_(ds.get_subdatasets(recursive=True, recursion_limit=2), [ 'sub dataset1/sub sub dataset1', 'sub dataset1/subm 1', 'sub dataset1/subm 2', 'sub dataset1', ]) # TODO: More Flavors! # TODO: There's something wrong with the nested testrepo! # Fear mongering detected! @with_testrepos('submodule_annex') @with_tempfile(mkdir=True) def test_is_installed(src, path): ds = Dataset(path) assert_false(ds.is_installed()) # get a clone: AnnexRepo(path, src) ok_(ds.is_installed()) # submodule still not installed: subds = Dataset(opj(path, 'subm 1')) assert_false(subds.is_installed()) subds.create() # get the submodule # This would init so there is a .git file with symlink info, which is # as we agreed is more pain than gain, so let's use our install which would # do it right, after all we are checking 'is_installed' ;) # from datalad.cmd import Runner # Runner().run(['git', 'submodule', 'update', '--init', 'subm 1'], cwd=path) with chpwd(path): get('subm 1') ok_(subds.is_installed()) # wipe it out rmtree(ds.path) assert_false(ds.is_installed()) @with_tempfile(mkdir=True) def test_dataset_contructor(path): # dataset needs a path assert_raises(TypeError, Dataset) assert_raises(AttributeError, Dataset, None) dsabs = Dataset(path) # always abspath ok_(os.path.isabs(dsabs.path)) eq_(path, dsabs.path) # no repo eq_(dsabs.repo, None) # same result when executed in that path and using relative paths with chpwd(path): dsrel = Dataset('.') eq_(dsrel.path, dsabs.path) # no repo either, despite directory existing now eq_(dsrel.repo, None) @with_tempfile(mkdir=True) def test_repo_cache(path): ds = Dataset(path) # none by default eq_(ds.repo, None) # make Git repo manually git = GitRepo(path=path, create=True) repo = ds.repo # got one assert_false(repo is None) # stays that one assert_true(ds.repo is repo) # now turn into an annex annex = AnnexRepo(path=path, create=True) # repo instance must change assert_false(ds.repo is repo) assert_true(isinstance(ds.repo, AnnexRepo)) @with_tempfile(mkdir=True) def test_subdatasets(path): # from scratch ds = Dataset(path) assert_false(ds.is_installed()) eq_(ds.get_subdatasets(), []) ds = ds.create() assert_true(ds.is_installed()) eq_(ds.get_subdatasets(), []) # create some file and commit it open(os.path.join(ds.path, 'test'), 'w').write('some') ds.add(path='test') assert_true(ds.is_installed()) ds.save("Hello!", version_tag=1) # Assuming that tmp location was not under a super-dataset eq_(ds.get_superdataset(), None) eq_(ds.get_superdataset(topmost=True), ds) # add itself as a subdataset (crazy, isn't it?) subds = ds.install('subds', source=path) assert_true(subds.is_installed()) eq_(subds.get_superdataset(), ds) eq_(subds.get_superdataset(topmost=True), ds) subdss = ds.get_subdatasets() eq_(len(subdss), 1) eq_(os.path.join(path, subdss[0]), subds.path) eq_(subds.path, ds.get_subdatasets(absolute=True)[0]) eq_(subdss, ds.get_subdatasets(recursive=True)) eq_(subdss, ds.get_subdatasets(fulfilled=True)) # don't have that right now assert_raises(NotImplementedError, ds.get_subdatasets, pattern='sub*') ds.save("with subds", version_tag=2) ds.recall_state(1) assert_true(ds.is_installed()) eq_(ds.get_subdatasets(), []) # very nested subdataset to test topmost subsubds = subds.install(_path_('d1/subds'), source=path) assert_true(subsubds.is_installed()) eq_(subsubds.get_superdataset(), subds) eq_(subsubds.get_superdataset(topmost=True), ds) # verify that '^' alias would work with chpwd(subsubds.path): dstop = Dataset('^') eq_(dstop, ds) # and while in the dataset we still can resolve into central one dscentral = Dataset('///') eq_(dscentral.path, LOCAL_CENTRAL_PATH) with chpwd(ds.path): dstop = Dataset('^') eq_(dstop, ds) # TODO actual submodule checkout is still there @with_tree(tree={'test.txt': 'whatever'}) def test_get_containing_subdataset(path): ds = create(path, force=True) ds.add(path='test.txt') ds.save("Initial commit") subds = ds.create("sub") subsubds = subds.create("subsub") eq_(ds.get_containing_subdataset(opj("sub", "subsub", "some")).path, subsubds.path) # the top of a subdataset belongs to the subdataset eq_(ds.get_containing_subdataset(opj("sub", "subsub")).path, subsubds.path) eq_(GitRepo.get_toppath(opj(ds.path, "sub", "subsub")), subsubds.path) eq_(ds.get_containing_subdataset(opj("sub", "some")).path, subds.path) eq_(ds.get_containing_subdataset("sub").path, subds.path) eq_(ds.get_containing_subdataset("some").path, ds.path) # make sure the subds is found, even when it is not present, but still # known shutil.rmtree(subds.path) eq_(ds.get_containing_subdataset(opj("sub", "some")).path, subds.path) eq_(ds.get_containing_subdataset("sub").path, subds.path) # # but now GitRepo disagrees... eq_(GitRepo.get_toppath(opj(ds.path, "sub")), ds.path) # and this stays, even if we give the mount point directory back os.makedirs(subds.path) eq_(GitRepo.get_toppath(opj(ds.path, "sub")), ds.path) outside_path = opj(os.pardir, "somewhere", "else") assert_raises(PathOutsideRepositoryError, ds.get_containing_subdataset, outside_path) assert_raises(PathOutsideRepositoryError, ds.get_containing_subdataset, opj(os.curdir, outside_path)) assert_raises(PathOutsideRepositoryError, ds.get_containing_subdataset, abspath(outside_path)) @with_tempfile(mkdir=True) def test_require_dataset(path): with chpwd(path): assert_raises( InsufficientArgumentsError, require_dataset, None) create('.') # in this folder by default assert_equal( require_dataset(None).path, path) assert_equal( require_dataset('some', check_installed=False).path, abspath('some')) assert_raises( ValueError, require_dataset, 'some', check_installed=True) @with_tempfile(mkdir=True) def test_dataset_id(path): ds = Dataset(path) assert_equal(ds.id, None) ds.create() dsorigid = ds.id # ID is always a UUID assert_equal(ds.id.count('-'), 4) assert_equal(len(ds.id), 36) # creating a new object for the same path # yields the same ID newds = Dataset(path) assert_false(ds is newds) assert_equal(ds.id, newds.id) # recreating the dataset does NOT change the id ds.create(no_annex=True, force=True) assert_equal(ds.id, dsorigid) # even adding an annex doesn't ds.create(force=True) assert_equal(ds.id, dsorigid) # dataset ID and annex UUID have nothing to do with each other # if an ID was already generated assert_true(ds.repo.uuid != ds.id) # creating a new object for the same dataset with an ID on record # yields the same ID newds = Dataset(path) assert_false(ds is newds) assert_equal(ds.id, newds.id) # even if we generate a dataset from scratch with an annex UUID right away, # this is also not the ID annexds = Dataset(opj(path, 'scratch')).create() assert_true(annexds.id != annexds.repo.uuid) datalad-0.4.1/datalad/distribution/tests/test_dataset_binding.py000066400000000000000000000025661301115270200251120ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test binding of functions to Dataset class """ from os.path import join as opj from ..dataset import Dataset, datasetmethod from datalad.tests.utils import assert_raises, eq_ def test_decorator(): @datasetmethod def func(a, b, dataset=None, some_more=True): return {'a': a, 'b': b, 'dataset': dataset, 'some_more': some_more} ds = Dataset(opj('some', 'where')) orig = func(1, 2, ds, False) eq_(orig['a'], 1) eq_(orig['b'], 2) eq_(orig['dataset'], ds) eq_(orig['some_more'], False) # general call bound = ds.func(1, 2, False) eq_(orig, bound) # use default value bound = ds.func(1, 2) orig['some_more'] = True eq_(orig, bound) # too few arguments: assert_raises(TypeError, ds.func, 1) # too much arguments, by using original call with bound function, # raises proper TypeError: assert_raises(TypeError, ds.func, 1, 2, ds, False) # test name parameter: @datasetmethod(name="new_name") def another(some, dataset=None): return some eq_(ds.new_name('whatever'), 'whatever')datalad-0.4.1/datalad/distribution/tests/test_dataset_config.py000066400000000000000000000023751301115270200247430ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test Dataset configuration """ from ..dataset import Dataset from os.path import join as opj from nose.tools import ok_, eq_, assert_false, assert_equal, assert_true from datalad.tests.utils import with_tree from datalad.api import create # Let's document any configuration option supported in this # reference configuration _config_file_content = """\ [datalad "dataset"] id = nothing """ _dataset_config_template = { 'ds': { '.datalad': { 'config': _config_file_content}}} @with_tree(tree=_dataset_config_template) def test_configuration_access(path): ds = Dataset(opj(path, 'ds')) # there is something prior creation assert_true(ds.config is not None) # creation must change the uuid setting assert_equal(ds.config['datalad.dataset.id'], 'nothing') # create resets this value and records the actual uuid ds.create(force=True) assert_equal(ds.config.get_value('datalad.dataset', 'id', default='nothing'), ds.id) datalad-0.4.1/datalad/distribution/tests/test_get.py000066400000000000000000000401671301115270200225510ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test get action """ import logging import re from os import curdir from os.path import join as opj, basename from glob import glob from datalad.api import get from datalad.api import install from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.exceptions import IncompleteResultsError from datalad.support.exceptions import RemoteNotAvailableError from datalad.tests.utils import ok_ from datalad.tests.utils import eq_ from datalad.tests.utils import with_tempfile from datalad.tests.utils import with_testrepos from datalad.tests.utils import with_tree from datalad.tests.utils import create_tree from datalad.tests.utils import assert_raises from datalad.tests.utils import assert_in from datalad.tests.utils import serve_path_via_http from datalad.tests.utils import assert_re_in from datalad.utils import swallow_logs, with_pathsep from datalad.utils import chpwd from datalad.utils import assure_list from datalad.utils import rmtree from ..dataset import Dataset def _make_dataset_hierarchy(path): origin = Dataset(path).create() origin_sub1 = origin.create('sub1') origin_sub2 = origin_sub1.create('sub2') with open(opj(origin_sub2.path, 'file_in_annex.txt'), "w") as f: f.write('content2') origin_sub3 = origin_sub2.create('sub3') with open(opj(origin_sub3.path, 'file_in_annex.txt'), "w") as f: f.write('content3') origin_sub4 = origin_sub3.create('sub4') origin.save(recursive=True, auto_add_changes=True) return origin, origin_sub1, origin_sub2, origin_sub3, origin_sub4 @with_tempfile(mkdir=True) @with_tempfile(content="doesntmatter") def test_get_invalid_call(path, file_outside): # no argument at all: assert_raises(InsufficientArgumentsError, get, None) assert_raises(InsufficientArgumentsError, get, []) # invalid dataset: assert_raises(ValueError, get, None, dataset=path) # have a plain git: ds = Dataset(path) ds.create(no_annex=True) with open(opj(path, "some.txt"), "w") as f: f.write("whatever") ds.add("some.txt", to_git=True) ds.save("Initial commit.") # no path given: assert_raises(InsufficientArgumentsError, get, dataset=path, source="some", path=None) # get on a plain git: # MIH: why do we warn? user got what was desired. if they want technical # background they better read books not warnings #with swallow_logs(new_level=logging.WARNING) as cml: # # but we don't fail if not annex -- just inform # out = ds.get(curdir) # assert_in('Found no annex. Could not perform any get operation.', # cml.out) # eq_(out, []) # make it an annex: AnnexRepo(path, init=True, create=True) # call get again on a file in git: result = ds.get("some.txt") # skipped silently, but no result for this file: eq_(len(result), 0) # invalid source: # yoh: but now we would need to add it to annex since clever code first # checks what needs to be fetched at all create_tree(path, {'annexed.dat': 'some'}) ds.add("annexed.dat") ds.repo.drop("annexed.dat", options=['--force']) with assert_raises(RemoteNotAvailableError) as ce: ds.get("annexed.dat", source='MysteriousRemote') eq_("MysteriousRemote", ce.exception.remote) # warning on not existing file: with swallow_logs(new_level=logging.WARNING) as cml: with assert_raises(IncompleteResultsError) as cme: ds.get("NotExistingFile.txt") result = cme.exception.results eq_(len(result), 0) assert_in("ignored non-existing paths", cml.out) # path outside repo: with swallow_logs(new_level=logging.WARNING) as cml: result = ds.get(file_outside) eq_(len(result), 0) assert_in("{0} is not part of a dataset, ignored".format(file_outside, ds), cml.out) # TODO: annex --json doesn't report anything when get fails to do get a # file from a specified source, where the file isn't available from. # File report for Joey (plus other failures like not existing when # called with --json) @with_testrepos('basic_annex', flavors='clone') def test_get_single_file(path): ds = Dataset(path) ok_(ds.is_installed()) ok_(ds.repo.file_has_content('test-annex.dat') is False) result = ds.get("test-annex.dat") eq_(len(result), 1) eq_(result[0]['file'], 'test-annex.dat') ok_(result[0]['success'] is True) eq_(result[0]['key'], ds.repo.get_file_key('test-annex.dat')) ok_(ds.repo.file_has_content('test-annex.dat') is True) @with_tree(tree={'file1.txt': 'whatever 1', 'file2.txt': 'whatever 2', 'file3.txt': 'whatever 3', 'file4.txt': 'whatever 4'}) @serve_path_via_http @with_tempfile(mkdir=True) def test_get_multiple_files(path, url, ds_dir): from os import listdir from datalad.support.network import RI file_list = [f for f in listdir(path) if not f.startswith('.')] # prepare urls: urls = [RI(url + f) for f in file_list] # prepare origin origin = Dataset(path).create(force=True) origin.add(file_list) origin.save("initial") ds = install(ds_dir, source=path) # no content present: ok_(not any(ds.repo.file_has_content(file_list))) # get two plus an invalid one: with assert_raises(IncompleteResultsError) as cme: ds.get(['file1.txt', 'file2.txt', 'not_existing.txt']) result = cme.exception.results # explicitly given not existing file was skipped: # (see test_get_invalid_call) eq_(set([item.get('file') for item in result]), {'file1.txt', 'file2.txt'}) ok_(all([x['success'] is True for x in result if x['file'] in ['file1.txt', 'file2.txt']])) ok_(all(ds.repo.file_has_content(['file1.txt', 'file2.txt']))) # get all of them: result = ds.get(curdir) # there were two files left to get: eq_(set([item.get('file') for item in result]), {'file3.txt', 'file4.txt'}) ok_(all(ds.repo.file_has_content(file_list))) @with_tree(tree={'file1.txt': 'something', 'subdir': {'file2.txt': 'something else', 'subsubdir': { 'file3.txt': 'something completely different', 'file4.txt': 'something' }}}) @with_tempfile(mkdir=True) def test_get_recurse_dirs(o_path, c_path): # prepare source: origin = Dataset(o_path).create(force=True) origin.save("Initial", auto_add_changes=True) ds = install(c_path, source=o_path) file_list = ['file1.txt', opj('subdir', 'file2.txt'), opj('subdir', 'subsubdir', 'file3.txt'), opj('subdir', 'subsubdir', 'file4.txt')] files_in_sub = [f for f in file_list if f.startswith(with_pathsep('subdir'))] # no content present: ok_(not any(ds.repo.file_has_content(file_list))) result = ds.get('subdir') # check result: eq_(set([item.get('file') for item in result]), set(files_in_sub)) ok_(all([x['success'] is True for x in result if x['file'] in files_in_sub])) eq_(len(result), len(files_in_sub)) # got all files beneath subdir: ok_(all(ds.repo.file_has_content(files_in_sub))) # additionally got file1.txt silently, since it has the same content as # subdir/subsubdir/file4.txt: ok_(ds.repo.file_has_content('file1.txt') is True) @with_testrepos('submodule_annex', flavors='local') @with_tempfile(mkdir=True) def test_get_recurse_subdatasets(src, path): ds = install(path, source=src) # ask for the two subdatasets specifically. This will obtain them, # but not any content of any files in them subds1, subds2 = ds.get(['subm 1', 'subm 2'], get_data=False) # there are 3 files to get: test-annex.dat within each dataset: rel_path_sub1 = opj(basename(subds1.path), 'test-annex.dat') rel_path_sub2 = opj(basename(subds2.path), 'test-annex.dat') annexed_files = {'test-annex.dat', rel_path_sub1, rel_path_sub2} # None of them is currently present: ok_(ds.repo.file_has_content('test-annex.dat') is False) ok_(subds1.repo.file_has_content('test-annex.dat') is False) ok_(subds2.repo.file_has_content('test-annex.dat') is False) # explicitly given path in subdataset => implicit recursion: # MIH: Nope, we fulfill the dataset handle, but that doesn't # imply fulfilling all file handles result = ds.get(rel_path_sub1, recursive=True) eq_(result[0].get('file'), rel_path_sub1) ok_(result[0].get('success', False) is True) ok_(subds1.repo.file_has_content('test-annex.dat') is True) # drop it: subds1.repo.drop('test-annex.dat') ok_(subds1.repo.file_has_content('test-annex.dat') is False) # now, with a path not explicitly pointing within a # subdataset, but recursive option: # get everything: result = ds.get('.', recursive=True) eq_(set([item.get('file') for item in result]), annexed_files) ok_(all(item.get('success', False) for item in result)) ok_(ds.repo.file_has_content('test-annex.dat') is True) ok_(subds1.repo.file_has_content('test-annex.dat') is True) ok_(subds2.repo.file_has_content('test-annex.dat') is True) # drop them: ds.repo.drop('test-annex.dat') subds1.repo.drop('test-annex.dat') subds2.repo.drop('test-annex.dat') ok_(ds.repo.file_has_content('test-annex.dat') is False) ok_(subds1.repo.file_has_content('test-annex.dat') is False) ok_(subds2.repo.file_has_content('test-annex.dat') is False) # now, the very same call, but without recursive: result = ds.get('.', recursive=False) eq_(len(result), 1) eq_(result[0]['file'], 'test-annex.dat') ok_(result[0]['success'] is True) ok_(ds.repo.file_has_content('test-annex.dat') is True) ok_(subds1.repo.file_has_content('test-annex.dat') is False) ok_(subds2.repo.file_has_content('test-annex.dat') is False) @with_testrepos('submodule_annex', flavors='local') @with_tempfile(mkdir=True) def test_get_greedy_recurse_subdatasets(src, path): ds = install(path, source=src) # GIMME EVERYTHING ds.get(['subm 1', 'subm 2']) # We got all content in the subdatasets subds1, subds2 = [Dataset(d) for d in ds.get_subdatasets(absolute=True)] ok_(ds.repo.file_has_content('test-annex.dat') is False) ok_(subds1.repo.file_has_content('test-annex.dat') is True) ok_(subds2.repo.file_has_content('test-annex.dat') is True) @with_testrepos('submodule_annex', flavors='local') @with_tempfile(mkdir=True) def test_get_install_missing_subdataset(src, path): ds = install(path=path, source=src) ds.create(force=True) # force, to cause dataset initialization subs = [Dataset(s_path) for s_path in ds.get_subdatasets(absolute=True)] ok_(all([not sub.is_installed() for sub in subs])) # we don't install anything, if no explicitly given path points into a # not yet installed subdataset: ds.get(curdir) ok_(all([not sub.is_installed() for sub in subs])) # but we do, whenever a given path is contained in such a subdataset: file_ = opj(subs[0].path, 'test-annex.dat') ds.get(file_) ok_(subs[0].is_installed()) ok_(subs[0].repo.file_has_content('test-annex.dat') is True) # but we fulfill any handles, and dataset handles too ds.get(curdir, recursive=True) ok_(all([sub.is_installed() for sub in subs])) # @with_tree(tree={'file_in_git.txt': 'no idea', # 'subds': {'file_in_annex.txt': 'content'}}) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_get_mixed_hierarchy(src, path): origin = Dataset(src).create(no_annex=True) origin_sub = origin.create('subds') with open(opj(origin.path, 'file_in_git.txt'), "w") as f: f.write('no idea') with open(opj(origin_sub.path, 'file_in_annex.txt'), "w") as f: f.write('content') origin.add('file_in_git.txt', to_git=True) origin_sub.add('file_in_annex.txt') origin.save(auto_add_changes=True) # now, install that thing: ds, subds = install(path, source=src, recursive=True) ok_(subds.repo.file_has_content("file_in_annex.txt") is False) # and get: with swallow_logs(new_level=logging.DEBUG) as cml: result = ds.get(curdir, recursive=True) assert_re_in('.*Found no annex at {0}. Skipped.'.format(ds), cml.out, flags=re.DOTALL) eq_(len(result), 1) eq_(result[0]['file'], opj("subds", "file_in_annex.txt")) ok_(result[0]['success'] is True) ok_(subds.repo.file_has_content("file_in_annex.txt") is True) @with_testrepos('submodule_annex', flavors='local') @with_tempfile(mkdir=True) def test_autoresolve_multiple_datasets(src, path): with chpwd(path): ds1 = install('ds1', source=src) ds2 = install('ds2', source=src) results = get([opj('ds1', 'test-annex.dat')] + glob(opj('ds2', '*.dat'))) # each ds has one file eq_(len(results), 2) ok_(ds1.repo.file_has_content('test-annex.dat') is True) ok_(ds2.repo.file_has_content('test-annex.dat') is True) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_get_autoresolve_recurse_subdatasets(src, path): origin = Dataset(src).create() origin_sub = origin.create('sub') origin_subsub = origin_sub.create('subsub') with open(opj(origin_subsub.path, 'file_in_annex.txt'), "w") as f: f.write('content') origin.save(recursive=True, auto_add_changes=True) ds = install(path, source=src) eq_(len(ds.get_subdatasets(fulfilled=True)), 0) results = get(opj(ds.path, 'sub'), recursive=True) eq_(len(ds.get_subdatasets(fulfilled=True, recursive=True)), 2) subsub = Dataset(opj(ds.path, 'sub', 'subsub')) ok_(subsub.is_installed()) assert_in(subsub, results) # all file handles are fulfilled by default ok_(Dataset(opj(ds.path, 'sub', 'subsub')).repo.file_has_content( "file_in_annex.txt") is True) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_recurse_existing(src, path): origin_ds = _make_dataset_hierarchy(src) # make sure recursion_limit works as expected across a range of depths for depth in range(len(origin_ds)): datasets = assure_list( install(path, source=src, recursive=True, recursion_limit=depth)) # we expect one dataset per level eq_(len(datasets), depth + 1) rmtree(path) # now install all but the last two levels, no data root, sub1, sub2 = install(path, source=src, recursive=True, recursion_limit=2) ok_(sub2.repo.file_has_content('file_in_annex.txt') is False) sub3 = Dataset(opj(sub2.path, 'sub3')) ok_(not sub3.is_installed()) # now get all content in all existing datasets, no new datasets installed # in the process files = root.get(curdir, recursive=True, recursion_limit='existing') eq_(len(files), 1) ok_(sub2.repo.file_has_content('file_in_annex.txt') is True) ok_(not sub3.is_installed()) # now pull down all remaining datasets, no data sub3, sub4 = root.get(curdir, recursive=True, get_data=False) ok_(sub4.is_installed()) ok_(sub3.repo.file_has_content('file_in_annex.txt') is False) # aaannd all data files = root.get(curdir, recursive=True) eq_(len(files), 1) ok_(sub3.repo.file_has_content('file_in_annex.txt') is True) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_get_in_unavailable_subdataset(src, path): origin_ds = _make_dataset_hierarchy(src) root = install(path, source=src) targetpath = opj('sub1', 'sub2') targetabspath = opj(root.path, targetpath) get(targetabspath) # we got the dataset, and its immediate content, but nothing below sub2 = Dataset(targetabspath) ok_(sub2.is_installed()) ok_(sub2.repo.file_has_content('file_in_annex.txt') is True) ok_(not Dataset(opj(targetabspath, 'sub3')).is_installed()) datalad-0.4.1/datalad/distribution/tests/test_install.py000066400000000000000000000631161301115270200234370ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test install action """ import logging import os from os.path import join as opj from os.path import isdir from os.path import exists from os.path import realpath from os.path import basename from os.path import dirname from mock import patch from datalad.api import create from datalad.api import install from datalad.api import get from datalad.consts import DATASETS_TOPURL from datalad.utils import chpwd from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.exceptions import InstallFailedError from datalad.support.exceptions import IncompleteResultsError from datalad.support.gitrepo import GitRepo from datalad.support.gitrepo import GitCommandError from datalad.support.annexrepo import AnnexRepo from datalad.cmd import Runner from datalad.tests.utils import create_tree from datalad.tests.utils import with_tempfile from datalad.tests.utils import assert_in from datalad.tests.utils import with_tree from datalad.tests.utils import with_testrepos from datalad.tests.utils import eq_ from datalad.tests.utils import ok_ from datalad.tests.utils import assert_false from datalad.tests.utils import ok_file_has_content from datalad.tests.utils import assert_not_in from datalad.tests.utils import assert_raises from datalad.tests.utils import ok_startswith from datalad.tests.utils import ok_clean_git from datalad.tests.utils import serve_path_via_http from datalad.tests.utils import swallow_logs from datalad.tests.utils import use_cassette from datalad.tests.utils import skip_if_no_network from datalad.utils import _path_ from datalad.utils import rmtree from ..dataset import Dataset from ..install import _get_installationpath_from_url from ..install import _get_git_url_from_source ############### # Test helpers: ############### def test_installationpath_from_url(): for p in ('lastbit', 'lastbit/', '/lastbit', 'lastbit.git', 'lastbit.git/', 'http://example.com/lastbit', 'http://example.com/lastbit.git', ): eq_(_get_installationpath_from_url(p), 'lastbit') def test_get_git_url_from_source(): # resolves datalad RIs: eq_(_get_git_url_from_source('///subds'), DATASETS_TOPURL + 'subds') assert_raises(NotImplementedError, _get_git_url_from_source, '//custom/subds') # doesn't harm others: eq_(_get_git_url_from_source('http://example.com'), 'http://example.com') eq_(_get_git_url_from_source('/absolute/path'), '/absolute/path') eq_(_get_git_url_from_source('file://localhost/some'), 'file://localhost/some') eq_(_get_git_url_from_source('localhost/another/path'), 'localhost/another/path') eq_(_get_git_url_from_source('user@someho.st/mydir'), 'user@someho.st/mydir') eq_(_get_git_url_from_source('ssh://somewhe.re/else'), 'ssh://somewhe.re/else') eq_(_get_git_url_from_source('git://github.com/datalad/testrepo--basic--r1'), 'git://github.com/datalad/testrepo--basic--r1') @with_tree(tree={'file.txt': '123'}) @serve_path_via_http @with_tempfile def _test_guess_dot_git(annex, path, url, tdir): repo = (AnnexRepo if annex else GitRepo)(path, create=True) repo.add('file.txt', commit=True, git=not annex) # we need to prepare to be served via http, otherwise it must fail with swallow_logs() as cml: assert_raises(GitCommandError, install, path=tdir, source=url) ok_(not exists(tdir)) Runner(cwd=path)(['git', 'update-server-info']) with swallow_logs() as cml: installed = install(tdir, source=url) assert_not_in("Failed to get annex.uuid", cml.out) eq_(realpath(installed.path), realpath(tdir)) ok_(exists(tdir)) ok_clean_git(tdir, annex=annex) def test_guess_dot_git(): for annex in False, True: yield _test_guess_dot_git, annex ###################### # Test actual Install: ###################### def test_insufficient_args(): assert_raises(InsufficientArgumentsError, install) assert_raises(InsufficientArgumentsError, install, description="some") assert_raises(InsufficientArgumentsError, install, None) assert_raises(InsufficientArgumentsError, install, None, description="some") @with_tempfile(mkdir=True) def test_invalid_args(path): assert_raises(ValueError, install, 'Zoidberg', source='Zoidberg') # install to an invalid URL assert_raises(ValueError, install, 'ssh://mars:Zoidberg', source='Zoidberg') # install to a remote location assert_raises(ValueError, install, 'ssh://mars/Zoidberg', source='Zoidberg') # make fake dataset ds = create(path) assert_raises(ValueError, install, '/higherup.', 'Zoidberg', dataset=ds) @skip_if_no_network @use_cassette('test_install_crcns') @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_install_crcns(tdir, ds_path): with chpwd(tdir): with swallow_logs(new_level=logging.INFO) as cml: install("all-nonrecursive", source='///') # since we didn't log decorations such as log level atm while # swallowing so lets check if exit code is returned or not # I will test both assert_not_in('ERROR', cml.out) # below one must not fail alone! ;) assert_not_in('with exit code', cml.out) # should not hang in infinite recursion with chpwd('all-nonrecursive'): get("crcns") ok_(exists(_path_("all-nonrecursive/crcns/.git/config"))) # and we could repeat installation and get the same result ds1 = install(_path_("all-nonrecursive/crcns")) ds2 = Dataset('all-nonrecursive').install('crcns') ok_(ds1.is_installed()) eq_(ds1, ds2) eq_(ds1.path, ds2.path) # to make sure they are a single dataset # again, but into existing dataset: ds = create(ds_path) crcns = ds.install("///crcns") ok_(crcns.is_installed()) eq_(crcns.path, opj(ds_path, "crcns")) assert_in(crcns.path, ds.get_subdatasets(absolute=True)) @skip_if_no_network @use_cassette('test_install_crcns') @with_tree(tree={'sub': {}}) def test_install_datasets_root(tdir): with chpwd(tdir): ds = install("///") ok_(ds.is_installed()) eq_(ds.path, opj(tdir, 'datasets.datalad.org')) # do it a second time: with swallow_logs(new_level=logging.INFO) as cml: result = install("///") assert_in("was already installed from", cml.out) eq_(result, ds) # and a third time into an existing something, that is not a dataset: with open(opj(tdir, 'sub', 'a_file.txt'), 'w') as f: f.write("something") with swallow_logs(new_level=logging.WARNING) as cml: result = install("sub", source='///') assert_in("already exists and is not an installed dataset", cml.out) ok_(result is None) @with_testrepos('.*basic.*', flavors=['local-url', 'network', 'local']) @with_tempfile(mkdir=True) def test_install_simple_local(src, path): origin = Dataset(path) # now install it somewhere else ds = install(path, source=src) eq_(ds.path, path) ok_(ds.is_installed()) if not isinstance(origin.repo, AnnexRepo): # this means it is a GitRepo ok_(isinstance(origin.repo, GitRepo)) # stays plain Git repo ok_(isinstance(ds.repo, GitRepo)) ok_(not isinstance(ds.repo, AnnexRepo)) ok_(GitRepo.is_valid_repo(ds.path)) eq_(set(ds.repo.get_indexed_files()), {'test.dat', 'INFO.txt'}) ok_clean_git(path, annex=False) else: # must be an annex ok_(isinstance(ds.repo, AnnexRepo)) ok_(AnnexRepo.is_valid_repo(ds.path, allow_noninitialized=False)) eq_(set(ds.repo.get_indexed_files()), {'test.dat', 'INFO.txt', 'test-annex.dat'}) ok_clean_git(path, annex=True) # no content was installed: ok_(not ds.repo.file_has_content('test-annex.dat')) uuid_before = ds.repo.uuid # installing it again, shouldn't matter: with swallow_logs(new_level=logging.INFO) as cml: ds = install(path, source=src) cml.assert_logged(msg="{0} was already installed from".format(ds), regex=False, level="INFO") ok_(ds.is_installed()) if isinstance(origin.repo, AnnexRepo): eq_(uuid_before, ds.repo.uuid) @with_testrepos(flavors=['local-url', 'network', 'local']) @with_tempfile def test_install_dataset_from_just_source(url, path): with chpwd(path, mkdir=True): ds = install(source=url) ok_startswith(ds.path, path) ok_(ds.is_installed()) ok_(GitRepo.is_valid_repo(ds.path)) ok_clean_git(ds.path, annex=False) assert_in('INFO.txt', ds.repo.get_indexed_files()) @with_testrepos(flavors=['network']) @with_tempfile def test_install_dataset_from_just_source_via_path(url, path): # for remote urls only, the source could be given to `path` # to allows for simplistic cmdline calls # Q (ben): remote urls only? Sure? => TODO with chpwd(path, mkdir=True): ds = install(url) ok_startswith(ds.path, path) ok_(ds.is_installed()) ok_(GitRepo.is_valid_repo(ds.path)) ok_clean_git(ds.path, annex=False) assert_in('INFO.txt', ds.repo.get_indexed_files()) @with_tree(tree={ 'ds': {'test.txt': 'some'}, }) @serve_path_via_http @with_tempfile(mkdir=True) def test_install_dataladri(src, topurl, path): # make plain git repo ds_path = opj(src, 'ds') gr = GitRepo(ds_path, create=True) gr.add('test.txt') gr.commit('demo') Runner(cwd=gr.path)(['git', 'update-server-info']) # now install it somewhere else with patch('datalad.support.network.DATASETS_TOPURL', topurl), \ swallow_logs(): ds = install(path, source='///ds') eq_(ds.path, path) ok_clean_git(path, annex=False) ok_file_has_content(opj(path, 'test.txt'), 'some') @with_testrepos('submodule_annex', flavors=['local', 'local-url', 'network']) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_install_recursive(src, path_nr, path_r): # first install non-recursive: ds = install(path_nr, source=src, recursive=False) ok_(ds.is_installed()) for sub in ds.get_subdatasets(recursive=True): ok_(not Dataset(opj(path_nr, sub)).is_installed(), "Unintentionally installed: %s" % opj(path_nr, sub)) # this also means, subdatasets to be listed as not fulfilled: eq_(set(ds.get_subdatasets(recursive=True, fulfilled=False)), {'subm 1', 'subm 2'}) # now recursively: ds_list = install(path_r, source=src, recursive=True) # installed a dataset and two subdatasets eq_(len(ds_list), 3) eq_(sum([isinstance(i, Dataset) for i in ds_list]), 3) # we recurse top down during installation, so toplevel should appear at # first position in returned list eq_(ds_list[0].path, path_r) top_ds = ds_list[0] ok_(top_ds.is_installed()) # the subdatasets are contained in returned list: # (Note: Until we provide proper (singleton) instances for Datasets, # need to check for their paths) assert_in(opj(top_ds.path, 'subm 1'), [i.path for i in ds_list]) assert_in(opj(top_ds.path, 'subm 2'), [i.path for i in ds_list]) eq_(len(top_ds.get_subdatasets(recursive=True)), 2) for sub in top_ds.get_subdatasets(recursive=True): subds = Dataset(opj(path_r, sub)) ok_(subds.is_installed(), "Not installed: %s" % opj(path_r, sub)) # no content was installed: ok_(not any(subds.repo.file_has_content( subds.repo.get_annexed_files()))) # no unfulfilled subdatasets: ok_(top_ds.get_subdatasets(recursive=True, fulfilled=False) == []) @with_testrepos('submodule_annex', flavors=['local']) @with_tempfile(mkdir=True) def test_install_recursive_with_data(src, path): # now again; with data: ds_list = install(path, source=src, recursive=True, get_data=True) # installed a dataset and two subdatasets, and two files: eq_(len(ds_list), 5) eq_(sum([isinstance(i, Dataset) for i in ds_list]), 3) # we recurse top down during installation, so toplevel should appear at # first position in returned list eq_(ds_list[0].path, path) top_ds = ds_list[0] ok_(top_ds.is_installed()) if isinstance(top_ds.repo, AnnexRepo): ok_(all(top_ds.repo.file_has_content(top_ds.repo.get_annexed_files()))) for sub in top_ds.get_subdatasets(recursive=True): subds = Dataset(opj(path, sub)) ok_(subds.is_installed(), "Not installed: %s" % opj(path, sub)) if isinstance(subds.repo, AnnexRepo): ok_(all(subds.repo.file_has_content(subds.repo.get_annexed_files()))) @with_testrepos(flavors=['local']) # 'local-url', 'network' # TODO: Somehow annex gets confused while initializing installed ds, whose # .git/config show a submodule url "file:///aaa/bbb%20b/..." # this is delivered by with_testrepos as the url to clone @with_tempfile def test_install_into_dataset(source, top_path): ds = create(top_path) ok_clean_git(ds.path) subds = ds.install("sub", source=source, save=False) if isinstance(subds.repo, AnnexRepo) and subds.repo.is_direct_mode(): ok_(exists(opj(subds.path, '.git'))) else: ok_(isdir(opj(subds.path, '.git'))) ok_(subds.is_installed()) assert_in('sub', ds.get_subdatasets()) # sub is clean: ok_clean_git(subds.path, annex=False) # top is not: assert_raises(AssertionError, ok_clean_git, ds.path, annex=False) ds.save('addsub') # now it is: ok_clean_git(ds.path, annex=False) # but we could also save while installing and there should be no side-effect # of saving any other changes if we state to not auto-save changes # Create a dummy change create_tree(ds.path, {'dummy.txt': 'buga'}) ok_clean_git(ds.path, untracked=['dummy.txt']) subds_ = ds.install("sub2", source=source, if_dirty='ignore') eq_(subds_.path, opj(ds.path, "sub2")) # for paranoid yoh ;) ok_clean_git(ds.path, untracked=['dummy.txt']) # and we should achieve the same behavior if we create a dataset # and then decide to "add" it create(_path_(top_path, 'sub3'), if_dirty='ignore') ok_clean_git(ds.path, untracked=['dummy.txt', 'sub3/']) ds.install('sub3', if_dirty='ignore') ok_clean_git(ds.path, untracked=['dummy.txt']) @skip_if_no_network @use_cassette('test_install_crcns') @with_tempfile def test_failed_install_multiple(top_path): ds = create(top_path) create(_path_(top_path, 'ds1')) create(_path_(top_path, 'ds3')) ok_clean_git(ds.path, annex=False, untracked=['ds1/', 'ds3/']) # specify install with multiple paths and one non-existing with assert_raises(IncompleteResultsError) as cme: ds.install(['ds1', 'ds2', '///crcns', '///nonexisting', 'ds3']) ok_clean_git(ds.path, annex=False) # those which succeeded should be saved now eq_(ds.get_subdatasets(), ['crcns', 'ds1', 'ds3']) # and those which didn't -- listed eq_(set(cme.exception.failed), {'///nonexisting', _path_(top_path, 'ds2')}) # but if there was only a single installation requested -- it will be # InstallFailedError to stay consistent with single install behavior # TODO: unify at some point with assert_raises(InstallFailedError) as cme: ds.install('ds2') with assert_raises(InstallFailedError) as cme: ds.install('///nonexisting') @with_testrepos('submodule_annex', flavors=['local', 'local-url', 'network']) @with_tempfile(mkdir=True) def test_install_known_subdataset(src, path): # get the superdataset: ds = install(path, source=src) # subdataset not installed: subds = Dataset(opj(path, 'subm 1')) assert_false(subds.is_installed()) assert_in('subm 1', ds.get_subdatasets(fulfilled=False)) assert_not_in('subm 1', ds.get_subdatasets(fulfilled=True)) # install it: ds.install('subm 1') ok_(subds.is_installed()) ok_(AnnexRepo.is_valid_repo(subds.path, allow_noninitialized=False)) # Verify that it is the correct submodule installed and not # new repository initiated eq_(set(subds.repo.get_indexed_files()), {'test.dat', 'INFO.txt', 'test-annex.dat'}) assert_not_in('subm 1', ds.get_subdatasets(fulfilled=False)) assert_in('subm 1', ds.get_subdatasets(fulfilled=True)) # now, get the data by reinstalling with -g: ok_(subds.repo.file_has_content('test-annex.dat') is False) with chpwd(ds.path): result = get(path='subm 1', dataset=os.curdir) eq_(len(result), 1) eq_(result[0]['file'], opj('subm 1', 'test-annex.dat')) ok_(subds.repo.file_has_content('test-annex.dat') is True) ok_(subds.is_installed()) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_implicit_install(src, dst): origin_top = create(src) origin_sub = origin_top.create("sub") origin_subsub = origin_sub.create("subsub") with open(opj(origin_top.path, "file1.txt"), "w") as f: f.write("content1") origin_top.add("file1.txt") with open(opj(origin_sub.path, "file2.txt"), "w") as f: f.write("content2") origin_sub.add("file2.txt") with open(opj(origin_subsub.path, "file3.txt"), "w") as f: f.write("content3") origin_subsub.add("file3.txt") origin_top.save(auto_add_changes=True) # first, install toplevel: ds = install(dst, source=src) ok_(ds.is_installed()) sub = Dataset(opj(ds.path, "sub")) ok_(not sub.is_installed()) subsub = Dataset(opj(sub.path, "subsub")) ok_(not subsub.is_installed()) # fail on obscure non-existing one assert_raises(InstallFailedError, ds.install, source='obscure') # install 3rd level and therefore implicitly the 2nd: result = ds.install(path=opj("sub", "subsub")) ok_(sub.is_installed()) ok_(subsub.is_installed()) eq_(result, subsub) # fail on obscure non-existing one in subds assert_raises(InstallFailedError, ds.install, source=opj('sub', 'obscure')) # clean up: rmtree(dst, chmod_files=True) ok_(not exists(dst)) # again first toplevel: ds = install(dst, source=src) ok_(ds.is_installed()) sub = Dataset(opj(ds.path, "sub")) ok_(not sub.is_installed()) subsub = Dataset(opj(sub.path, "subsub")) ok_(not subsub.is_installed()) # now implicit but without an explicit dataset to install into # (deriving from CWD): with chpwd(dst): result = get(path=opj("sub", "subsub")) ok_(sub.is_installed()) ok_(subsub.is_installed()) eq_(result, [subsub]) @with_tempfile(mkdir=True) def test_failed_install(dspath): ds = create(dspath) assert_raises(InstallFailedError, ds.install, "sub", source="http://nonexistingreallyanything.somewhere/bla") @with_testrepos('submodule_annex', flavors=['local']) @with_tempfile(mkdir=True) def test_install_list(path, top_path): # we want to be able to install several things, if these are known # (no 'source' allowed). Therefore first toplevel: ds = install(top_path, source=path, recursive=False) assert_not_in('annex.hardlink', ds.config) ok_(ds.is_installed()) sub1 = Dataset(opj(top_path, 'subm 1')) sub2 = Dataset(opj(top_path, 'subm 2')) ok_(not sub1.is_installed()) ok_(not sub2.is_installed()) # fails, when `source` is passed: assert_raises(ValueError, ds.install, path=['subm 1', 'subm 2'], source='something') # now should work: result = ds.install(path=['subm 1', 'subm 2']) ok_(sub1.is_installed()) ok_(sub2.is_installed()) eq_(set([i.path for i in result]), {sub1.path, sub2.path}) # and if we request it again via get, result should be empty get_result = ds.get(path=['subm 1', 'subm 2'], get_data=False) eq_(get_result, []) @with_testrepos('submodule_annex', flavors=['local']) @with_tempfile(mkdir=True) def test_reckless(path, top_path): ds = install(top_path, source=path, reckless=True) eq_(ds.config.get('annex.hardlink', None), 'true') eq_(ds.repo.repo_info()['untrusted repositories'][0]['here'], True) @with_tree(tree={'top_file.txt': 'some', 'sub 1': {'sub1file.txt': 'something else', 'subsub': {'subsubfile.txt': 'completely different', } }, 'sub 2': {'sub2file.txt': 'meaningless', } }) @with_tempfile(mkdir=True) def test_install_recursive_repeat(src, path): subsub_src = Dataset(opj(src, 'sub 1', 'subsub')).create(force=True) sub1_src = Dataset(opj(src, 'sub 1')).create(force=True) sub2_src = Dataset(opj(src, 'sub 2')).create(force=True) top_src = Dataset(src).create(force=True) top_src.save(auto_add_changes=True, recursive=True) # install top level: top_ds = install(path, source=src) ok_(top_ds.is_installed() is True) sub1 = Dataset(opj(path, 'sub 1')) ok_(sub1.is_installed() is False) sub2 = Dataset(opj(path, 'sub 2')) ok_(sub2.is_installed() is False) subsub = Dataset(opj(path, 'sub 1', 'subsub')) ok_(subsub.is_installed() is False) # install again, now with data and recursive, but recursion_limit 1: result = get(os.curdir, dataset=path, recursive=True, recursion_limit=1) # top-level dataset was not reobtained assert_not_in(top_ds, result) assert_in(sub1, result) assert_in(sub2, result) assert_not_in(subsub, result) ok_(top_ds.repo.file_has_content('top_file.txt') is True) ok_(sub1.repo.file_has_content('sub1file.txt') is True) ok_(sub2.repo.file_has_content('sub2file.txt') is True) # install sub1 again, recursively and with data top_ds.install('sub 1', recursive=True, get_data=True) ok_(subsub.is_installed()) ok_(subsub.repo.file_has_content('subsubfile.txt')) @with_testrepos('submodule_annex', flavors=['local']) @with_tempfile(mkdir=True) @with_tempfile def test_install_skip_list_arguments(src, path, path_outside): ds = install(path, source=src) ok_(ds.is_installed()) # install a list with valid and invalid items: with swallow_logs(new_level=logging.WARNING) as cml: with assert_raises(IncompleteResultsError) as cme: ds.install( path=['subm 1', 'not_existing', path_outside, 'subm 2'], get_data=False) result = cme.exception.results for skipped in [opj(ds.path, 'not_existing'), path_outside]: cml.assert_logged(msg="ignored non-existing paths: {}\n".format( [opj(ds.path, 'not_existing'), path_outside]), regex=False, level='WARNING') pass ok_(isinstance(result, list)) eq_(len(result), 2) for sub in [Dataset(opj(path, 'subm 1')), Dataset(opj(path, 'subm 2'))]: assert_in(sub, result) ok_(sub.is_installed()) # return of get is always a list, even if just one thing was gotten # in this case 'subm1' was already obtained above, so this will get this # content of the subdataset with assert_raises(IncompleteResultsError) as cme: ds.install(path=['subm 1', 'not_existing']) with assert_raises(IncompleteResultsError) as cme: ds.get(path=['subm 1', 'not_existing']) result = cme.exception.results eq_(len(result), 1) eq_(result[0]['file'], 'subm 1/test-annex.dat') @with_testrepos('submodule_annex', flavors=['local']) @with_tempfile(mkdir=True) def test_install_skip_failed_recursive(src, path): # install top level: ds = install(path, source=src) sub1 = Dataset(opj(path, 'subm 1')) sub2 = Dataset(opj(path, 'subm 2')) # sabotage recursive installation of 'subm 1' by polluting the target: with open(opj(path, 'subm 1', 'blocking.txt'), "w") as f: f.write("sdfdsf") with swallow_logs(new_level=logging.WARNING) as cml: result = ds.get(os.curdir, recursive=True) # toplevel dataset was in the house already assert_not_in(ds, result) assert_in(sub2, result) assert_not_in(sub1, result) cml.assert_logged( msg="Target {} already exists and is not an installed dataset. Skipped.".format(sub1.path), regex=False, level='WARNING') @with_tree(tree={'top_file.txt': 'some', 'sub 1': {'sub1file.txt': 'something else', 'subsub': {'subsubfile.txt': 'completely different', } }, 'sub 2': {'sub2file.txt': 'meaningless', } }) @with_tempfile(mkdir=True) def test_install_noautoget_data(src, path): subsub_src = Dataset(opj(src, 'sub 1', 'subsub')).create(force=True) sub1_src = Dataset(opj(src, 'sub 1')).create(force=True) sub2_src = Dataset(opj(src, 'sub 2')).create(force=True) top_src = Dataset(src).create(force=True) top_src.save(auto_add_changes=True, recursive=True) # install top level: cdss = install(path, source=src, recursive=True) # there should only be datasets in the list of installed items, # and none of those should have any data for their annexed files yet for ds in cdss: assert_false(any(ds.repo.file_has_content(ds.repo.get_annexed_files()))) @with_tempfile @with_tempfile def test_install_source_relpath(src, dest): ds1 = create(src) src_ = basename(src) with chpwd(dirname(src)): ds2 = install(dest, source=src_)datalad-0.4.1/datalad/distribution/tests/test_publish.py000066400000000000000000000223161301115270200234340ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test publish action """ import logging import os from os.path import join as opj, abspath, basename from ..dataset import Dataset from datalad.api import publish, install from datalad.utils import chpwd from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from nose.tools import ok_, eq_, assert_false, assert_is_instance from datalad.tests.utils import with_tempfile, assert_in, with_tree,\ with_testrepos, assert_not_in from datalad.tests.utils import SkipTest from datalad.tests.utils import assert_cwd_unchanged, skip_if_on_windows from datalad.tests.utils import assure_dict_from_str, assure_list_from_str from datalad.tests.utils import ok_generator from datalad.tests.utils import assert_not_in from datalad.tests.utils import assert_raises from datalad.tests.utils import ok_startswith from datalad.tests.utils import skip_if_no_module from datalad.tests.utils import ok_clean_git from datalad.tests.utils import swallow_logs from datalad.tests.utils import assert_not_in @with_testrepos('submodule_annex', flavors=['local']) #TODO: Use all repos after fixing them @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_publish_simple(origin, src_path, dst_path): # prepare src source = install(src_path, source=origin, recursive=True)[0] # forget we cloned it (provide no 'origin' anymore), which should lead to # setting tracking branch to target: source.repo.remove_remote("origin") # create plain git at target: target = GitRepo(dst_path, create=True) target.checkout("TMP", ["-b"]) source.repo.add_remote("target", dst_path) res = publish(dataset=source, to="target") eq_(res, ([source], [])) ok_clean_git(src_path, annex=False) ok_clean_git(dst_path, annex=False) eq_(list(target.get_branch_commits("master")), list(source.repo.get_branch_commits("master"))) # don't fail when doing it again res = publish(dataset=source, to="target") eq_(res, ([source], [])) ok_clean_git(src_path, annex=False) ok_clean_git(dst_path, annex=False) eq_(list(target.get_branch_commits("master")), list(source.repo.get_branch_commits("master"))) eq_(list(target.get_branch_commits("git-annex")), list(source.repo.get_branch_commits("git-annex"))) # 'target/master' should be tracking branch at this point, so # try publishing without `to`: # some modification: with open(opj(src_path, 'test_mod_file'), "w") as f: f.write("Some additional stuff.") source.repo.add(opj(src_path, 'test_mod_file'), git=True, commit=True, msg="Modified.") ok_clean_git(src_path, annex=False) res = publish(dataset=source) eq_(res, ([source], [])) ok_clean_git(dst_path, annex=False) eq_(list(target.get_branch_commits("master")), list(source.repo.get_branch_commits("master"))) eq_(list(target.get_branch_commits("git-annex")), list(source.repo.get_branch_commits("git-annex"))) @with_testrepos('submodule_annex', flavors=['local']) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_publish_recursive(origin, src_path, dst_path, sub1_pub, sub2_pub): # prepare src source = install(src_path, source=origin, recursive=True)[0] # create plain git at target: target = GitRepo(dst_path, create=True) target.checkout("TMP", ["-b"]) source.repo.add_remote("target", dst_path) # subdatasets have no remote yet, so recursive publishing should fail: with assert_raises(ValueError) as cm: publish(dataset=source, to="target", recursive=True) assert_in("No sibling 'target' found.", str(cm.exception)) # now, set up targets for the submodules: sub1_target = GitRepo(sub1_pub, create=True) sub1_target.checkout("TMP", ["-b"]) sub2_target = AnnexRepo(sub2_pub, create=True) sub2_target.checkout("TMP", ["-b"]) sub1 = GitRepo(opj(src_path, 'subm 1'), create=False) sub2 = GitRepo(opj(src_path, 'subm 2'), create=False) sub1.add_remote("target", sub1_pub) sub2.add_remote("target", sub2_pub) # publish recursively with swallow_logs(new_level=logging.DEBUG) as cml: res = publish(dataset=source, to="target", recursive=True) assert_not_in( 'forced update', cml.out, "we probably haven't merged git-annex before pushing" ) # testing result list # (Note: Dataset lacks __eq__ for now. Should this be based on path only?) assert_is_instance(res, tuple) assert_is_instance(res[0], list) assert_is_instance(res[1], list) eq_(res[1], []) # nothing failed/was skipped for item in res[0]: assert_is_instance(item, Dataset) eq_({res[0][0].path, res[0][1].path, res[0][2].path}, {src_path, sub1.path, sub2.path}) eq_(list(target.get_branch_commits("master")), list(source.repo.get_branch_commits("master"))) eq_(list(target.get_branch_commits("git-annex")), list(source.repo.get_branch_commits("git-annex"))) eq_(list(sub1_target.get_branch_commits("master")), list(sub1.get_branch_commits("master"))) eq_(list(sub1_target.get_branch_commits("git-annex")), list(sub1.get_branch_commits("git-annex"))) eq_(list(sub2_target.get_branch_commits("master")), list(sub2.get_branch_commits("master"))) eq_(list(sub2_target.get_branch_commits("git-annex")), list(sub2.get_branch_commits("git-annex"))) # test for publishing with --since. By default since no changes, only current pushed res_ = publish(dataset=source, recursive=True) # only current one would get pushed eq_(set(r.path for r in res_[0]), {src_path}) # all get pushed res_ = publish(dataset=source, recursive=True, since='HEAD^') eq_(set(r.path for r in res_[0]), {src_path, sub1.path, sub2.path}) # Let's now update one subm with open(opj(sub2.path, "file.txt"), 'w') as f: f.write('') sub2.add('file.txt') sub2.commit("") # TODO: Doesn't work: https://github.com/datalad/datalad/issues/636 #source.save("changed sub2", auto_add_changes=True) source.repo.commit("", options=['-a']) res_ = publish(dataset=source, recursive=True) # only updated ones were published eq_(set(r.path for r in res_[0]), {src_path, sub2.path}) @with_testrepos('submodule_annex', flavors=['local']) #TODO: Use all repos after fixing them @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_publish_with_data(origin, src_path, dst_path, sub1_pub, sub2_pub): # prepare src source = install(src_path, source=origin, recursive=True)[0] source.repo.get('test-annex.dat') # create plain git at target: target = AnnexRepo(dst_path, create=True) target.checkout("TMP", ["-b"]) source.repo.add_remote("target", dst_path) # now, set up targets for the submodules: sub1_target = GitRepo(sub1_pub, create=True) sub1_target.checkout("TMP", ["-b"]) sub2_target = GitRepo(sub2_pub, create=True) sub2_target.checkout("TMP", ["-b"]) sub1 = GitRepo(opj(src_path, 'subm 1'), create=False) sub2 = GitRepo(opj(src_path, 'subm 2'), create=False) sub1.add_remote("target", sub1_pub) sub2.add_remote("target", sub2_pub) # TMP: Insert the fetch to prevent GitPython to fail after the push, # because it cannot resolve the SHA of the old commit of the remote, # that git reports back after the push. # TODO: Figure out, when to fetch things in general; Alternatively: # Is there an option for push, that prevents GitPython from failing? source.repo.fetch("target") res = publish(dataset=source, to="target", path=['test-annex.dat']) eq_(res, ([source, 'test-annex.dat'], [])) eq_(list(target.get_branch_commits("master")), list(source.repo.get_branch_commits("master"))) # TODO: last commit in git-annex branch differs. Probably fine, # but figure out, when exactly to expect this for proper testing: eq_(list(target.get_branch_commits("git-annex"))[1:], list(source.repo.get_branch_commits("git-annex"))[1:]) # we need compare target/master: target.checkout("master") eq_(target.file_has_content(['test-annex.dat']), [True]) source.repo.fetch("target") res = publish(dataset=source, to="target", path=['.']) eq_(res, ([source, 'test-annex.dat'], [])) source.repo.fetch("target") import glob res = publish(dataset=source, to="target", path=glob.glob1(source.path, '*')) # Note: This leads to recursive publishing, since expansion of '*' # contains the submodules themselves in this setup # collect result paths: result_paths = [] for item in res[0]: if isinstance(item, Dataset): result_paths.append(item.path) else: result_paths.append(item) eq_({source.path, opj(source.path, "subm 1"), opj(source.path, "subm 2"), 'test-annex.dat'}, set(result_paths)) datalad-0.4.1/datalad/distribution/tests/test_target_ssh.py000066400000000000000000000302741301115270200241330ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test create publication target ssh web server action """ import os import re from os.path import join as opj, basename, exists from git.exc import GitCommandError from ..dataset import Dataset from datalad.api import publish, install, create_sibling from datalad.utils import chpwd from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from nose.tools import eq_, assert_false from datalad.tests.utils import with_tempfile, assert_in, \ with_testrepos from datalad.tests.utils import SkipTest from datalad.tests.utils import ok_file_has_content from datalad.tests.utils import ok_exists from datalad.tests.utils import ok_endswith from datalad.tests.utils import assert_not_in from datalad.tests.utils import assert_raises from datalad.tests.utils import skip_ssh from datalad.tests.utils import assert_dict_equal from datalad.tests.utils import assert_set_equal from datalad.tests.utils import assert_not_equal from datalad.tests.utils import assert_no_errors_logged from datalad.tests.utils import get_mtimes_and_digests from datalad.tests.utils import swallow_logs from datalad.utils import on_windows from datalad.utils import _path_ import logging def _test_correct_publish(target_path, rootds=False, flat=True): paths = [_path_(".git/hooks/post-update")] # hooks enabled in all datasets not_paths = [] # _path_(".git/datalad/metadata")] # metadata only on publish # ATM we run post-update hook also upon create since it might # be a reconfiguration (TODO: I guess could be conditioned) # web-interface html pushed to dataset root web_paths = ['index.html', _path_(".git/datalad/web")] if rootds: paths += web_paths # and not to subdatasets elif not flat: not_paths += web_paths for path in paths: ok_exists(opj(target_path, path)) for path in not_paths: assert_false(exists(opj(target_path, path))) # correct ls_json command in hook content (path wrapped in quotes) ok_file_has_content(_path_(target_path, '.git/hooks/post-update'), '.*datalad ls -a --json file \'%s\'.*' % target_path, re_=True, flags=re.DOTALL) # shortcut # but we can rely on it ATM only if "server" (i.e. localhost) has # recent enough git since then we expect an error msg to be spit out from datalad.support.external_versions import external_versions assert_create_sshwebserver = ( assert_no_errors_logged(create_sibling) if external_versions['cmd:git'] >= '2.4' else create_sibling ) @skip_ssh @with_testrepos('.*basic.*', flavors=['local']) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_target_ssh_simple(origin, src_path, target_rootpath): # prepare src source = install(src_path, source=origin) target_path = opj(target_rootpath, "basic") # it will try to fetch it so would fail as well since sshurl is wrong with swallow_logs(new_level=logging.ERROR) as cml, \ assert_raises(GitCommandError): create_sibling( dataset=source, target="local_target", sshurl="ssh://localhost", target_dir=target_path, ui=True) # is not actually happening on one of the two basic cases -- TODO figure it out # assert_in('enableremote local_target failed', cml.out) GitRepo(target_path, create=False) # raises if not a git repo assert_in("local_target", source.repo.get_remotes()) eq_("ssh://localhost", source.repo.get_remote_url("local_target")) # should NOT be able to push now, since url isn't correct: # TODO: assumption is wrong if ~ does have .git! fix up! assert_raises(GitCommandError, publish, dataset=source, to="local_target") # Both must be annex or git repositories src_is_annex = AnnexRepo.is_valid_repo(src_path) eq_(src_is_annex, AnnexRepo.is_valid_repo(target_path)) # And target one should be known to have a known UUID within the source if annex if src_is_annex: annex = AnnexRepo(src_path) local_target_cfg = annex.repo.remotes["local_target"].config_reader.get # for some reason this was "correct" # eq_(local_target_cfg('annex-ignore'), 'false') # but after fixing creating siblings in # 21f6dd012b2c7b9c0b8b348dcfb3b0ace7e8b2ec it started to fail # I think it is legit since we are trying to fetch now before calling # annex.enable_remote so it doesn't set it up, and fails before assert_raises(Exception, local_target_cfg, 'annex-ignore') # hm, but ATM wouldn't get a uuid since url is wrong assert_raises(Exception, local_target_cfg, 'annex-uuid') # do it again without force: with assert_raises(RuntimeError) as cm: assert_create_sshwebserver( dataset=source, target="local_target", sshurl="ssh://localhost", target_dir=target_path) eq_("Target directory %s already exists." % target_path, str(cm.exception)) if src_is_annex: target_description = AnnexRepo(target_path, create=False).get_description() assert_not_equal(target_description, None) assert_not_equal(target_description, target_path) ok_endswith(target_description, target_path) # now, with force and correct url, which is also used to determine # target_dir # Note: on windows absolute path is not url conform. But this way it's easy # to test, that ssh path is correctly used. if not on_windows: # add random file under target_path, to explicitly test existing=replace open(opj(target_path, 'random'), 'w').write('123') assert_create_sshwebserver( dataset=source, target="local_target", sshurl="ssh://localhost" + target_path, existing='replace') eq_("ssh://localhost" + target_path, source.repo.get_remote_url("local_target")) eq_("ssh://localhost" + target_path, source.repo.get_remote_url("local_target", push=True)) # ensure target tree actually replaced by source assert_false(exists(opj(target_path, 'random'))) if src_is_annex: annex = AnnexRepo(src_path) local_target_cfg = annex.repo.remotes["local_target"].config_reader.get eq_(local_target_cfg('annex-ignore'), 'false') eq_(local_target_cfg('annex-uuid').count('-'), 4) # valid uuid # again, by explicitly passing urls. Since we are on localhost, the # local path should work: cpkwargs = dict( dataset=source, target="local_target", sshurl="ssh://localhost", target_dir=target_path, target_url=target_path, target_pushurl="ssh://localhost" + target_path, ui=True, ) assert_create_sshwebserver(existing='replace', **cpkwargs) if src_is_annex: target_description = AnnexRepo(target_path, create=False).get_description() eq_(target_description, target_path) eq_(target_path, source.repo.get_remote_url("local_target")) eq_("ssh://localhost" + target_path, source.repo.get_remote_url("local_target", push=True)) _test_correct_publish(target_path) # now, push should work: publish(dataset=source, to="local_target") # and we should be able to 'reconfigure' def process_digests_mtimes(digests, mtimes): # it should have triggered a hook, which would have created log and metadata files check_metadata = False for part in 'logs', 'metadata': metafiles = [k for k in digests if k.startswith(_path_('.git/datalad/%s/' % part))] # This is in effect ONLY if we have "compatible" datalad installed on remote # end. ATM we don't have easy way to guarantee that AFAIK (yoh), # so let's not check/enforce (TODO) # assert(len(metafiles) >= 1) # we might have 2 logs if timestamps do not collide ;) # Let's actually do it to some degree if part == 'logs': # always should have those: assert (len(metafiles) >= 1) with open(opj(target_path, metafiles[0])) as f: if 'no datalad found' not in f.read(): check_metadata = True if part == 'metadata': eq_(len(metafiles), bool(check_metadata)) for f in metafiles: digests.pop(f) mtimes.pop(f) # and just pop some leftovers from annex for f in list(digests): if f.startswith('.git/annex/mergedrefs'): digests.pop(f) mtimes.pop(f) orig_digests, orig_mtimes = get_mtimes_and_digests(target_path) process_digests_mtimes(orig_digests, orig_mtimes) import time; time.sleep(0.1) # just so that mtimes change assert_create_sshwebserver(existing='reconfigure', **cpkwargs) digests, mtimes = get_mtimes_and_digests(target_path) process_digests_mtimes(digests, mtimes) assert_dict_equal(orig_digests, digests) # nothing should change in terms of content # but some files should have been modified modified_files = {k for k in mtimes if orig_mtimes.get(k, 0) != mtimes.get(k, 0)} # collect which files were expected to be modified without incurring any changes ok_modified_files = { _path_('.git/hooks/post-update'), 'index.html', # files which hook would manage to generate _path_('.git/info/refs'), '.git/objects/info/packs' } if external_versions['cmd:git'] >= '2.4': # on elderly git we don't change receive setting ok_modified_files.add(_path_('.git/config')) ok_modified_files.update({f for f in digests if f.startswith(_path_('.git/datalad/web'))}) assert_set_equal(modified_files, ok_modified_files) @skip_ssh @with_testrepos('submodule_annex', flavors=['local']) @with_tempfile(mkdir=True) @with_tempfile def test_target_ssh_recursive(origin, src_path, target_path): # prepare src source = install(src_path, source=origin, recursive=True)[0] sub1 = Dataset(opj(src_path, "subm 1")) sub2 = Dataset(opj(src_path, "subm 2")) for flat in False, True: target_path_ = target_dir_tpl = target_path + "-" + str(flat) if flat: target_dir_tpl += "/%NAME" sep = '-' else: sep = os.path.sep if flat: # now that create_sibling also does fetch -- the related problem # so skipping this early raise SkipTest('TODO: Make publish work for flat datasets, it currently breaks') remote_name = 'remote-' + str(flat) # TODO: there is f.ckup with paths so assert_create fails ATM # And let's test without explicit dataset being provided with chpwd(source.path): #assert_create_sshwebserver( create_sibling( target=remote_name, sshurl="ssh://localhost" + target_path_, target_dir=target_dir_tpl, recursive=True, ui=True) # raise if git repos were not created for suffix in [sep + 'subm 1', sep + 'subm 2', '']: target_dir = opj(target_path_, basename(src_path) if flat else "").rstrip(os.path.sep) + suffix # raise if git repos were not created GitRepo(target_dir, create=False) _test_correct_publish(target_dir, rootds=not suffix, flat=flat) for repo in [source.repo, sub1.repo, sub2.repo]: assert_not_in("local_target", repo.get_remotes()) # now, push should work: publish(dataset=source, to=remote_name) datalad-0.4.1/datalad/distribution/tests/test_uninstall.py000066400000000000000000000301661301115270200240010ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test uninstall action """ import os from os.path import join as opj, split as psplit from os.path import exists, lexists from os.path import realpath from os.path import isdir from glob import glob from datalad.api import uninstall from datalad.api import install from datalad.support.exceptions import InsufficientArgumentsError, CommandError from datalad.tests.utils import ok_ from datalad.tests.utils import eq_ from datalad.tests.utils import with_testrepos from datalad.tests.utils import SkipTest from datalad.tests.utils import assert_raises from datalad.tests.utils import assert_in from datalad.tests.utils import ok_file_under_git from datalad.tests.utils import ok_clean_git from datalad.tests.utils import with_tempfile from datalad.tests.utils import with_tree from datalad.utils import chpwd from datalad.support.external_versions import external_versions from ..dataset import Dataset @with_tempfile() def test_safetynet(path): ds = Dataset(path).create() os.makedirs(opj(ds.path, 'deep', 'down')) for p in (ds.path, opj(ds.path, 'deep'), opj(ds.path, 'deep', 'down')): with chpwd(p): # will never remove PWD, or anything outside the dataset for target in (ds.path, os.curdir, os.pardir, opj(os.pardir, os.pardir)): assert_raises(ValueError, uninstall, path=target, remove_handles=True) @with_tempfile() def test_uninstall_nonexisting(path): ds = Dataset(path) assert_raises(ValueError, uninstall, dataset=ds) @with_tempfile() def test_clean_subds_removal(path): ds = Dataset(path).create() subds1 = ds.create('one') ds.create('two') ds.save(auto_add_changes=True) eq_(sorted(ds.get_subdatasets()), ['one', 'two']) ok_clean_git(ds.path) # now kill one assert_raises(ValueError, ds.uninstall, 'one', remove_handles=True, remove_history=True) res = ds.uninstall('one', remove_handles=True, remove_history=True, recursive=True) eq_(res, [subds1]) ok_(not subds1.is_installed()) ok_clean_git(ds.path) # two must remain eq_(ds.get_subdatasets(), ['two']) # one is gone assert(not exists(subds1.path)) @with_testrepos('.*basic.*', flavors=['local']) def test_uninstall_invalid(path): assert_raises(InsufficientArgumentsError, uninstall) # makes no sense to call uninstall, but ask it to do nothing assert_raises(ValueError, uninstall, remove_handles=False, remove_data=False) ds = Dataset(path) # TODO make these two cases uniform if hasattr(ds.repo, 'drop'): assert_raises(Exception, uninstall, dataset=ds, path='not_existent') else: eq_(uninstall(dataset=ds, path='not_existent'), []) @with_testrepos('basic_annex', flavors=['clone']) def test_uninstall_annex_file(path): ds = Dataset(path) ok_(ds.is_installed()) ok_file_under_git(path, 'test-annex.dat', annexed=True) ds.repo.get('test-annex.dat') ok_(ds.repo.file_has_content('test-annex.dat')) # remove file's content: res = ds.uninstall(path='test-annex.dat') # test it happened: ok_(not ds.repo.file_has_content('test-annex.dat')) ok_file_under_git(path, 'test-annex.dat', annexed=True) # test result: eq_(res, ['test-annex.dat']) ds.repo.get('test-annex.dat') # remove file: ds.uninstall(path='test-annex.dat', remove_handles=True) assert_raises(AssertionError, ok_file_under_git, path, 'test-annex.dat', annexed=True) assert_raises(AssertionError, ok_file_under_git, path, 'test-annex.dat', annexed=False) ok_(not exists(opj(path, 'test-annex.dat'))) @with_testrepos('.*basic.*', flavors=['clone']) def test_uninstall_git_file(path): ds = Dataset(path) ok_(ds.is_installed()) ok_(exists(opj(path, 'INFO.txt'))) ok_file_under_git(path, 'INFO.txt') # uninstalling data only doesn't make sense: # this will only get a warning now #assert_raises(ValueError, ds.uninstall, path='INFO.txt', data_only=True) # uninstall removes the file: res = ds.uninstall(path='INFO.txt', remove_handles=True) assert_raises(AssertionError, ok_file_under_git, path, 'INFO.txt') ok_(not exists(opj(path, 'INFO.txt'))) eq_(res, ['INFO.txt']) @with_testrepos('submodule_annex', flavors=['local']) @with_tempfile(mkdir=True) def test_uninstall_subdataset(src, dst): ds = install(dst, source=src, recursive=True)[0] ok_(ds.is_installed()) for subds_path in ds.get_subdatasets(): subds = Dataset(opj(ds.path, subds_path)) ok_(subds.is_installed()) annexed_files = subds.repo.get_annexed_files() subds.repo.get(annexed_files) # uninstall data of subds: assert_raises(ValueError, ds.uninstall, path=subds_path) res = ds.uninstall(path=subds_path, recursive=True) ok_(all([f in res for f in annexed_files])) ok_(all([not i for i in subds.repo.file_has_content(annexed_files)])) # subdataset is still known assert_in(subds_path, ds.get_subdatasets()) for subds_path in ds.get_subdatasets(): # uninstall subds itself: assert_raises(ValueError, ds.uninstall, path=subds_path, remove_handles=True, remove_history=True) if os.environ.get('DATALAD_TESTS_DATALADREMOTE') \ and external_versions['git'] < '2.0.9': raise SkipTest("Known problem with GitPython. See " "https://github.com/gitpython-developers/GitPython/pull/521") res = ds.uninstall(path=subds_path, remove_handles=True, remove_history=True, recursive=True) subds = Dataset(opj(ds.path, subds_path)) eq_(res[0], subds) ok_(not subds.is_installed()) ok_(not exists(subds.path)) @with_tree({ 'deep': { 'dir': { 'keep': 'keep1', 'kill': 'kill1'}}, 'keep': 'keep2', 'kill': 'kill2'}) def test_uninstall_multiple_paths(path): ds = Dataset(path).create(force=True, save=False) subds = ds.create('deep', force=True, if_dirty='ignore') subds.add('.', recursive=True) ds.add('.', recursive=True) ds.save(auto_add_changes=True) ok_clean_git(ds.path) # drop content of all 'kill' files # must not work without recursive topfile = 'kill' deepfile = opj('deep', 'dir', 'kill') assert_raises(ValueError, ds.uninstall, [topfile, deepfile], check=False) # use a tuple not a list! should also work ds.uninstall((topfile, deepfile), recursive=True, check=False) ok_clean_git(ds.path) files_left = glob(opj(ds.path, '*', '*', '*')) + glob(opj(ds.path, '*')) ok_(all([f.endswith('keep') for f in files_left if exists(f) and not isdir(f)])) ok_(not ds.repo.file_has_content(topfile)) ok_(not subds.repo.file_has_content(opj(*psplit(deepfile)[1:]))) # drop handles for all 'kill' files ds.uninstall([topfile, deepfile], recursive=True, check=False, remove_handles=True) ok_clean_git(ds.path) files_left = glob(opj(ds.path, '*', '*', '*')) + glob(opj(ds.path, '*')) ok_(all([f.endswith('keep') for f in files_left if exists(f) and not isdir(f)])) ok_(not any([f.endswith(topfile) for f in files_left])) @with_tempfile() def test_uninstall_dataset(path): ds = Dataset(path) ok_(not ds.is_installed()) ds.create() ok_(ds.is_installed()) ok_clean_git(ds.path) # would only drop data ds.uninstall() # actually same as this, for cmdline compat reasons ds.uninstall(path=[]) ok_clean_git(ds.path) # removing all handles equal removal of entire dataset, needs safety switch assert_raises(ValueError, ds.uninstall, remove_handles=True) ds.uninstall(remove_handles=True, remove_history=True) # completely gone ok_(not ds.is_installed()) ok_(not exists(ds.path)) @with_tree({'one': 'test', 'two': 'test'}) def test_remove_file_handle_only(path): ds = Dataset(path).create(force=True) ds.add(os.curdir) ok_clean_git(ds.path) # make sure there is any key ok_(len(ds.repo.get_file_key('one'))) # both files link to the same key eq_(ds.repo.get_file_key('one'), ds.repo.get_file_key('two')) rpath_one = realpath(opj(ds.path, 'one')) eq_(rpath_one, realpath(opj(ds.path, 'two'))) path_two = opj(ds.path, 'two') ok_(exists(path_two)) # remove one handle, should not affect the other ds.uninstall('two', remove_data=False, remove_handles=True) eq_(rpath_one, realpath(opj(ds.path, 'one'))) ok_(exists(rpath_one)) ok_(not exists(path_two)) @with_tree({'deep': {'dir': {'test': 'testcontent'}}}) def test_uninstall_recursive(path): ds = Dataset(path).create(force=True, if_dirty='ignore') subds = ds.create('deep', force=True, if_dirty='ignore') # we add one file eq_(len(subds.add('.', if_dirty='ignore')), 1) # save all -> all clean ds.save(auto_add_changes=True, recursive=True) ok_clean_git(subds.path) ok_clean_git(ds.path) # now uninstall in subdataset through superdataset target_fname = opj('deep', 'dir', 'test') # sane starting point ok_(exists(opj(ds.path, target_fname))) # uninstallation fails without --recursive assert_raises(ValueError, ds.uninstall, target_fname) # doesn't have the minimum number of copies for a safe drop # TODO: better exception assert_raises(CommandError, ds.uninstall, target_fname, recursive=True) # this should do it ds.uninstall(target_fname, check=False, recursive=True) # link is dead lname = opj(ds.path, target_fname) ok_(not exists(lname)) # entire hierarchy saved ok_clean_git(subds.path) ok_clean_git(ds.path) # now same with actual handle removal assert_raises(ValueError, ds.uninstall, target_fname, remove_handles=True) # content is dropped already, so no checks in place anyway ds.uninstall(target_fname, check=True, remove_handles=True, recursive=True) ok_(not exists(lname) and not lexists(lname)) ok_clean_git(subds.path) ok_clean_git(ds.path) @with_tempfile() def test_remove_dataset_hierarchy(path): ds = Dataset(path).create() ds.create('deep') ds.save(auto_add_changes=True) ok_clean_git(ds.path) # fail on missing `remove_handles`, always needs to come with `remove_handles` assert_raises(ValueError, ds.uninstall, remove_history=True) # fail on missing --recursive because subdataset is present assert_raises(ValueError, ds.uninstall, remove_handles=True, remove_history=True) ds.uninstall(remove_history=True, remove_handles=True, recursive=True) # completely gone ok_(not ds.is_installed()) ok_(not exists(ds.path)) @with_tempfile() def test_careless_subdataset_uninstall(path): # nested datasets ds = Dataset(path).create() subds1 = ds.create('deep1') ds.create('deep2') eq_(sorted(ds.get_subdatasets()), ['deep1', 'deep2']) ok_clean_git(ds.path) # now we kill the sub without the parent knowing subds1.uninstall(remove_history=True, remove_handles=True) ok_(not exists(subds1.path)) ok_(ds.repo.dirty) # parent still knows the sub eq_(sorted(ds.get_subdatasets()), ['deep1', 'deep2']) # save the parent later on ds.save(auto_add_changes=True) # subds still gone # subdataset appearance is normalized to an empty directory ok_(exists(subds1.path)) # parent still knows the sub eq_(ds.get_subdatasets(), ['deep1', 'deep2']) # and they lived happily ever after ok_clean_git(ds.path) @with_tempfile() def test_kill(path): # nested datasets with load ds = Dataset(path).create() with open(opj(ds.path, "file.dat"), 'w') as f: f.write("load") ds.repo.add("file.dat") subds1 = ds.create('deep1') eq_(sorted(ds.get_subdatasets()), ['deep1']) ok_clean_git(ds.path) # and we fail to uninstall since content can't be dropped assert_raises(CommandError, ds.uninstall) eq_(ds.uninstall(kill=True), [path]) ok_(not exists(path)) datalad-0.4.1/datalad/distribution/tests/test_update.py000066400000000000000000000125661301115270200232560ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test update action """ import os from os.path import join as opj, abspath, basename from ..dataset import Dataset from datalad.api import update, install, uninstall from datalad.utils import chpwd from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from nose.tools import ok_, eq_, assert_false, assert_is_instance from datalad.tests.utils import with_tempfile, assert_in, with_tree,\ with_testrepos, assert_not_in from datalad.tests.utils import SkipTest from datalad.tests.utils import assert_cwd_unchanged, skip_if_on_windows from datalad.tests.utils import assure_dict_from_str, assure_list_from_str from datalad.tests.utils import create_tree from datalad.tests.utils import assert_not_in from datalad.tests.utils import assert_raises from datalad.tests.utils import ok_file_has_content from datalad.tests.utils import skip_if_no_module from datalad.tests.utils import ok_clean_git, swallow_outputs @with_testrepos('submodule_annex', flavors=['local']) #TODO: Use all repos after fixing them @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_update_simple(origin, src_path, dst_path): # prepare src source = install(src_path, source=origin, recursive=True)[0] # forget we cloned it (provide no 'origin' anymore), which should lead to # setting tracking branch to target: source.repo.remove_remote("origin") # get a clone to update later on: dest = install(dst_path, source=src_path, recursive=True)[0] # test setup done; # assert all fine ok_clean_git(dst_path) ok_clean_git(src_path) # update yields nothing => up-to-date # TODO: how to test besides not failing? dest.update() ok_clean_git(dst_path) # modify origin: with open(opj(src_path, "update.txt"), "w") as f: f.write("Additional content") source.add(path="update.txt") source.save("Added update.txt") ok_clean_git(src_path) # update without `merge` only fetches: dest.update() # modification is not known to active branch: assert_not_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # modification is known to branch origin/master assert_in("update.txt", dest.repo.get_files("origin/master")) # merge: dest.update(merge=True) # modification is now known to active branch: assert_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # it's known to annex, but has no content yet: dest.repo.get_file_key("update.txt") # raises if unknown eq_([False], dest.repo.file_has_content(["update.txt"])) # smoke-test if recursive update doesn't fail if submodule is removed dest.uninstall('subm 1', kill=True) dest.update(recursive=True) dest.update(merge=True, recursive=True) # and now test recursive update with merging in differences create_tree(opj(source.path, 'subm 2'), {'load.dat': 'heavy'}) source.save(message="saving changes within subm2", recursive=True, auto_add_changes=True) dest.update(merge=True, recursive=True) # and now we can get new file dest.get('subm 2/load.dat') ok_file_has_content(opj(dest.path, 'subm 2', 'load.dat'), 'heavy') def test_update_recursive(): raise SkipTest("TODO more tests to add to above ones") @with_testrepos('.*annex.*', flavors=['clone']) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_update_fetch_all(src, remote_1, remote_2): rmt1 = AnnexRepo(remote_1, src) rmt2 = AnnexRepo(remote_2, src) ds = Dataset(src) ds.add_sibling(name="sibling_1", url=remote_1) ds.add_sibling(name="sibling_2", url=remote_2) # modify the remotes: with open(opj(remote_1, "first.txt"), "w") as f: f.write("some file load") rmt1.add("first.txt", commit=True) # TODO: Modify an already present file! with open(opj(remote_2, "second.txt"), "w") as f: f.write("different file load") rmt2.add("second.txt", git=True, commit=True, msg="Add file to git.") # fetch all remotes ds.update(fetch_all=True) # no merge, so changes are not in active branch: assert_not_in("first.txt", ds.repo.get_files(ds.repo.get_active_branch())) assert_not_in("second.txt", ds.repo.get_files(ds.repo.get_active_branch())) # but we know the changes in remote branches: assert_in("first.txt", ds.repo.get_files("sibling_1/master")) assert_in("second.txt", ds.repo.get_files("sibling_2/master")) # no merge strategy for multiple remotes yet: assert_raises(NotImplementedError, ds.update, merge=True, fetch_all=True) # merge a certain remote: ds.update(name="sibling_1", merge=True) # changes from sibling_2 still not present: assert_not_in("second.txt", ds.repo.get_files(ds.repo.get_active_branch())) # changes from sibling_1 merged: assert_in("first.txt", ds.repo.get_files(ds.repo.get_active_branch())) # it's known to annex, but has no content yet: ds.repo.get_file_key("first.txt") # raises if unknown eq_([False], ds.repo.file_has_content(["first.txt"])) datalad-0.4.1/datalad/distribution/tests/test_utils.py000066400000000000000000000103231301115270200231210ustar00rootroot00000000000000# ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test distribution utils """ import logging import os from os.path import join as opj from os.path import isdir from os.path import exists from os.path import realpath from os.path import basename from os.path import dirname from mock import patch from datalad.distribution.utils import _get_flexible_source_candidates from datalad.distribution.utils import _get_flexible_source_candidates_for_submodule from datalad.tests.utils import create_tree from datalad.tests.utils import with_tempfile from datalad.tests.utils import assert_in from datalad.tests.utils import with_tree from datalad.tests.utils import with_testrepos from datalad.tests.utils import eq_ from datalad.tests.utils import ok_ from datalad.tests.utils import assert_false from datalad.tests.utils import ok_file_has_content from datalad.tests.utils import assert_not_in from datalad.tests.utils import assert_raises from datalad.tests.utils import ok_startswith from datalad.tests.utils import ok_clean_git from datalad.tests.utils import serve_path_via_http from datalad.tests.utils import swallow_logs from datalad.tests.utils import use_cassette from datalad.tests.utils import skip_if_no_network from datalad.utils import _path_ from datalad.utils import rmtree def test_get_flexible_source_candidates(): f = _get_flexible_source_candidates # for http and https (dummy transport) we should get /.git source added eq_(f('http://e.c'), ['http://e.c', 'http://e.c/.git']) eq_(f('http://e.c/s/p'), ['http://e.c/s/p', 'http://e.c/s/p/.git']) # for those candidates should be just the original address, since git # understands those just fine for s in ('http://e.c/.git', '/', 'relative/path', 'smallrelative', './neighbor', '../../look/into/parent/bedroom', 'p:somewhere', 'user@host:/full/path', ): eq_(_get_flexible_source_candidates(s), [s]) # Now a few relative ones eq_(f('../r', '.'), ['../r']) eq_(f('../r', 'ssh://host/path'), ['ssh://host/r']) eq_(f('sub', 'ssh://host/path'), ['ssh://host/path/sub']) eq_(f('../r', 'http://e.c/p'), ['http://e.c/r', 'http://e.c/r/.git']) eq_(f('sub', 'http://e.c/p'), ['http://e.c/p/sub', 'http://e.c/p/sub/.git']) # tricky ones eq_(f('sub', 'http://e.c/p/.git'), ['http://e.c/p/sub/.git']) eq_(f('../s1/s2', 'http://e.c/p/.git'), ['http://e.c/s1/s2/.git']) # incorrect ones will stay incorrect eq_(f('../s1/s2', 'http://e.c/.git'), ['http://e.c/../s1/s2/.git']) # when source is not relative, but base_url is specified as just the destination path, # not really a "base url" as name was suggesting, then it should be ignored eq_(f('http://e.c/p', '/path'), ['http://e.c/p', 'http://e.c/p/.git']) @with_tempfile @with_tempfile def test_get_flexible_source_candidates_for_submodule(t, t2): f = _get_flexible_source_candidates_for_submodule # for now without mocking -- let's just really build a dataset from datalad.api import create from datalad.api import install ds = create(t) clone = install(t2, source=t) # first one could just know about itself or explicit url provided sshurl = 'ssh://e.c' httpurl = 'http://e.c' sm_httpurls = [httpurl, httpurl + '/.git'] eq_(f(ds, 'sub'), []) eq_(f(ds, 'sub', sshurl), [sshurl]) eq_(f(ds, 'sub', httpurl), sm_httpurls) eq_(f(ds, 'sub', None), []) # otherwise really we have no clue were to get from # but if we work on dsclone then it should also add urls deduced from its # own location default remote for current branch eq_(f(clone, 'sub'), [t + '/sub']) eq_(f(clone, 'sub', sshurl), [t + '/sub', sshurl]) eq_(f(clone, 'sub', httpurl), [t + '/sub'] + sm_httpurls) eq_(f(clone, 'sub'), [t + '/sub']) # otherwise really we have no clue were to get from # TODO: check that http:// urls for the dataset itself get resolved # TODO: many more!!datalad-0.4.1/datalad/distribution/uninstall.py000066400000000000000000000275761301115270200216130ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for uninstalling dataset content """ __docformat__ = 'restructuredtext' import os import logging from os.path import relpath, split as psplit from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.param import Parameter from datalad.support.constraints import EnsureStr, EnsureNone from datalad.distribution.dataset import Dataset, EnsureDataset, \ datasetmethod, resolve_path, require_dataset from datalad.interface.base import Interface from datalad.interface.save import Save from datalad.interface.common_opts import if_dirty_opt from datalad.interface.common_opts import recursion_flag from datalad.interface.utils import handle_dirty_dataset from datalad.utils import rmtree from datalad.utils import getpwd from datalad.utils import assure_list lgr = logging.getLogger('datalad.distribution.uninstall') def _uninstall(ds, paths, check, remove_history, remove_data, remove_handles, recursive): # all input paths are assumed to belong to the given dataset, # and in particular not to any subdataset # recode paths to be relative to the root of the dataset paths = [relpath(p, start=ds.path) for p in paths] if os.curdir in paths: # we can take a shortcut if the entire thing goes away lgr.debug('uninstall entire content in {}'.format(ds)) paths = [os.curdir] results = [] # at this point we can only have two things: # 1. files and directories in the current dataset # 2. the entire dataset # alway drop data first if remove_data: if hasattr(ds.repo, 'drop'): opts = ['--force'] if not check else [] results.extend(ds.repo.drop(paths, options=opts)) else: lgr.warning("cannot uninstall data only, not an annex dataset") if not remove_handles: # we are done here return results, False if os.curdir in paths: if not remove_history: raise ValueError( "will not remove the entire dataset (with history) unless forced") # special mode that makes everything disappear, including subdatasets for subds in ds.get_subdatasets( absolute=True, fulfilled=True, recursive=True, recursion_limit=1): if not recursive: raise ValueError( "will not remove subdatasets without the recursive flag") subds = Dataset(subds) lgr.warning("removing subdataset {} from {}".format(subds, ds)) res, gone = _uninstall( subds, [subds.path], check=check, remove_history=remove_history, remove_data=True, # we always want everything to go at this point remove_handles=True, recursive=recursive) results.extend(res) results.append(ds) return results, True # and now make the handles disappear # always recurse into directories results.extend(ds.repo.remove(paths, r=True)) return results, False class Uninstall(Interface): """Uninstall a dataset component or entire dataset(s) """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), metavar="DATASET", doc="""specify the dataset to perform the uninstall operation on. If no dataset is given, an attempt is made to identify the dataset based on the current working directory and/or the `path` given""", constraints=EnsureDataset() | EnsureNone()), path=Parameter( args=("path",), metavar="PATH", doc="path/name of the component to be uninstalled", nargs="*", constraints=EnsureStr() | EnsureNone()), remove_data=Parameter( args=("--dont-remove-data",), doc="""whether to drop data associated with matching file handles during uninstallation.[CMD: This option prevents data from being dropped CMD]""", action="store_false", dest='remove_data'), remove_handles=Parameter( args=("--remove-handles",), doc="""if given, matching file handles are removed. This flag is required for deleting entire datasets""", action="store_true"), recursive=recursion_flag, check=Parameter( args=("--nocheck",), doc="""whether to perform checks to assure the configured minimum number (remote) source for data to be uninstalled.[CMD: Give this option to skip checks CMD]""", action="store_false", dest='check'), remove_history=Parameter( args=("--remove-history",), doc="""whether to permit operations that remove recorded dataset history, for example when removing entire datasets completely. Such changes are not recoverable, use with care""", action="store_true",), kill=Parameter( args=("--kill",), action="store_true", doc="""**WARNING -- extremely dangerous**. It will simply force remove, without consideration of being a dataset, a file, or a directory or any other option given to uninstall. To be used only with full awareness of its consequences""" ), if_dirty=if_dirty_opt, ) @staticmethod @datasetmethod(name='uninstall') def __call__( path=None, dataset=None, remove_data=True, remove_handles=False, recursive=False, remove_history=False, check=True, kill=False, if_dirty='save-before'): # upfront check prior any resolution attempt to avoid disaster if dataset is None and not path: raise InsufficientArgumentsError( "insufficient information for uninstallation (needs at " "least a dataset or a path. To uninstall an entire dataset " "it needs to be given explicitly.") if remove_history and not remove_handles: raise ValueError("`remove_history` flag, requires `remove_handles` flag") if not remove_data and not remove_handles: raise ValueError("instructed to neither drop data, nor remove handles: cannot perform") results = [] ds = require_dataset( dataset, check_installed=True, purpose='uninstall') # always yields list; empty if None path = assure_list(path) if not len(path): # AKA "everything" path.append(ds.path) # XXX Important to resolve against `dataset` input argument, and # not against the `ds` resolved dataset path = [resolve_path(p, dataset) for p in path] if kill: lgr.warning("Force-removing %d paths", len(path)) for p in path: rmtree(p) results.append(p) return results # make sure we get to an expected state handle_dirty_dataset(ds, if_dirty) # sort paths into the respective datasets that contain them # considering 1st-level subdatasets at most # NOTE: little dance with two dicts is necessary, because ATM our # Datasets are not hashable enough for PY3 whocares_paths = {} whocares_ds = {} pwd = getpwd() for p in path: if remove_handles: # behave like `rm -r` and refuse to remove where we are rpath = relpath(p, start=pwd) if rpath == os.curdir \ or rpath == os.pardir \ or set(psplit(rpath)) == {os.pardir}: raise ValueError( "refusing to remove current or parent directory") containerds = ds.get_containing_subdataset(p, recursion_limit=1) if not recursive and containerds.path != ds.path: raise ValueError( "will not uninstall content in subdatasets without the recursive flag") ps = whocares_paths.get(containerds.path, []) ps.append(p) whocares_paths[containerds.path] = ps whocares_ds[containerds.path] = containerds ds_gonealready = False if ds.path in whocares_paths: # start with the content of this dataset, as any somewhat # total recursive removal here would have most impact lgr.debug("Uninstall content in {}".format(ds)) res, ds_gonealready = _uninstall( whocares_ds[ds.path], whocares_paths[ds.path], check=check, remove_history=remove_history, remove_data=remove_data, remove_handles=remove_handles, recursive=recursive) results.extend(res) if ds_gonealready: rmtree(ds.path) # the underlying repo is gone, the assert makes sure that the Dataset # instance becomes aware of that assert(not ds.is_installed()) return results # otherwise deal with any other subdataset for subdspath in whocares_paths: subds = whocares_ds[subdspath] subdsrelpath = relpath(subdspath, start=ds.path) if subds == ds: continue res, subds_gone = _uninstall( subds, whocares_paths[subdspath], check=check, remove_history=remove_history, remove_data=remove_data, remove_handles=remove_handles, recursive=recursive) results.extend(res) if subds_gone: # clean divorce, if we lost the subds in the process # find the submodule that matches the patch # regular access goes by name, but we cannot trust # our own consistency, yet submodule = [sm for sm in ds.repo.repo.submodules if sm.path == subdsrelpath][0] submodule.remove() elif remove_handles: # we could have removed handles -> save Save.__call__( message='[DATALAD] uninstalled content', dataset=subds, auto_add_changes=False, recursive=False) # add this change to the parent, but don't save, will do in # one go below ds.repo.add(subdsrelpath, git=True) if remove_handles: # something of the original dataset is left at this point # and all subdatasets have been saved already # -> save changes Save.__call__( message='[DATALAD] uninstalled content', dataset=ds, auto_add_changes=False, recursive=False) return results @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui if not res: ui.message("Nothing was uninstalled") return msg = "{n} {obj} uninstalled:\n".format( obj='items were' if len(res) > 1 else 'item was', n=len(res)) for item in res: if isinstance(item, Dataset): msg += "Dataset: %s\n" % item.path else: msg += "File: %s\n" % item ui.message(msg) datalad-0.4.1/datalad/distribution/update.py000066400000000000000000000163671301115270200210600ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for updating a dataset """ __docformat__ = 'restructuredtext' import logging from os.path import join as opj from datalad.interface.base import Interface from datalad.support.constraints import EnsureStr from datalad.support.constraints import EnsureNone from datalad.support.gitrepo import GitRepo from datalad.support.param import Parameter from datalad.utils import knows_annex from .dataset import Dataset from .dataset import EnsureDataset from .dataset import datasetmethod from .dataset import require_dataset lgr = logging.getLogger('datalad.distribution.update') class Update(Interface): """Update a dataset from a sibling. """ _params_ = dict( name=Parameter( args=("name",), doc="""name of the sibling to update from""", nargs="?", constraints=EnsureStr() | EnsureNone()), dataset=Parameter( args=("-d", "--dataset"), doc=""""specify the dataset to update. If no dataset is given, an attempt is made to identify the dataset based on the current working directory""", constraints=EnsureDataset() | EnsureNone()), merge=Parameter( args=("--merge",), action="store_true", doc="merge changes from sibling `name` or the remote branch, " "configured to be the tracking branch if no sibling was " "given", ), # TODO: How to document it without using the term 'tracking branch'? recursive=Parameter( args=("-r", "--recursive"), action="store_true", doc="""if set this updates all possibly existing subdatasets, too"""), fetch_all=Parameter( args=("--fetch-all",), action="store_true", doc="fetch updates from all siblings", ), reobtain_data=Parameter( args=("--reobtain-data",), action="store_true", doc="TODO"), ) @staticmethod @datasetmethod(name='update') def __call__(name=None, dataset=None, merge=False, recursive=False, fetch_all=False, reobtain_data=False): """ """ # TODO: Is there an 'update filehandle' similar to install and publish? # What does it mean? if reobtain_data: # TODO: properly define, what to do raise NotImplementedError("TODO: Option '--reobtain-data' not " "implemented yet.") # shortcut ds = require_dataset(dataset, check_installed=True, purpose='updating') assert (ds.repo is not None) repos_to_update = [ds.repo] if recursive: repos_to_update += [GitRepo(opj(ds.path, sub_path), create=False) for sub_path in ds.get_subdatasets(recursive=True, fulfilled=True)] # only work on those which are installed # TODO: current implementation disregards submodules organization, # it just updates/merge each one individually whenever in the simplest # case we just need a call to # git submodule update --recursive # if name was not provided, and there is no --merge # If we do --merge we should at the end call save for repo in repos_to_update: # get all remotes which have references (would exclude # special remotes) remotes = repo.get_remotes(with_refs_only=True) if name and name not in remotes: lgr.warning("'%s' not known to dataset %s.\nSkipping" % (name, repo.path)) continue # Currently '--merge' works for single remote only: # TODO: - condition still incomplete # - We can merge if a remote was given or there is a # tracking branch # - we also can fetch all remotes independently on whether or # not we merge a certain remote if not name and len(remotes) > 1 and merge: lgr.debug("Found multiple remotes:\n%s" % remotes) raise NotImplementedError("No merge strategy for multiple " "remotes implemented yet.") lgr.info("Updating dataset '%s' ..." % repo.path) # fetch remote(s): repo.fetch(remote=name, all_=fetch_all) # if `repo` is an annex and we didn't fetch the entire remote # anyway, explicitly fetch git-annex branch: # TODO: This isn't correct. `fetch_all` fetches all remotes. # Apparently, we currently fetch an entire remote anyway. Is this # what we want? Do we want to specify a refspec instead? # yoh: we should leave it to git and its configuration. # So imho we should just extract to fetch everything git would fetch if knows_annex(repo.path) and not fetch_all: if name: # we are updating from a certain remote, so git-annex branch # should be updated from there as well: repo.fetch(remote=name) # TODO: what does failing here look like? else: # we have no remote given, therefore # check for tracking branch's remote: track_remote, track_branch = repo.get_tracking_branch() if track_remote: # we have a "tracking remote" repo.fetch(remote=track_remote) # merge: if merge: lgr.info("Applying changes from tracking branch...") # TODO: Adapt. # TODO: Rethink default remote/tracking branch. See above. # We need a "tracking remote" but custom refspec to fetch from # that remote cmd_list = ["git", "pull"] if name: cmd_list.append(name) # branch needed, if not default remote # => TODO: use default remote/tracking branch to compare # (see above, where git-annex is fetched) # => TODO: allow for passing a branch # (or more general refspec?) # For now, just use the same name cmd_list.append(repo.get_active_branch()) std_out, std_err = repo._git_custom_command('', cmd_list) lgr.info(std_out) if knows_annex(repo.path): # annex-apply: lgr.info("Updating annex ...") std_out, std_err = repo._git_custom_command( '', ["git", "annex", "merge"]) lgr.info(std_out) # TODO: return value? datalad-0.4.1/datalad/distribution/utils.py000066400000000000000000000405331301115270200207260ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Distribution utility functions """ import logging from os import linesep from os.path import exists from os.path import isdir from os.path import join as opj from os.path import islink from os.path import isabs from os.path import relpath from os.path import curdir from os.path import normpath from six.moves.urllib.parse import quote as urlquote from datalad.support.gitrepo import GitRepo from datalad.support.gitrepo import GitCommandError from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import PathOutsideRepositoryError from datalad.support.exceptions import InstallFailedError from datalad.support.network import DataLadRI from datalad.support.network import URL from datalad.support.network import RI from datalad.support.network import PathRI from datalad.dochelpers import exc_str from datalad.utils import swallow_logs from datalad.utils import rmtree from datalad.utils import knows_annex from datalad.utils import with_pathsep as _with_sep from .dataset import Dataset lgr = logging.getLogger('datalad.distribution.utils') def _install_subds_inplace(ds, path, relativepath, name=None): """Register an existing repository in the repo tree as a submodule""" ds.repo.add_submodule(relativepath, url=None, name=name) _fixup_submodule_dotgit_setup(ds, relativepath) # return newly added submodule as a dataset return Dataset(path) def _fixup_submodule_dotgit_setup(ds, relativepath): """Implementation of our current of .git in a subdataset Each subdataset/module has its own .git directory where a standalone repository would have it. No gitdir files, no symlinks. """ # move .git to superrepo's .git/modules, remove .git, create # .git-file path = opj(ds.path, relativepath) src_dotgit = get_git_dir(path) # at this point install always yields the desired result # just make sure assert(src_dotgit == '.git') def get_git_dir(path): """figure out a repo's gitdir '.git' might be a directory, a symlink or a file Parameter --------- path: str currently expected to be the repos base dir Returns ------- str relative path to the repo's git dir; So, default would be ".git" """ from os.path import isfile from os import readlink dot_git = opj(path, ".git") if not exists(dot_git): raise RuntimeError("Missing .git in %s." % path) elif islink(dot_git): git_dir = readlink(dot_git) elif isdir(dot_git): git_dir = ".git" elif isfile(dot_git): with open(dot_git) as f: git_dir = f.readline() if git_dir.startswith("gitdir:"): git_dir = git_dir[7:] git_dir = git_dir.strip() return git_dir # TODO: evolved to sth similar to get_containing_subdataset. Therefore probably # should move into this one as an option. But: consider circular imports! def install_necessary_subdatasets(ds, path, reckless): """Installs subdatasets of `ds`, that are necessary to obtain in order to have access to `path`. Gets the subdataset containing `path` regardless of whether or not it was already installed. While doing so, installs everything necessary in between the uppermost installed one and `path`. Note: `ds` itself has to be installed. Parameters ---------- ds: Dataset path: str reckless: bool Returns ------- Dataset the last (deepest) subdataset, that was installed """ assert ds.is_installed() # figuring out what dataset to start with: start_ds = ds.get_containing_subdataset(path, recursion_limit=None) if start_ds.is_installed(): return start_ds # we try to install subdatasets as long as there is anything to # install in between the last one installed and the actual thing # to get (which is `path`): cur_subds = start_ds # Note, this is not necessarily `ds`: # MIH: would be good to know why? cur_par_ds = cur_subds.get_superdataset() assert cur_par_ds is not None while not cur_subds.is_installed(): lgr.info("Installing subdataset %s%s", cur_subds, ' in order to get %s' % path if cur_subds.path != path else '') # get submodule info submodules = cur_par_ds.repo.get_submodules() submodule = [sm for sm in submodules if sm.path == relpath(cur_subds.path, start=cur_par_ds.path)][0] # install using helper that give some flexibility regarding where to # get the module from _install_subds_from_flexible_source( cur_par_ds, submodule.path, submodule.url, reckless) cur_par_ds = cur_subds # Note: PathOutsideRepositoryError should not happen here. # If so, there went something fundamentally wrong, so raise something # different, to not let the caller mix it up with a "regular" # PathOutsideRepositoryError from above. (Although it could be # detected via its `repo` attribute) try: cur_subds = \ cur_subds.get_containing_subdataset(path, recursion_limit=None) except PathOutsideRepositoryError as e: raise RuntimeError("Unexpected failure: {0}".format(exc_str(e))) return cur_subds def _get_git_url_from_source(source): """Return URL for cloning associated with a source specification For now just resolves DataLadRIs """ # TODO: Probably RF this into RI.as_git_url(), that would be overridden # by subclasses or sth. like that if not isinstance(source, RI): source_ri = RI(source) else: source_ri = source if isinstance(source_ri, DataLadRI): # we have got our DataLadRI as the source, so expand it source = source_ri.as_git_url() else: source = str(source_ri) return source def _install_subds_from_flexible_source(ds, sm_path, sm_url, reckless): """Tries to obtain a given subdataset from several meaningful locations""" # compose a list of candidate clone URLs clone_urls = _get_flexible_source_candidates_for_submodule( ds, sm_path, sm_url) # now loop over all candidates and try to clone subds = Dataset(opj(ds.path, sm_path)) try: clone_url = _clone_from_any_source(clone_urls, subds.path) except GitCommandError as e: raise InstallFailedError( msg="Failed to install %s from %s (%s)" % ( subds, clone_urls, exc_str(e)) ) # do fancy update if sm_path in ds.get_subdatasets(absolute=False, recursive=False): lgr.debug("Update cloned subdataset {0} in parent".format(subds)) ds.repo.update_submodule(sm_path, init=True) else: # submodule is brand-new and previously unknown ds.repo.add_submodule(sm_path, url=clone_url) _fixup_submodule_dotgit_setup(ds, sm_path) _handle_possible_annex_dataset(subds, reckless) return subds def _get_tracking_source(ds): """Returns name and url of a potential configured source tracking remote""" vcs = ds.repo repo = vcs.repo # if we have a remote, let's check the location of that remote # for the presence of the desired submodule tracking_branch = repo.active_branch.tracking_branch() remote_name = None remote_url = '' if tracking_branch: # name of the default remote for the active branch remote_name = repo.active_branch.tracking_branch().remote_name remote_url = vcs.get_remote_url(remote_name, push=False) return remote_name, remote_url def _get_flexible_source_candidates(src, base_url=None): """Get candidates to try cloning from. Primarily to mitigate the problem that git doesn't append /.git while cloning from non-bare repos over dummy protocol (http*). Also to simplify creation of urls whenever base url and relative path within it provided Parameters ---------- src : string or RI Full or relative (then considered within base_url if provided) path base_url : string or RI, optional Returns ------- candidates : list of str List of RIs (path, url, ssh targets) to try to install from """ candidates = [] ri = RI(src) if isinstance(ri, PathRI) and not isabs(ri.path) and base_url: ri = RI(base_url) if ri.path.endswith('/.git'): base_path = ri.path[:-5] base_suffix = '.git' else: base_path = ri.path base_suffix = '' ri.path = normpath(opj(base_path, src, base_suffix)) src = str(ri) candidates.append(src) if isinstance(ri, URL): if ri.scheme in {'http', 'https'}: # additionally try to consider .git: if not src.rstrip('/').endswith('/.git'): candidates.append( '{0}/.git'.format(src.rstrip('/'))) # TODO: # We need to provide some error msg with InstallFailedError, since now # it just swallows everything. # yoh: not sure if this comment applies here, but could be still applicable # outisde return candidates def _get_flexible_source_candidates_for_submodule(ds, sm_path, sm_url=None): """Retrieve candidates from where to install the submodule Even if url for submodule is provided explicitly -- first tries urls under parent's module tracking branch remote. TODO: reconsider? yoh just maintained prev behavior for now """ clone_urls = [] # if we have a remote, let's check the location of that remote # for the presence of the desired submodule remote_name, remote_url = _get_tracking_source(ds) # Directly on parent's ds url if remote_url: # attempt: submodule checkout at parent remote URL # We might need to quote sm_path portion, e.g. for spaces etc if isinstance(RI(remote_url), URL): sm_path_url = urlquote(sm_path) else: sm_path_url = sm_path clone_urls += [ u for u in _get_flexible_source_candidates(sm_path_url, remote_url) if u not in clone_urls ] # attempt: provided (configured?) submodule URL # TODO: consider supporting DataLadRI here? or would confuse # git and we wouldn't want that (i.e. not allow pure git clone # --recursive) if sm_url: clone_urls += _get_flexible_source_candidates( sm_url, remote_url if remote_url else ds.path) return clone_urls def _clone_from_any_source(sources, dest): # should not be the case, but we need to distinguish between failure # of git-clone, due to existing target and an unsuccessful clone # attempt. See below. existed = dest and exists(dest) for source_ in sources: try: lgr.debug("Retrieving a dataset from URL: " "{0}".format(source_)) with swallow_logs(): GitRepo(dest, url=source_, create=True) return source_ # do not bother with other sources if succeeded except GitCommandError as e: lgr.debug("Failed to retrieve from URL: " "{0}".format(source_)) if not existed and dest \ and exists(dest): lgr.debug("Wiping out unsuccessful clone attempt at " "{}".format(dest)) rmtree(dest) if source_ == sources[-1]: # Note: The following block is evaluated whenever we # fail even with the last try. Not nice, but currently # necessary until we get a more precise exception: #################################### # TODO: We may want to introduce a --force option to # overwrite the target. # TODO: Currently assuming if `existed` and there is a # GitCommandError means that these both things are connected. # Need newer GitPython to get stderr from GitCommandError # (already fixed within GitPython.) if existed: # rudimentary check for an installed dataset at target: # (TODO: eventually check for being the one, that this # is about) dest_ds = Dataset(dest) if dest_ds.is_installed(): lgr.info("{0} appears to be installed already." "".format(dest_ds)) break else: lgr.warning("Target {0} already exists and is not " "an installed dataset. Skipped." "".format(dest)) # Keep original in debug output: lgr.debug("Original failure:{0}" "{1}".format(linesep, exc_str(e))) return None ################## # Re-raise if failed even with the last candidate lgr.debug("Unable to establish repository instance at " "{0} from {1}" "".format(dest, sources)) raise def _recursive_install_subds_underneath(ds, recursion_limit, reckless, start=None): content_by_ds = {} if isinstance(recursion_limit, int) and recursion_limit <= 0: return content_by_ds # loop over submodules not subdatasets to get the url right away # install using helper that give some flexibility regarding where to # get the module from for sub in ds.repo.get_submodules(): subds = Dataset(opj(ds.path, sub.path)) if start is not None and not subds.path.startswith(_with_sep(start)): # this one we can ignore, not underneath the start path continue if not subds.is_installed(): try: lgr.info("Installing subdataset %s", subds.path) subds = _install_subds_from_flexible_source( ds, sub.path, sub.url, reckless) # we want the entire thing, but mark this subdataset # as automatically installed content_by_ds[subds.path] = [curdir] except Exception as e: # skip, if we didn't manage to install subdataset lgr.warning( "Installation of subdatasets %s failed, skipped", subds) lgr.debug("Installation attempt failed with exception: %s", exc_str(e)) continue # otherwise recurse # we can skip the start expression, we know we are within content_by_ds.update(_recursive_install_subds_underneath( subds, recursion_limit=recursion_limit - 1 if isinstance(recursion_limit, int) else recursion_limit, reckless=reckless )) return content_by_ds def _handle_possible_annex_dataset(dataset, reckless): # in any case check whether we need to annex-init the installed thing: if knows_annex(dataset.path): # init annex when traces of a remote annex can be detected if reckless: lgr.debug( "Instruct annex to hardlink content in %s from local " "sources, if possible (reckless)", dataset.path) dataset.config.add( 'annex.hardlink', 'true', where='local', reload=True) lgr.debug("Initializing annex repo at %s", dataset.path) repo = AnnexRepo(dataset.path, init=True) if reckless: repo._run_annex_command('untrust', annex_options=['here']) def _save_installed_datasets(ds, installed_datasets): paths = [relpath(subds.path, ds.path) for subds in installed_datasets] paths_str = ", ".join(paths) msg = "installed subdataset{}: {}".format( "s" if len(paths_str) > 1 else "", paths_str) lgr.info("Saving possible changes to {0} - {1}".format( ds, msg)) ds.save( files=paths, message='[DATALAD] ' + msg, auto_add_changes=False, recursive=False) datalad-0.4.1/datalad/dochelpers.py000066400000000000000000000244061301115270200172000ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Utils to help with docstrings etc. Largerly borrowed from PyMVPA (as of upstream/2.4.1-23-g170496e). Copyright of the same developers as DataLad """ import logging import re import textwrap import os import sys import traceback lgr = logging.getLogger("datalad.docutils") __add_init2doc = False __in_ipython = False # TODO: determine exists('running ipython env') # if ran within IPython -- might need to add doc to init if __in_ipython: __rst_mode = False # either to do ReST links at all # if versions['ipython'] <= '0.8.1': # __add_init2doc = True else: __rst_mode = True # # Predefine some sugarings depending on syntax convention to be used # # XXX Might need to be removed or become proper cfg parameter __rst_conventions = 'numpy' if __rst_conventions == 'epydoc': _rst_sep = "`" _rst_indentstr = " " def _rst_section(section_name): """Provide section heading""" return ":%s:" % section_name elif __rst_conventions == 'numpy': _rst_sep = "" _rst_indentstr = "" def _rst_section(section_name): """Provide section heading""" return "%s\n%s" % (section_name, '-' * len(section_name)) else: raise ValueError("Unknown convention %s for RST" % __rst_conventions) def _rst(s, snotrst=''): """Produce s only in __rst mode""" if __rst_mode: return s else: return snotrst def _rst_underline(text, markup): """Add and underline RsT string matching the length of the given string. """ return text + '\n' + markup * len(text) def single_or_plural(single, plural, n, include_count=False): """Little helper to spit out single or plural version of a word. """ ni = int(n) msg = "%d " % ni if include_count else "" if ni > 1 or ni == 0: # 1 forest, 2 forests, 0 forests return msg + plural else: return msg + single def handle_docstring(text, polite=True): """Take care of empty and non existing doc strings.""" if text is None or not len(text): if polite: return '' # No documentation found. Sorry!' else: return '' else: # Problem is that first line might often have no offset, so might # need to be ignored from dedent call if not text.startswith(' '): lines = text.split('\n') text2 = '\n'.join(lines[1:]) return lines[0] + "\n" + textwrap.dedent(text2) else: return textwrap.dedent(text) def _indent(text, istr=_rst_indentstr): """Simple indenter """ return '\n'.join(istr + s for s in text.split('\n')) __parameters_str_re = re.compile("[\n^]\s*:?Parameters?:?\s*\n(:?\s*-+\s*\n)?") """regexp to match :Parameter: and :Parameters: stand alone in a line or Parameters ---------- in multiple lines""" def _split_out_parameters(initdoc): """Split documentation into (header, parameters, suffix) Parameters ---------- initdoc : string The documentation string """ # TODO: bind it to the only word in the line p_res = __parameters_str_re.search(initdoc) if p_res is None: return initdoc, "", "" else: # Could have been accomplished also via re.match # where new line is after :Parameters: # parameters header index ph_i = p_res.start() # parameters body index pb_i = p_res.end() # end of parameters try: pe_i = initdoc.index('\n\n', pb_i) except ValueError: pe_i = len(initdoc) result = (initdoc[:ph_i].rstrip('\n '), initdoc[pb_i:pe_i], initdoc[pe_i:]) # XXX a bit of duplication of effort since handle_docstring might # do splitting internally return handle_docstring(result[0], polite=False).strip('\n'), \ textwrap.dedent(result[1]).strip('\n'), \ textwrap.dedent(result[2]).strip('\n') __re_params = re.compile('(?:\n\S.*?)+$') __re_spliter1 = re.compile('(?:\n|\A)(?=\S)') __re_spliter2 = re.compile('[\n:]') def _parse_parameters(paramdoc): """Parse parameters and return list of (name, full_doc_string) It is needed to remove multiple entries for the same parameter like it could be with adding parameters from the parent class It assumes that previously parameters were unwrapped, so their documentation starts at the beginning of the string, like what should it be after _split_out_parameters """ entries = __re_spliter1.split(paramdoc) result = [(__re_spliter2.split(e)[0].strip(), e) for e in entries if e != ''] lgr.debug('parseParameters: Given "%s", we split into %s' % (paramdoc, result)) return result def get_docstring_split(f): """Given a function, break it up into portions Parameters ---------- f : function Returns ------- (initial doc string, params (as list of tuples), suffix string) """ if not hasattr(f, '__doc__') or f.__doc__ in (None, ""): return None, None, None initdoc, params, suffix = _split_out_parameters( f.__doc__) params_list = _parse_parameters(params) return initdoc, params_list, suffix def borrowdoc(cls, methodname=None): """Return a decorator to borrow docstring from another `cls`.`methodname` It should not be used for __init__ methods of classes derived from ClassWithCollections since __doc__'s of those are handled by the AttributeCollector anyways. Common use is to borrow a docstring from the class's method for an adapter function (e.g. sphere_searchlight borrows from Searchlight) Examples -------- To borrow `__repr__` docstring from parent class `Mapper`, do:: @borrowdoc(Mapper) def __repr__(self): ... Parameters ---------- cls Usually a parent class methodname : None or str Name of the method from which to borrow. If None, would use the same name as of the decorated method """ def _borrowdoc(method): """Decorator which assigns to the `method` docstring from another """ if methodname is None: other_method = getattr(cls, method.__name__) else: other_method = getattr(cls, methodname) if hasattr(other_method, '__doc__'): method.__doc__ = other_method.__doc__ return method return _borrowdoc def borrowkwargs(cls=None, methodname=None, exclude=None): """Return a decorator which would borrow docstring for ``**kwargs`` Notes ----- TODO: take care about ``*args`` in a clever way if those are also present Examples -------- In the simplest scenario -- just grab all arguments from parent class:: @borrowkwargs(A) def met1(self, bu, **kwargs): pass Parameters ---------- methodname : None or str Name of the method from which to borrow. If None, would use the same name as of the decorated method exclude : None or list of arguments to exclude If function does not pass all ``**kwargs``, you would need to list those here to be excluded from borrowed docstring """ def _borrowkwargs(method): """Decorator which borrows docstrings for ``**kwargs`` for the `method` """ if cls: if methodname is None: other_method = getattr(cls, method.__name__) else: other_method = getattr(cls, methodname) elif methodname: other_method = methodname # TODO: # method.__doc__ = enhanced_from(other_method.__doc__) mdoc, odoc = method.__doc__, other_method.__doc__ if mdoc is None: mdoc = '' mpreamble, mparams, msuffix = _split_out_parameters(mdoc) opreamble, oparams, osuffix = _split_out_parameters(odoc) mplist = _parse_parameters(mparams) oplist = _parse_parameters(oparams) known_params = set([i[0] for i in mplist]) # !!! has to not rebind exclude variable skip_params = exclude or [] # handle None skip_params = set(['kwargs', '**kwargs'] + skip_params) # combine two and filter out items to skip aplist = [i for i in mplist if not i[0] in skip_params] aplist += [i for i in oplist if not i[0] in skip_params.union(known_params)] docstring = mpreamble if len(aplist): params_ = '\n'.join([i[1].rstrip() for i in aplist]) docstring += "\n\n%s\n" \ % _rst_section('Parameters') + _indent(params_) if msuffix != "": docstring += "\n\n" + msuffix docstring = handle_docstring(docstring) # Finally assign generated doc to the method method.__doc__ = docstring return method return _borrowkwargs # TODO: make limit respect config/environment parameter # TODO: document, what limit even is about ;-) def exc_str(exc=None, limit=None): """Enhanced str for exceptions. Should include original location Parameters ---------- Exception to """ out = str(exc) if limit is None: # TODO: config logging.exceptions.traceback_levels = 1 limit = int(os.environ.get('DATALAD_EXC_STR_TBLIMIT', '1')) try: exctype, value, tb = sys.exc_info() if not exc: exc = value out = str(exc) # verify that it seems to be the exception we were passed #assert(isinstance(exc, exctype)) if exc: assert(exc is value) entries = traceback.extract_tb(tb) if entries: out += " [%s]" % (','.join(['%s:%s:%d' % (os.path.basename(x[0]), x[2], x[1]) for x in entries[-limit:]])) except: # MIH: TypeError? return out # To the best of our abilities finally: # As the bible teaches us: # https://docs.python.org/2/library/sys.html#sys.exc_info del tb return out datalad-0.4.1/datalad/downloaders/000077500000000000000000000000001301115270200170115ustar00rootroot00000000000000datalad-0.4.1/datalad/downloaders/__init__.py000066400000000000000000000013501301115270200211210ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Sub-module to provide access (as to download/query etc) to the remote sites """ __docformat__ = 'restructuredtext' from logging import getLogger lgr = getLogger('datalad.providers') # TODO: we might not need to instantiate it right here # lgr.debug("Initializing data providers credentials interface") # providers = Providers().from_config_files() datalad-0.4.1/datalad/downloaders/base.py000066400000000000000000000527031301115270200203040ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Provide access to stuff (html, data files) via HTTP and HTTPS """ __docformat__ = 'restructuredtext' import msgpack import os import time from abc import ABCMeta, abstractmethod from os.path import exists, join as opj, isdir from six import PY2 from six import binary_type, PY3 from .. import cfg from ..ui import ui from ..utils import auto_repr from ..dochelpers import exc_str from .credentials import CREDENTIAL_TYPES from logging import getLogger lgr = getLogger('datalad.downloaders') # TODO: remove headers, HTTP specific @auto_repr class DownloaderSession(object): """Base class to encapsulate information and possibly a session to download the content The idea is that corresponding downloader provides all necessary information and if necessary some kind of session to facilitate .download method """ def __init__(self, size=None, filename=None, url=None, headers=None): self.size = size self.filename = filename self.headers = headers self.url = url def download(self, f=None, pbar=None, size=None): raise NotImplementedError("must be implemented in subclases") # TODO: get_status ? @auto_repr class BaseDownloader(object): """Base class for the downloaders""" _DEFAULT_AUTHENTICATOR = None _DOWNLOAD_SIZE_TO_VERIFY_AUTH = 10000 __metaclass__ = ABCMeta def __init__(self, credential=None, authenticator=None): """ Parameters ---------- credential: Credential, optional Provides necessary credential fields to be used by authenticator authenticator: Authenticator, optional Authenticator to use for authentication. """ if not authenticator and self._DEFAULT_AUTHENTICATOR: authenticator = self._DEFAULT_AUTHENTICATOR() if authenticator: if not credential: msg = "Both authenticator and credentials must be provided." \ " Got only authenticator %s" % repr(authenticator) if ui.yesno( title=msg, text="Do you want to enter %s credentials to be used?" % authenticator.DEFAULT_CREDENTIAL_TYPE ): credential = CREDENTIAL_TYPES[authenticator.DEFAULT_CREDENTIAL_TYPE]( "session-only-for-%s" % id(authenticator)) credential.enter_new() # TODO: give an option to store those credentials, and generate a basic provider # record? else: raise ValueError(msg) self.credential = credential self.authenticator = authenticator self._cache = None # for fetches, not downloads def access(self, method, url, allow_old_session=True, **kwargs): """Generic decorator to manage access to the URL via some method Parameters ---------- method : callable A callable, usually a method of the same class, which we decorate with access handling, and pass url as the first argument url : string URL to access *args, **kwargs Passed into the method call Returns ------- None or bytes """ # TODO: possibly wrap this logic outside within a decorator, which # would just call the corresponding method authenticator = self.authenticator needs_authentication = authenticator and authenticator.requires_authentication attempt, incomplete_attempt = 0, 0 while True: attempt += 1 if attempt > 20: # are we stuck in a loop somehow? I think logic doesn't allow this atm raise RuntimeError("Got to the %d'th iteration while trying to download %s" % (attempt, url)) try: used_old_session = False access_denied = False used_old_session = self._establish_session(url, allow_old=allow_old_session) if not allow_old_session: assert(not used_old_session) lgr.log(5, "Calling out into %s for %s" % (method, url)) result = method(url, **kwargs) # assume success if no puke etc break except AccessDeniedError as e: lgr.debug("Access was denied: %s", exc_str(e)) access_denied = True except IncompleteDownloadError as e: incomplete_attempt += 1 if incomplete_attempt > 5: # give up raise lgr.debug("Failed to download fully, will try again: %s", exc_str(e)) # TODO: may be fail ealier than after 20 attempts in such a case? except DownloadError: # TODO Handle some known ones, possibly allow for a few retries, otherwise just let it go! raise if access_denied: # moved logic outside of except for clarity if needs_authentication: # so we knew it needs authentication if used_old_session: # Let's try with fresh ones allow_old_session = False continue else: # we did use new cookies, we knew that authentication is needed # but still failed. So possible cases: # 1. authentication credentials changed/were revoked # - allow user to re-enter credentials # 2. authentication mechanisms changed # - we can't do anything here about that # 3. bug in out code which would render authentication/cookie handling # ineffective # - not sure what to do about it if ui.yesno( title="Authentication to access {url} has failed".format(url=url), text="Do you want to enter other credentials in case they were updated?"): self.credential.enter_new() allow_old_session = False continue else: raise DownloadError("Failed to download from %s given available credentials" % url) else: # None or False if needs_authentication is False: # those urls must or should NOT require authentication but we got denied raise DownloadError("Failed to download from %s, which must be available without " "authentication but access was denied" % url) else: assert(needs_authentication is None) # So we didn't know if authentication necessary, and it seems to be necessary, so # Let's ask the user to setup authentication mechanism for this website raise AccessDeniedError( "Access to %s was denied but we don't know about this data provider. " "You would need to configure data provider authentication using TODO " % url) return result @staticmethod def _get_temp_download_filename(filepath): """Given a filepath, return the one to use as temp file during download """ # TODO: might better reside somewhere under .datalad/tmp or .git/datalad/tmp return filepath + ".datalad-download-temp" @abstractmethod def get_downloader_session(self, url): """ Parameters ---------- url : str Returns ------- downloader_into_fp: callable Which takes two parameters: file, pbar target_size: int or None (if unknown) url: str Possibly redirected url url_filename: str or None Filename as decided from the (redirected) url headers : dict or None """ raise NotImplementedError("Must be implemented in the subclass") def _verify_download(self, url, downloaded_size, target_size, file_=None, content=None): """Verify that download finished correctly""" if (self.authenticator and downloaded_size < self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) \ and hasattr(self.authenticator, 'failure_re') \ and self.authenticator.failure_re: assert hasattr(self.authenticator, 'check_for_auth_failure'), \ "%s has failure_re defined but no check_for_auth_failure" \ % self.authenticator if file_: with open(file_) as fp: content = fp.read(self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) else: assert(content is not None) self.authenticator.check_for_auth_failure( content, "Download of the url %s has failed: " % url) if target_size and target_size != downloaded_size: raise (IncompleteDownloadError if target_size > downloaded_size else UnaccountedDownloadError)( "Downloaded size %d differs from originally announced %d" % (downloaded_size, target_size)) def _download(self, url, path=None, overwrite=False, size=None, stats=None): """Download content into a file Parameters ---------- url: str URL to download path: str, optional Path to file where to store the downloaded content. If None, filename deduced from the url and saved in curdir size: int, optional Limit in size to be downloaded Returns ------- None or string Returns downloaded filename """ downloader_session = self.get_downloader_session(url) status = self.get_status_from_headers(downloader_session.headers) target_size = downloader_session.size if size is not None: target_size = min(target_size, size) #### Specific to download if path: if isdir(path): # provided path is a directory under which to save filename = downloader_session.filename filepath = opj(path, filename) else: filepath = path else: filepath = downloader_session.filename existed = exists(filepath) if existed and not overwrite: raise DownloadError("File %s already exists" % filepath) # FETCH CONTENT # TODO: pbar = ui.get_progressbar(size=response.headers['size']) try: temp_filepath = self._get_temp_download_filename(filepath) if exists(temp_filepath): # eventually we might want to continue the download lgr.warning( "Temporary file %s from the previous download was found. " "It will be overriden" % temp_filepath) # TODO. also logic below would clean it up atm with open(temp_filepath, 'wb') as fp: # TODO: url might be a bit too long for the beast. # Consider to improve to make it animated as well, or shorten here pbar = ui.get_progressbar(label=url, fill_text=filepath, maxval=target_size) t0 = time.time() downloader_session.download(fp, pbar, size=size) downloaded_time = time.time() - t0 pbar.finish() downloaded_size = os.stat(temp_filepath).st_size # (headers.get('Content-type', "") and headers.get('Content-Type')).startswith('text/html') # and self.authenticator.html_form_failure_re: # TODO: use information in authenticator self._verify_download(url, downloaded_size, target_size, temp_filepath) # adjust atime/mtime according to headers/status if status.mtime: lgr.log(5, "Setting mtime for %s to be %s", temp_filepath, status.mtime) os.utime(temp_filepath, (time.time(), status.mtime)) # place successfully downloaded over the filepath os.rename(temp_filepath, filepath) if stats: stats.downloaded += 1 stats.overwritten += int(existed) stats.downloaded_size += downloaded_size stats.downloaded_time += downloaded_time except (AccessDeniedError, IncompleteDownloadError) as e: raise except Exception as e: e_str = exc_str(e, limit=5) lgr.error("Failed to download {url} into {filepath}: {e_str}".format( **locals() )) raise DownloadError(exc_str(e)) # for now finally: if exists(temp_filepath): # clean up lgr.debug("Removing a temporary download %s", temp_filepath) os.unlink(temp_filepath) return filepath def download(self, url, path=None, **kwargs): """Fetch content as pointed by the URL optionally into a file Parameters ---------- url : string URL to access path : string, optional Filename or existing directory to store downloaded content under. If not provided -- deduced from the url Returns ------- string file path """ # TODO: may be move all the path dealing logic here # but then it might require sending request anyways for Content-Disposition # so probably nah lgr.info("Downloading %r into %r", url, path) return self.access(self._download, url, path=path, **kwargs) @property def cache(self): if self._cache is None: # TODO: move this all logic outside into a dedicated caching beast lgr.info("Initializing cache for fetches") if PY2: import anydbm as dbm else: import dbm # Initiate cache. # Very rudimentary caching for now, might fail many ways cache_dir = cfg.obtain('datalad.locations.cache') if not exists(cache_dir): os.makedirs(cache_dir) cache_path = opj(cache_dir, 'crawl_cache.dbm') self._cache = dbm.open(cache_path, 'c') import atexit atexit.register(self._cache.close) return self._cache def _fetch(self, url, cache=None, size=None, allow_redirects=True): """Fetch content from a url into a file. Very similar to _download but lacks any "file" management and decodes content Parameters ---------- url: str URL to download cache: bool, optional If None, config is consulted either results should be cached. Cache is operating based on url, so no verification of any kind is carried out Returns ------- bytes, dict content, headers """ lgr.log(3, "_fetch(%r, cache=%r, size=%r, allow_redirects=%r)", url, cache, size, allow_redirects) if cache is None: cache = cfg.obtain('datalad.crawl.cache', default=False) if cache: cache_key = msgpack.dumps(url) lgr.debug("Loading content for url %s from cache", url) res = self.cache.get(cache_key) if res is not None: try: return msgpack.loads(res, encoding='utf-8') except Exception as exc: lgr.warning("Failed to unpack loaded from cache for %s: %s", url, exc_str(exc)) downloader_session = self.get_downloader_session(url, allow_redirects=allow_redirects) target_size = downloader_session.size if size is not None: if size == 0: # no download of the content was requested -- just return headers and be done return None, downloader_session.headers target_size = min(size, target_size) # FETCH CONTENT try: # Consider to improve to make it animated as well, or shorten here #pbar = ui.get_progressbar(label=url, fill_text=filepath, maxval=target_size) content = downloader_session.download(size=size) #pbar.finish() downloaded_size = len(content) # now that we know size based on encoded content, let's decode into string type if PY3 and isinstance(content, binary_type): content = content.decode() # downloaded_size = os.stat(temp_filepath).st_size self._verify_download(url, downloaded_size, target_size, None, content=content) except (AccessDeniedError, IncompleteDownloadError) as e: raise except Exception as e: e_str = exc_str(e, limit=5) lgr.error("Failed to fetch {url}: {e_str}".format(**locals())) raise DownloadError(exc_str(e, limit=8)) # for now if cache: # apparently requests' CaseInsensitiveDict is not serialazable # TODO: may be we should reuse that type everywhere, to avoid # out own handling for case-handling self.cache[cache_key] = msgpack.dumps((content, dict(downloader_session.headers))) return content, downloader_session.headers def fetch(self, url, **kwargs): """Fetch and return content (not decoded) as pointed by the URL Parameters ---------- url : string URL to access Returns ------- bytes content """ lgr.info("Fetching %r", url) # Do not return headers, just content out = self.access(self._fetch, url, **kwargs) # import pdb; pdb.set_trace() return out[0] def get_status(self, url, old_status=None, **kwargs): """Return status of the url as a dict, None if N/A Parameters ---------- url : string URL to access old_status : FileStatus, optional Previous status record. If provided, might serve as a shortcut to assess if status has changed, and if not -- return the same record Returns ------- dict dict-like beast depicting the status of the URL if accessible. Returned value should be sufficient to tell if the URL content has changed by comparing to previously obtained value. If URL is not reachable, None would be returned """ return self.access(self._get_status, url, old_status=old_status, **kwargs) # TODO: borrow from itself... ? # @borrowkwargs(BaseDownloader, 'get_status') def _get_status(self, url, old_status=None): # the tricky part is only to make sure that we are getting the target URL # and not some page saying to login, that is why we need to fetch some content # in those cases, and not just check the headers download_size = self._DOWNLOAD_SIZE_TO_VERIFY_AUTH \ if self.authenticator \ and hasattr(self.authenticator, 'failure_re') \ and self.authenticator.failure_re \ else 0 _, headers = self._fetch(url, cache=False, size=download_size) # extract from headers information to depict the status of the url status = self.get_status_from_headers(headers) if old_status is not None: raise NotImplementedError("Do not know yet how to deal with old_status. TODO") return status @classmethod @abstractmethod def get_status_from_headers(cls, headers): raise NotImplementedError("Implement in the subclass: %s" % cls) def get_target_url(self, url): """Return url after possible redirections Parameters ---------- url : string URL to access Returns ------- str """ return self.access(self._get_target_url, url) def _get_target_url(self, url): return self.get_downloader_session(url).url # Exceptions. might migrate elsewhere # MIH: Completely non-obvious why this is here from ..support.exceptions import * # # Authenticators XXX might go into authenticators.py # class Authenticator(object): """Abstract common class for different types of authentication Derived classes should get parameterized with options from the config files from "provider:" sections """ requires_authentication = True # TODO: figure out interface DEFAULT_CREDENTIAL_TYPE = 'user_password' def authenticate(self, *args, **kwargs): """Derived classes will provide specific implementation """ if self.requires_authentication: raise NotImplementedError("Authentication for %s not yet implemented" % self.__class__) class NotImplementedAuthenticator(Authenticator): pass class NoneAuthenticator(Authenticator): """Whenever no authentication is necessary and that is stated explicitly""" requires_authentication = False pass datalad-0.4.1/datalad/downloaders/configs/000077500000000000000000000000001301115270200204415ustar00rootroot00000000000000datalad-0.4.1/datalad/downloaders/configs/crawdad.cfg000066400000000000000000000003101301115270200225210ustar00rootroot00000000000000[provider:crawdad] url_re = http://(?P\s\.)?crawdad.org/.* credential = crawdad authentication_type = http_auth [credential:crawdad] url = http://crawdad.org/joinup.html type = user_password datalad-0.4.1/datalad/downloaders/configs/crcns.cfg000066400000000000000000000034111301115270200222310ustar00rootroot00000000000000[provider:crcns] url_re = https?://crcns.org/.* ftp://crcns.org/.* # certificates = ??? ; (uses https) credential = crcns # for HTTP: could be html_form (see below), user_password, authentication_type = html_form html_form_url = https://crcns.org/login_form html_form_tagid = login_form html_form_fields = __ac_name={user} __ac_password={password} submit=Log in form.submitted=1 js_enabled=0 cookies_enabled= html_form_failure_re = (Login failed|Please log in) html_form_success_re = You are now logged in # TODO: think about separating authenticator into a separate type of section, # similar to credential. Would results in closer to code arrangement # BENEFITS: we might then specify multiple pairs, e.g. for : # THINK: what if I have few different credentials for the same portal, which # I use for different subsets of data (collaborations etc). # With separateing authenticator out we could easier reuse pairs # of credential_authenticator pairs! [provider:crcns-nersc] url_re = https://portal.nersc.gov/project/crcns/download/.* credential = crcns authentication_type = html_form html_form_url = https://portal.nersc.gov/project/crcns/download/index.php # Just the first/only form # html_form_tagid = login_form html_form_fields = username={user} password={password} submit=Login fn=alm-1/filelist.txt html_form_failure_re = (
|Error: Invalid user name or password) #html_form_success_re = Logged in as [credential:crcns] # url where to request credentials url = https://crcns.org/request-account/ # (user_password|aws-s3(access_key,secret_key for S3) type = user_password datalad-0.4.1/datalad/downloaders/configs/hcp.cfg000066400000000000000000000035221301115270200216760ustar00rootroot00000000000000# lots of js and some redirects, not yet clear what's needed to work properly # the hcp-http below works! [provider:hcp-web] url_re = __https://db.humanconnectome.org/.*__ credential = hcp-db authentication_type = html_form html_form_url = https://db.humanconnectome.org/app/template/Login.vm html_form_tagid = login_form html_form_fields = login_method=LDAP j_username={user} j_password={password} XNAT_CSRF= html_form_failure_re = HTTP Status 404 html_form_success_re = Public Connectome Data [provider:hcp-http] url_re = https://db.humanconnectome.org/data/.* credential = hcp-db authentication_type = http_auth http_auth_url = https://db.humanconnectome.org/data/JSESSION [provider:hcp-xnat] url_re = xnat://somewhereIdon'tknowyet credential = hcp-db authentication_type = xnat [provider:hcp-s3] url_re = s3://hcp-openaccess.* credential = hcp-s3 authentication_type = aws-s3 [provider:balsa] url_re = https?://balsa.wustl.edu/.* # or could be a different custom one! so we might allow for a choice # https://balsa.wustl.edu/register/register credential = hcp-db authentication_type = html_form # https://balsa.wustl.edu/login/auth html_form_url = https://balsa.wustl.edu/j_spring_security_check html_form_tagid = loginForm html_form_fields = j_username={user} j_password={password} _spring_security_remember_me= # seems to lead us to the same page with the same content... # but with a different Location which we don't handle yet to forward us further... TODO! #html_form_failure_re = Login to BALSA html_form_success_re = Logged in as [credential:hcp-db] url = https://db.humanconnectome.org/app/template/Login.vm type = user_password [credential:hcp-s3] url = https://wiki.humanconnectome.org/display/PublicData/Connecting+to+Connectome+Data+via+AWS type = aws-s3 datalad-0.4.1/datalad/downloaders/configs/kaggle.cfg000066400000000000000000000011221301115270200223500ustar00rootroot00000000000000[provider:kaggle] url_re = https?://(.*\.|)kaggle.com/.* credential = kaggle authentication_type = html_form html_form_url = https://www.kaggle.com/account/login html_form_tagid = login-account html_form_fields = UserName={user} Password={password} get-started=Login standalonesigninjs=False html_form_session_cookies = .ASPXAUTH #html_form_fields_pass = __RequestVerificationToken html_form_failure_re = The username or password provided is incorrect [credential:kaggle] url = https://www.kaggle.com/account/login type = user_password datalad-0.4.1/datalad/downloaders/configs/nda.cfg000066400000000000000000000015631301115270200216710ustar00rootroot00000000000000[provider:NDA] # There are 5 used buckets ATM with url_re = s3://(ndar_c|NDAR_C)entral(_\d+)?($|/.*) credential = NDA authentication_type = nda-s3 [credential:NDA] # url where to request credentials url = https://ndar.nih.gov/access.html type = nda-s3 # # Configuration used by the crawler to fetch "interesting" selection of stuff # from miNDAR which has its own authentication scheme, and actually we might need # it per each username # So we might somehow support it? i.e. name here should not be necessarily # THE name, and multiple actual user_password entries should be created and # explicitly chosen (e.g. based on username) # [provider:miNDAR] url_re = mindarvpc\..*.rds\.amazonaws\.com credential = miNDAR # ATM this is just to load credential... RFing is needed authentication_type = none [credential:miNDAR] url = https://ndar.nih.gov/access.html type = user_password datalad-0.4.1/datalad/downloaders/configs/nitrc.cfg000066400000000000000000000031411301115270200222400ustar00rootroot00000000000000# File release system -- the same credentials, and some datasets do not need # any credentials but some do [provider:nitrc-frs] # TODO: we can't list all links which require authentication so we must # allow for some to be attempted to be downloaded WITHOUT authentication # and then only if fail -- should attempt to authenticate. For now, to # move progress forward, just listed a few of those creatures since # they aren't usable yet since folks need to agree over and over again # (upon each login) to download files -- so we need to figure out how # to accomplish the mission here url_re = https?://fcon_1000\.projects\.nitrc\.org/indi/adhd200/index.html https?://www\.nitrc\.org/frs/downloadlink\.php/(7058|3075|3479|9108) credential = nitrc authentication_type = html_form html_form_url = https://www.nitrc.org/account/login.php html_form_fields = form_loginname={user} form_pw={password} myloginButton=Log in with SSL login=Log in with SSL html_form_failure_re = Invalid Password Or User Name #authentication_type = http_auth #http_auth_url = https://www.nitrc.org/account/login.php #http_auth_failure_re = Invalid Password Or User Name # requires JSESSIONID= cookie for the session... ??? [provider:nitrc-ir] url_re = https?://www.nitrc.org/ir/.* credential = nitrc authentication_type = http_auth http_auth_url = https://www.nitrc.org/ir/ # if login pointer present -- we failed to login http_auth_failure_re = (Member login|Please use the Login Name and Password) [credential:nitrc] url = https://www.nitrc.org/account/register.php type = user_password datalad-0.4.1/datalad/downloaders/configs/openfmri.cfg000066400000000000000000000005041301115270200227400ustar00rootroot00000000000000[provider:openfmri] url_re = https?://openfmri.org/.* # does not require any, good folks authentication_type = none # TODO: make it proper catch-all handling # see https://github.com/datalad/datalad/issues/322 [provider:openfmri-s3] url_re = s3://openfmri($|/.*) credential = datalad-test-s3 authentication_type = aws-s3 datalad-0.4.1/datalad/downloaders/configs/providers.cfg000066400000000000000000000004411301115270200231360ustar00rootroot00000000000000# # Sample configuration file to define a provider and associated with it credentials # [provider:datalad-test-s3] url_re = s3://datalad-test.* credential = datalad-test-s3 authentication_type = aws-s3 # # Credentials # [credential:datalad-test-s3] url = team@datalad.org type = aws-s3 datalad-0.4.1/datalad/downloaders/credentials.py000066400000000000000000000240141301115270200216610ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Interface information about credentials Provides minimalistic interface to deal (query, request, store) with most common types of credentials. To be used by Authenticators """ __dev_doc__ = """ Possibly useful in the future 3rd part developments https://github.com/omab/python-social-auth social authentication/registration mechanism with support for several frameworks and auth providers """ import time import calendar from collections import OrderedDict from ..dochelpers import exc_str from ..support.keyring_ import keyring as keyring_ from ..ui import ui from ..utils import auto_repr from ..support.network import iso8601_to_epoch from logging import getLogger lgr = getLogger('datalad.downloaders.credentials') @auto_repr class Credential(object): """Base class for different types of credentials """ # Should be defined in a subclass as an OrderedDict of fields # name -> dict(attributes) _FIELDS = None _KNOWN_ATTRS = { 'hidden', # UI should not display the value 'optional', # Not mandatory thus not requested if not set } def __init__(self, name, url=None, keyring=None): """ Parameters ---------- name : str Name of the credential, as it would be identified by in the centralized storage of credentials url : str, optional URL string to point users to where to seek obtaining the credentials keyring : a keyring An object providing (g|s)et_password. If None, keyring module is used as is """ self.name = name self.url = url self._keyring = keyring or keyring_ self._prepare() def _prepare(self): """Additional house-keeping possibly to be performed in subclasses Created to avoid all the passing args/kwargs of __init__ all the time """ # Basic checks for f, fattrs in self._FIELDS.items(): unknown_attrs = set(fattrs).difference(self._KNOWN_ATTRS) if unknown_attrs: raise ValueError("Unknown attributes %s. Known are: %s" % (unknown_attrs, self._KNOWN_ATTRS)) def _is_field_optional(self, f): return self._FIELDS[f].get('optional', False) def _is_field_hidden(self, f): return self._FIELDS[f].get('hidden', False) @property def is_known(self): """Return True if values for all fields of the credential are known""" try: return all( self._is_field_optional(f) or self._keyring.get(self.name, f) is not None for f in self._FIELDS) except Exception as exc: lgr.warning("Failed to query keyring: %s" % exc_str(exc)) return False def _ask_field_value(self, f): return ui.question( f, title="You need to authenticate with %r credentials." % self.name + " %s provides information on how to gain access" % self.url if self.url else '', hidden=self._is_field_hidden(f)) def _ask_and_set(self, f): v = self._ask_field_value(f) self.set(**{f: v}) return v def enter_new(self): """Enter new values for the credential fields""" # Use ui., request credential fields corresponding to the type for f in self._FIELDS: if not self._is_field_optional(f): self._ask_and_set(f) def __call__(self): """Obtain credentials from a keyring and if any is not known -- ask""" name = self.name fields = {} for f in self._FIELDS: v = self._keyring.get(name, f) if not self._is_field_optional(f): while v is None: # was not known v = self._ask_and_set(f) fields[f] = v elif v is not None: fields[f] = v return fields def set(self, **kwargs): """Set field(s) of the credential""" for f, v in kwargs.items(): if f not in self._FIELDS: raise ValueError("Unknown field %s. Known are: %s" % (f, self._FIELDS.keys())) self._keyring.set(self.name, f, v) def get(self, f, default=None): """Get a field of the credential""" if f not in self._FIELDS: raise ValueError("Unknown field %s. Known are: %s" % (f, self._FIELDS.keys())) try: return self._keyring.get(self.name, f) except: # MIH: what could even happen? _keyring not a dict? return default def delete(self): """Deletes credential values from the keyring""" for f in self._FIELDS: self._keyring.delete(self.name, f) class UserPassword(Credential): """Simple type of a credential which consists of user/password pair""" _FIELDS = OrderedDict([('user', {}), ('password', {'hidden': True})]) is_expired = False # no expiration provisioned class AWS_S3(Credential): """Credential for AWS S3 service""" _FIELDS = OrderedDict([('key_id', {}), ('secret_id', {'hidden': True}), ('session', {'optional': True}), ('expiration', {'optional': True}), ]) @property def is_expired(self): exp = self.get('expiration', None) if not exp: return True exp_epoch = iso8601_to_epoch(exp) expire_in = (exp_epoch - calendar.timegm(time.localtime())) / 3600. lgr.debug( ("Credential %s has expired %.2fh ago" if expire_in <= 0 else "Credential %s will expire in %.2fh") % (self.name, expire_in)) return expire_in <= 0 @auto_repr class CompositeCredential(Credential): """Credential which represent a sequence of Credentials where front one is exposed to user """ # To be defined in sub-classes _CREDENTIAL_CLASSES = None _CREDENTIAL_ADAPTERS = None def _prepare(self): assert len(self._CREDENTIAL_CLASSES) > 1, "makes sense only if there is > 1 credential" assert len(self._CREDENTIAL_CLASSES) == len(self._CREDENTIAL_ADAPTERS) + 1, \ "there should be 1 less of adapter than _CREDENTIAL_CLASSES" for C in self._CREDENTIAL_CLASSES: assert issubclass(C, Credential), "%s must be a subclass of Credential" % C # First credential should bear the name and url credentials = [self._CREDENTIAL_CLASSES[0](self.name, url=self.url, keyring=self._keyring)] # and we just reuse its _FIELDS for _ask_field_value etc self._FIELDS = credentials[0]._FIELDS # the rest with index suffix, but storing themselves in the same keyring for iC, C in enumerate(self._CREDENTIAL_CLASSES[1:]): credentials.append( C(name="%s:%d" % (self.name, iC + 1), url=None, keyring=self._keyring) ) self._credentials = credentials super(CompositeCredential, self)._prepare() # Here it becomes tricky, since theoretically it is the "tail" # ones which might expire etc, so we wouldn't exactly know what # new credentials outside process wanted -- it would be silly to ask # right away the "entry" credentials if it is just the expiration of the # tail credentials def enter_new(self): # should invalidate/remove all tail credentials to avoid failing attempts to login self._credentials[0].enter_new() for c in self._credentials[1:]: c.delete() def __call__(self): """Obtain credentials from a keyring and if any is not known -- ask""" # Start from the tail until we have credentials set idx = len(self._credentials) - 1 for c in self._credentials[::-1]: if c.is_known and not c.is_expired: break idx -= 1 if idx < 0: # none was known, all the same -- start with the first one idx = 0 # TODO: consider moving logic of traversal into authenticator since it is # the one spitting out authentication error etc # Theoretically we could just reuse 'fields' from adapter in the next step # but let's do full circle, so that if any "normalization" is done by # Credentials we take that into account for c, adapter, next_c in zip( self._credentials[idx:], self._CREDENTIAL_ADAPTERS[idx:], self._credentials[idx + 1:]): fields = c() next_fields = adapter(**fields) next_c.set(**next_fields) return self._credentials[-1]() def _nda_adapter(user=None, password=None): from datalad.support.third.nda_aws_token_generator import NDATokenGenerator gen = NDATokenGenerator() token = gen.generate_token(user, password) # There are also session and expiration we ignore... TODO anything about it?!!! # we could create a derived AWS_S3 which would also store session and expiration # and then may be Composite could use those???? return dict(key_id=token.access_key, secret_id=token.secret_key, session=token.session, expiration=token.expiration) class NDA_S3(CompositeCredential): """Credential to access NDA AWS So for NDA we need a credential which is a composite credential. User provides UserPassword and then some adapter generates AWS_S3 out of it """ _CREDENTIAL_CLASSES = (UserPassword, AWS_S3) _CREDENTIAL_ADAPTERS = (_nda_adapter,) CREDENTIAL_TYPES = { 'user_password': UserPassword, 'aws-s3': AWS_S3, 'nda-s3': NDA_S3, } datalad-0.4.1/datalad/downloaders/http.py000066400000000000000000000466111301115270200203520ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Provide access to stuff (html, data files) via HTTP and HTTPS """ import re import requests import requests.auth import io from six import BytesIO from ..utils import assure_list_from_str, assure_dict_from_str from ..dochelpers import borrowkwargs from ..ui import ui from ..utils import auto_repr from ..dochelpers import exc_str from ..support.network import get_url_filename from ..support.network import get_response_disposition_filename from ..support.network import rfc2822_to_epoch from ..support.cookies import cookies_db from ..support.status import FileStatus from .base import Authenticator from .base import BaseDownloader, DownloaderSession from .base import DownloadError, AccessDeniedError, AccessFailedError, UnhandledRedirectError from logging import getLogger from ..log import LoggerHelper lgr = getLogger('datalad.http') try: import requests_ftp requests_ftp.monkeypatch_session() except ImportError as e: lgr.debug("Failed to import requests_ftp, thus no ftp support: %s" % exc_str(e)) pass if lgr.getEffectiveLevel() <= 1: # Let's also enable requests etc debugging # These two lines enable debugging at httplib level (requests->urllib3->http.client) # You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA. # The only thing missing will be the response.body which is not logged. from six.moves import http_client # TODO: nohow wrapped with logging, plain prints (heh heh), so formatting will not be consistent http_client.HTTPConnection.debuglevel = 1 # for requests we can define logging properly requests_log = LoggerHelper(logtarget="requests.packages.urllib3").get_initialized_logger() requests_log.setLevel(lgr.getEffectiveLevel()) requests_log.propagate = True __docformat__ = 'restructuredtext' def check_response_status(response, err_prefix="", session=None): """Check if response's status_code signals problem with authentication etc ATM succeeds only if response code was 200 """ if not err_prefix: err_prefix = "Access to %s has failed: " % response.url # 401 would be for digest authentication mechanism, or if we first ask which mechanisms are # supported.... must be linked into the logic if we decide to automagically detect which # mechanism or to give more sensible error message err_msg = err_prefix + "status code %d" % response.status_code if response.status_code in {404}: # It could have been that form_url is wrong, so let's just say that # TODO: actually may be that is where we could use tagid and actually determine the form submission url raise DownloadError(err_prefix + "not found") elif 400 <= response.status_code < 500: raise AccessDeniedError(err_msg) elif response.status_code in {200}: pass elif response.status_code in {301, 302, 307}: # TODO: apparently tests do not excercise this one yet if session is None: raise AccessFailedError(err_msg + " no session was provided") redirs = list(session.resolve_redirects(response, response.request)) if len(redirs) > 1: lgr.warning("Multiple redirects aren't supported yet. Taking first") elif len(redirs) == 0: raise AccessFailedError("No redirects were resolved") raise UnhandledRedirectError(err_msg, url=redirs[0].url) else: raise AccessFailedError(err_msg) @auto_repr class HTTPBaseAuthenticator(Authenticator): """Base class for html_form and http_auth authenticators """ def __init__(self, url=None, failure_re=None, success_re=None, session_cookies=None, **kwargs): """ Parameters ---------- url : str, optional URL where to find the form/login to authenticate. If not provided, an original query url which will be provided to the __call__ of the authenticator will be used failure_re : str or list of str, optional success_re : str or list of str, optional Regular expressions to determine either login has failed or succeeded. TODO: we might condition when it gets ran session_cookies : str or list of str, optional Session cookies to store (besides auth response cookies) """ super(HTTPBaseAuthenticator, self).__init__(**kwargs) self.url = url self.failure_re = assure_list_from_str(failure_re) self.success_re = assure_list_from_str(success_re) self.session_cookies = assure_list_from_str(session_cookies) def authenticate(self, url, credential, session, update=False): # we should use specified URL for this authentication first lgr.info("http session: Authenticating into session for %s", url) post_url = self.url if self.url else url credentials = credential() # TODO: With digestauth there is no POST strictly necessary. # The whole thing relies on server first spitting out 401 # and client GETing again with 'Authentication:' header # So we need custom handling for those, while keeping track not # of cookies per se, but of 'Authentication:' header which is # to be used in subsequent GETs response = self._post_credential(credentials, post_url, session) err_prefix = "Authentication to %s failed: " % post_url try: check_response_status(response, err_prefix, session=session) except DownloadError: # It might have happened that the return code was 'incorrect' # and we did get some feedback, which we could analyze to # figure out actual problem. E.g. in case of nersc of crcns # it returns 404 (not found) with text in the html if response is not None and response.text: self.check_for_auth_failure(response.text, err_prefix) raise response_text = response.text self.check_for_auth_failure(response_text, err_prefix) if self.success_re: # the one which must be used to verify success # verify that we actually logged in for success_re in self.success_re: if not re.search(success_re, response_text): raise AccessDeniedError( err_prefix + " returned output did not match 'success' regular expression %s" % success_re ) cookies_dict = {} if response.cookies: cookies_dict = requests.utils.dict_from_cookiejar(response.cookies) if self.session_cookies: # any session cookies to store cookies_dict.update({k: session.cookies[k] for k in self.session_cookies}) if cookies_dict: if (url in cookies_db) and update: cookies_db[url].update(cookies_dict) else: cookies_db[url] = cookies_dict # assign cookies for this session for c, v in cookies_dict.items(): if c not in session.cookies or session.cookies[c] != v: session.cookies[c] = v # .update(cookies_dict) return response def _post_credential(self, credentials, post_url, session): raise NotImplementedError("Must be implemented in subclass") def check_for_auth_failure(self, content, err_prefix=""): if self.failure_re: # verify that we actually logged in for failure_re in self.failure_re: if re.search(failure_re, content): raise AccessDeniedError( err_prefix + "returned output which matches regular expression %s" % failure_re ) @auto_repr class HTMLFormAuthenticator(HTTPBaseAuthenticator): """Authenticate by opening a session via POSTing to HTML form """ def __init__(self, fields, tagid=None, **kwargs): """ Example specification in the .ini config file [provider:crcns] ... credential = crcns ; is not given to authenticator as is authentication_type = html_form # TODO: may be rename into post_url html_form_url = https://crcns.org/login_form # probably not needed actually since form_url # html_form_tagid = login_form html_form_fields = __ac_name={user} __ac_password={password} submit=Log in form.submitted=1 js_enabled=0 cookies_enabled= html_form_failure_re = (Login failed|Please log in) html_form_success_re = You are now logged in Parameters ---------- fields : str or dict String or a dictionary, which will be used (along with credential) information to feed into the form tagid : str, optional id of the HTML in the document to use. If None, and page contains a single form, that one will be used. If multiple forms -- error will be raise **kwargs : dict, optional Passed to super class HTTPBaseAuthenticator """ super(HTMLFormAuthenticator, self).__init__(**kwargs) self.fields = assure_dict_from_str(fields) self.tagid = tagid def _post_credential(self, credentials, post_url, session): post_fields = { k: v.format(**credentials) for k, v in self.fields.items() } response = session.post(post_url, data=post_fields) lgr.debug("Posted to %s fields %s, got response %s with headers %s", post_url, list(post_fields.keys()), response, list(response.headers.keys())) return response @auto_repr class HTTPRequestsAuthenticator(HTTPBaseAuthenticator): """Base class for various authenticators using requests pre-crafted ones """ REQUESTS_AUTHENTICATOR = None def __init__(self, **kwargs): # so we have __init__ solely for a custom docstring super(HTTPRequestsAuthenticator, self).__init__(**kwargs) def _post_credential(self, credentials, post_url, session): response = session.post(post_url, data={}, auth=self.REQUESTS_AUTHENTICATOR(credentials['user'], credentials['password'])) return response @auto_repr class HTTPBasicAuthAuthenticator(HTTPRequestsAuthenticator): """Authenticate via basic HTTP authentication Example specification in the .ini config file [provider:hcp-db] ... credential = hcp-db authentication_type = http_auth http_auth_url = .... TODO Parameters ---------- **kwargs : dict, optional Passed to super class HTTPBaseAuthenticator """ REQUESTS_AUTHENTICATOR = requests.auth.HTTPBasicAuth @auto_repr class HTTPDigestAuthAuthenticator(HTTPRequestsAuthenticator): """Authenticate via HTTP digest authentication """ REQUESTS_AUTHENTICATOR = requests.auth.HTTPDigestAuth def _post_credential(self, *args, **kwargs): raise NotImplementedError("not yet functioning, see https://github.com/kennethreitz/requests/issues/2934") @auto_repr class HTTPDownloaderSession(DownloaderSession): def __init__(self, size=None, filename=None, url=None, headers=None, response=None, chunk_size=1024 ** 2): super(HTTPDownloaderSession, self).__init__( size=size, filename=filename, url=url, headers=headers, ) self.chunk_size = chunk_size self.response = response def download(self, f=None, pbar=None, size=None): response = self.response # content_gzipped = 'gzip' in response.headers.get('content-encoding', '').split(',') # if content_gzipped: # raise NotImplemented("We do not support (yet) gzipped content") # # see https://rationalpie.wordpress.com/2010/06/02/python-streaming-gzip-decompression/ # # for ways to implement in python 2 and 3.2's gzip is working better with streams total = 0 return_content = f is None if f is None: # no file to download to # TODO: actually strange since it should have been decoded then... f = BytesIO() # must use .raw to be able avoiding decoding/decompression while downloading # to a file chunk_size_ = min(self.chunk_size, size) if size is not None else self.chunk_size # XXX With requests_ftp BytesIO is provided as response.raw for ftp urls, # which has no .stream, so let's do ducktyping and provide our custom stream # via BufferedReader for such cases, while maintaining the rest of code # intact. TODO: figure it all out, since doesn't scale for any sizeable download if not hasattr(response.raw, 'stream'): def _stream(): buf = io.BufferedReader(response.raw) v = True while v: v = buf.read(chunk_size_) yield v stream = _stream() else: # XXX TODO -- it must be just a dirty workaround # As we discovered with downloads from NITRC all headers come with # Content-Encoding: gzip which leads requests to decode them. But the point # is that ftp links (yoh doesn't think) are gzip compressed for the transfer decode_content = not response.url.startswith('ftp://') stream = response.raw.stream(chunk_size_, decode_content=decode_content) for chunk in stream: if chunk: # filter out keep-alive new chunks chunk_len = len(chunk) if size is not None and total + chunk_len > size: # trim the download to match target size chunk = chunk[:size - total] chunk_len = len(chunk) total += chunk_len f.write(chunk) try: # TODO: pbar is not robust ATM against > 100% performance ;) if pbar: pbar.update(total) except Exception as e: lgr.warning("Failed to update progressbar: %s" % exc_str(e)) # TEMP # see https://github.com/niltonvolpato/python-progressbar/pull/44 ui.out.flush() if size is not None and total >= size: break # we have done as much as we were asked if return_content: out = f.getvalue() return out @auto_repr class HTTPDownloader(BaseDownloader): """A stateful downloader to maintain a session to the website """ @borrowkwargs(BaseDownloader) def __init__(self, **kwargs): super(HTTPDownloader, self).__init__(**kwargs) self._session = None def _establish_session(self, url, allow_old=True): """ Parameters ---------- allow_old: bool, optional If a Downloader allows for persistent sessions by some means -- flag instructs either to use previous session, or establish a new one Returns ------- bool To state if old instance of a session/authentication was used """ if allow_old: if self._session: lgr.debug("http session: Reusing previous") return True # we used old elif url in cookies_db: cookie_dict = cookies_db[url] lgr.debug("http session: Creating new with old cookies %s", list(cookie_dict.keys())) self._session = requests.Session() # not sure what happens if cookie is expired (need check to that or exception will prolly get thrown) # TODO dict_to_cookiejar doesn't preserve all fields when reversed self._session.cookies = requests.utils.cookiejar_from_dict(cookie_dict) # TODO cookie could be expired w/ something like (but docs say it should be expired automatically): # http://docs.python-requests.org/en/latest/api/#requests.cookies.RequestsCookieJar.clear_expired_cookies # self._session.cookies.clear_expired_cookies() return True lgr.debug("http session: Creating brand new session") self._session = requests.Session() if self.authenticator: self.authenticator.authenticate(url, self.credential, self._session) return False def get_downloader_session(self, url, allow_redirects=True, use_redirected_url=True): # TODO: possibly make chunk size adaptive # TODO: make it not this ugly -- but at the moment we are testing end-file size # while can't know for sure if content was gunziped and either it all went ok. # So safer option -- just request to not have it gzipped headers = {'Accept-Encoding': ''} response = self._session.get(url, stream=True, allow_redirects=allow_redirects, headers=headers) check_response_status(response, session=self._session) headers = response.headers lgr.debug("Establishing session for url %s, response headers: %s", url, headers) target_size = int(headers.get('Content-Length', '0').strip()) or None if use_redirected_url and response.url and response.url != url: lgr.debug("URL %s was redirected to %s and thus the later will be used" % (url, response.url)) url = response.url # Consult about filename. Since we already have headers, # should not result in an additional request url_filename = get_url_filename(url, headers=headers) headers['Url-Filename'] = url_filename return HTTPDownloaderSession( size=target_size, url=response.url, filename=url_filename, headers=headers, response=response ) @classmethod def get_status_from_headers(cls, headers): """Given HTTP headers, return 'status' record to assess later if link content was changed """ # used for quick checks for HTTP or S3? # TODO: So we will base all statuses on this set? e.g. for Last-Modified if to be # mapping from field to its type converter HTTP_HEADERS_TO_STATUS = { 'Content-Length': int, 'Content-Disposition': str, 'Last-Modified': rfc2822_to_epoch, 'Url-Filename': str, } # Allow for webserver to return them in other casing HTTP_HEADERS_TO_STATUS_lower = {s.lower(): (s, t) for s, t in HTTP_HEADERS_TO_STATUS.items()} status = {} if headers: for header_key in headers: try: k, t = HTTP_HEADERS_TO_STATUS_lower[header_key.lower()] except KeyError: continue status[k] = t(headers[header_key]) # convert to FileStatus return FileStatus( size=status.get('Content-Length'), mtime=status.get('Last-Modified'), filename=get_response_disposition_filename( status.get('Content-Disposition')) or status.get('Url-Filename') ) datalad-0.4.1/datalad/downloaders/providers.py000066400000000000000000000340321301115270200214020ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Data providers - bind downloaders and credentials together """ from glob import glob from logging import getLogger from six import iteritems import re from os.path import dirname, abspath, join as pathjoin from six.moves.urllib.parse import urlparse from .base import NoneAuthenticator, NotImplementedAuthenticator from .http import HTMLFormAuthenticator, HTTPBasicAuthAuthenticator, HTTPDigestAuthAuthenticator from .http import HTTPDownloader from .s3 import S3Authenticator, S3Downloader from ..support.configparserinc import SafeConfigParserWithIncludes from ..support.external_versions import external_versions from ..utils import assure_list_from_str from ..utils import auto_repr lgr = getLogger('datalad.downloaders.providers') # dict to bind authentication_type's to authenticator classes # parameters will be fetched from config file itself AUTHENTICATION_TYPES = { 'html_form': HTMLFormAuthenticator, 'http_auth': HTTPBasicAuthAuthenticator, 'http_basic_auth': HTTPBasicAuthAuthenticator, 'http_digest_auth': HTTPDigestAuthAuthenticator, 'aws-s3': S3Authenticator, # TODO: check if having '-' is kosher 'nda-s3': S3Authenticator, 'xnat': NotImplementedAuthenticator, 'none': NoneAuthenticator, } from .credentials import CREDENTIAL_TYPES def resolve_url_to_name(d, url): """Given a directory (e.g. of SiteInformation._items or Credential._items) go through url_re and find the corresponding item and returns its key (i.e. name) """ for k, rec in iteritems(d): for url_re in rec.get('url_re', '').split('\n'): if url_re: if re.search(url_re, url): return k return None @auto_repr class Provider(object): """Class to bring together url_res, credential, and authenticator """ # TODO: we might need a lazy loading of the submodules which would provide # specific downloaders while importing needed Python modules "on demand" DOWNLOADERS = { 'http': {'class': HTTPDownloader, 'externals': {'requests'}}, 'https': {'class': HTTPDownloader, 'externals': {'requests'}}, 'ftp': {'class': HTTPDownloader, 'externals': {'requests', 'boto'}}, 's3': {'class': S3Downloader, 'externals': {'boto'}} # ... TODO } def __init__(self, name, url_res, credential=None, authenticator=None, downloader=None): """ Parameters ---------- name: str url_res: list of str Regular expressions credential: Credential, optional authenticator: Authenticator, optional downloader: Downloader, optional """ self.name = name self.url_res = assure_list_from_str(url_res) self.credential = credential self.authenticator = authenticator self._downloader = downloader @property def downloader(self): return self._downloader @staticmethod def get_scheme_from_url(url): """Given a URL return scheme to decide which downloader class to use """ url_split = urlparse(url) return url_split.scheme # , url_split.netloc) @classmethod def _get_downloader_class(cls, url): key = cls.get_scheme_from_url(url) if key in cls.DOWNLOADERS: entry = cls.DOWNLOADERS[key] klass = entry['class'] for ext in entry.get('externals', set()): if external_versions[ext] is None: raise RuntimeError( "For using %s downloader, you need '%s' dependency " "which seems to be missing" % (klass, ext) ) return klass else: raise ValueError("Do not know how to handle url %s for scheme %s. Known: %s" % (url, key, cls.DOWNLOADERS.keys())) def get_downloader(self, url, **kwargs): """Assigns proper downloader given the URL If one is known -- verifies its appropriateness for the given url. ATM we do not support multiple types of downloaders per single provider """ if self._downloader is None: # we need to create a new one Downloader = self._get_downloader_class(url) # we might need to provide it with credentials and authenticator # Let's do via kwargs so we could accomodate cases when downloader does not necessarily # cares about those... duck typing or what it is in action kwargs = kwargs.copy() if self.credential: kwargs['credential'] = self.credential if self.authenticator: kwargs['authenticator'] = self.authenticator self._downloader = Downloader(**kwargs) return self._downloader class Providers(object): """ So we could provide handling for URLs with corresponding credentials and specific (reusable) downloader. Internally it contains Providers and interfaces them based on a given URL. Each provider in turn takes care about associated with it Downloader. """ _DEFAULT_PROVIDERS = None def __init__(self, providers=None): """ """ self._providers = providers or [] # a set of providers to handle connections without authentication. # Will be setup one per each protocol schema self._default_providers = {} def __repr__(self): return "%s(%s)" % (self.__class__.__name__, "" if not self._providers else repr(self._providers)) def __len__(self): return len(self._providers) def __getitem__(self, index): return self._providers[index] def __iter__(self): return self._providers.__iter__() @classmethod def from_config_files(cls, files=None, reload=False): """Would load information about related/possible websites requiring authentication from - codebase (for now) datalad/downloaders/configs/providers.cfg - current dataset .datalad/providers/ - user dir ~/.config/datalad/providers/ - system-wide datalad installation/config /etc/datalad/providers/ For sample configs look into datalad/downloaders/configs/providers.cfg If files is None, loading is "lazy". Specify reload=True to force reload. reset_default_providers could also be used to reset the memoized providers """ # lazy part if files is None and cls._DEFAULT_PROVIDERS and not reload: return cls._DEFAULT_PROVIDERS config = SafeConfigParserWithIncludes() # TODO: support all those other paths files_orig = files if files is None: files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg')) config.read(files) # We need first to load Providers and credentials providers = {} credentials = {} for section in config.sections(): if ':' in section: type_, name = section.split(':', 1) assert type_ in {'provider', 'credential'}, "we know only providers and credentials, got type %s" % type_ items = { o: config.get(section, o) for o in config.options(section) } # side-effect -- items get poped locals().get(type_ + "s")[name] = getattr( cls, '_process_' + type_)(name, items) if len(items): raise ValueError("Unprocessed fields left for %s: %s" % (name, str(items))) else: lgr.warning("Do not know how to treat section %s here" % section) # link credentials into providers lgr.debug("Assigning credentials into %d providers" % len(providers)) for provider in providers.values(): if provider.credential: if provider.credential not in credentials: raise ValueError("Unknown credential %s. Known are: %s" % (provider.credential, ", ".join(credentials.keys()))) provider.credential = credentials[provider.credential] providers = Providers(list(providers.values())) if files_orig is None: # Store providers for lazy access cls._DEFAULT_PROVIDERS = providers return providers @classmethod def reset_default_providers(cls): """Resets to None memoized by from_config_files providers """ cls._DEFAULT_PROVIDERS = None @classmethod def _process_provider(cls, name, items): """Process a dictionary specifying the provider and output the Provider instance """ assert 'authentication_type' in items, "Must have authentication_type specified" auth_type = items.pop('authentication_type') if auth_type not in AUTHENTICATION_TYPES: raise ValueError("Unknown authentication_type=%s. Known are: %s" % (auth_type, ', '.join(AUTHENTICATION_TYPES))) if auth_type != 'none': authenticator = AUTHENTICATION_TYPES[auth_type]( # Extract all the fields as keyword arguments **{k[len(auth_type) + 1:]: items.pop(k) for k in list(items.keys()) if k.startswith(auth_type + "_")} ) else: authenticator = None # bringing url_re to "standard" format of a list and populating _providers_ordered url_res = assure_list_from_str(items.pop('url_re')) assert url_res, "current implementation relies on having url_re defined" credential = items.pop('credential', None) # credential instance will be assigned later after all of them are loaded return Provider(name=name, url_res=url_res, authenticator=authenticator, credential=credential) @classmethod def _process_credential(cls, name, items): assert 'type' in items, "Credential must specify type. Missing in %s" % name cred_type = items.pop('type') if not cred_type in CREDENTIAL_TYPES: raise ValueError("I do not know type %s credential. Known: %s" % (cred_type, CREDENTIAL_TYPES.keys())) return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None)) def get_provider(self, url, only_nondefault=False): """Given a URL returns matching provider """ nproviders = len(self._providers) for i in range(nproviders): provider = self._providers[i] if not provider.url_res: continue for url_re in provider.url_res: if re.match(url_re, url): if i != 0: # place it first # TODO: optimize with smarter datastructures if this becomes a burden del self._providers[i] self._providers = [provider] + self._providers assert(len(self._providers) == nproviders) lgr.debug("Returning provider %s for url %s", provider, url) return provider if only_nondefault: return None # None matched -- so we should get a default one per each of used # protocols scheme = Provider.get_scheme_from_url(url) if scheme not in self._default_providers: lgr.debug("Initializing default provider for %s" % scheme) self._default_providers[scheme] = Provider(name="", url_res=["%s://.*" % scheme]) provider = self._default_providers[scheme] lgr.debug("No dedicated provider, returning default one for %s: %s", scheme, provider) return provider # TODO: avoid duplication somehow ;) # Sugarings to get easier access to downloaders def download(self, url, *args, **kwargs): return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs) def fetch(self, url, *args, **kwargs): return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs) def get_status(self, url, *args, **kwargs): return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs) def needs_authentication(self, url): provider = self.get_provider(url, only_nondefault=True) if provider is None: return None return provider.authenticator is not None # # TODO: UNUSED? # def get_credentials(self, url, new=False): # """Ask user to enter credentials for a provider matching url # """ # # find a match among _items # provider = self.get_provider(url) # if new or not provider: # rec = self._get_new_record_ui(url) # rec['url_re'] = "TODO" # present to user and ask to edit # name = urlparse(url).netloc # self._items[name] = rec # if ui.yesno("Do you want to store credentials for %s" % name): # self.store_credentials() # else: # return self._items[name] # # def store_credentials(self, name): # # TODO: store self._items[name] in appropriate (user) creds # # for later reuse # raise NotImplementedError() # # def _get_new_record_ui(self, url): # # TODO: should be a dialog with the fields appropriate for this particular # # type of credentials # ui.message("To access %s we would need credentials." % url) # if url in self.providers: # ui.message("If you don't yet have credentials, please visit %s" # % self.providers.get(url, 'credentials_url')) # return { 'user': ui.question("Username:"), # 'password': ui.password() } datalad-0.4.1/datalad/downloaders/s3.py000066400000000000000000000204261301115270200177140ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Provide access to stuff (html, data files) via HTTP and HTTPS """ from six.moves.urllib.parse import urlsplit from ..utils import auto_repr from ..utils import assure_dict_from_str from ..dochelpers import borrowkwargs from ..support.network import get_url_straight_filename from ..support.network import rfc2822_to_epoch, iso8601_to_epoch from .base import Authenticator from .base import BaseDownloader, DownloaderSession from .base import DownloadError, TargetFileAbsent from ..support.s3 import boto, S3ResponseError, OrdinaryCallingFormat from ..support.s3 import get_bucket from ..support.status import FileStatus import logging from logging import getLogger lgr = getLogger('datalad.http') boto_lgr = logging.getLogger('boto') # not in effect at all, probably those are setup later #boto_lgr.handlers = lgr.handlers # Use our handlers __docformat__ = 'restructuredtext' @auto_repr class S3Authenticator(Authenticator): """Authenticator for S3 AWS """ DEFAULT_CREDENTIAL_TYPE = 'aws-s3' def __init__(self, *args, **kwargs): super(S3Authenticator, self).__init__(*args, **kwargs) self.connection = None self.bucket = None def authenticate(self, bucket_name, credential, cache=True): """Authenticates to the specified bucket using provided credentials Returns ------- bucket """ if not boto: raise RuntimeError("%s requires boto module which is N/A" % self) # Shut up boto if we do not care to listen ;) boto_lgr.setLevel( logging.CRITICAL if lgr.getEffectiveLevel() > 1 else logging.DEBUG ) # credential might contain 'session' token as well # which could be provided as security_token=., # see http://stackoverflow.com/questions/7673840/is-there-a-way-to-create-a-s3-connection-with-a-sessions-token conn_kwargs = {} if bucket_name.lower() != bucket_name: # per http://stackoverflow.com/a/19089045/1265472 conn_kwargs['calling_format'] = OrdinaryCallingFormat() credentials = credential() lgr.info("S3 session: Connecting to the bucket %s", bucket_name) self.connection = conn = boto.connect_s3( credentials['key_id'], credentials['secret_id'], security_token=credentials.get('session'), **conn_kwargs ) self.bucket = bucket = get_bucket(conn, bucket_name) return bucket @auto_repr class S3DownloaderSession(DownloaderSession): def __init__(self, size=None, filename=None, url=None, headers=None, key=None): super(S3DownloaderSession, self).__init__( size=size, filename=filename, headers=headers, url=url ) self.key = key def download(self, f=None, pbar=None, size=None): # S3 specific (the rest is common with e.g. http) def pbar_callback(downloaded, totalsize): assert (totalsize == self.key.size) if pbar: try: pbar.update(downloaded) except: # MIH: what does it do? MemoryError? pass # do not let pbar spoil our fun headers = {} # report for every % for files > 10MB, otherwise every 10% kwargs = dict(headers=headers, cb=pbar_callback, num_cb=100 if self.key.size > 10*(1024**2) else 10) if size: headers['Range'] = 'bytes=0-%d' % (size - 1) if f: # TODO: May be we could use If-Modified-Since # see http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html self.key.get_contents_to_file(f, **kwargs) else: return self.key.get_contents_as_string(encoding='utf-8', **kwargs) @auto_repr class S3Downloader(BaseDownloader): """Downloader from AWS S3 buckets """ _DEFAULT_AUTHENTICATOR = S3Authenticator @borrowkwargs(BaseDownloader) def __init__(self, **kwargs): super(S3Downloader, self).__init__(**kwargs) self._bucket = None @property def bucket(self): return self._bucket def reset(self): self._bucket = None @classmethod def _parse_url(cls, url): """Parses s3:// url and returns bucket name, prefix, additional query elements as a dict (such as VersionId)""" rec = urlsplit(url) assert(rec.scheme == 's3') # TODO: needs replacement to assure_ since it doesn't # deal with non key=value return rec.netloc, rec.path.lstrip('/'), assure_dict_from_str(rec.query, sep='&') or {} def _establish_session(self, url, allow_old=True): """ Parameters ---------- allow_old: bool, optional If a Downloader allows for persistent sessions by some means -- flag instructs either to use previous session, or establish a new one Returns ------- bool To state if old instance of a session/authentication was used """ bucket_name = self._parse_url(url)[0] if allow_old and self._bucket: if self._bucket.name == bucket_name: lgr.debug("S3 session: Reusing previous bucket") return True # we used old else: lgr.warning("No support yet for multiple buckets per S3Downloader") lgr.debug("S3 session: Reconnecting to the bucket") self._bucket = self.authenticator.authenticate(bucket_name, self.credential) return False def get_downloader_session(self, url, **kwargs): bucket_name, url_filepath, params = self._parse_url(url) if params: newkeys = set(params.keys()) - {'versionId'} if newkeys: raise NotImplementedError("Did not implement support for %s" % newkeys) assert(self._bucket.name == bucket_name) # must be the same try: key = self._bucket.get_key(url_filepath, version_id=params.get('versionId', None)) except S3ResponseError as e: raise DownloadError("S3 refused to provide the key for %s from url %s: %s" % (url_filepath, url, e)) if key is None: raise TargetFileAbsent("No key returned for %s from url %s" % (url_filepath, url)) target_size = key.size # S3 specific headers = { 'Content-Length': key.size, 'Content-Disposition': key.name } if key.last_modified: headers['Last-Modified'] = rfc2822_to_epoch(key.last_modified) # Consult about filename url_filename = get_url_straight_filename(url) return S3DownloaderSession( size=target_size, filename=url_filename, url=url, headers=headers, key=key ) @classmethod def get_key_headers(cls, key, dateformat='rfc2822'): headers = { 'Content-Length': key.size, 'Content-Disposition': key.name } if key.last_modified: # boto would return time string the way amazon returns which returns # it in two different ones depending on how key information was obtained: # https://github.com/boto/boto/issues/466 headers['Last-Modified'] = {'rfc2822': rfc2822_to_epoch, 'iso8601': iso8601_to_epoch}[dateformat](key.last_modified) return headers @classmethod def get_status_from_headers(cls, headers): # TODO: duplicated with http functionality # convert to FileStatus return FileStatus( size=headers.get('Content-Length'), mtime=headers.get('Last-Modified'), filename=headers.get('Content-Disposition') ) @classmethod def get_key_status(cls, key, dateformat='rfc2822'): return cls.get_status_from_headers(cls.get_key_headers(key, dateformat=dateformat)) datalad-0.4.1/datalad/downloaders/tests/000077500000000000000000000000001301115270200201535ustar00rootroot00000000000000datalad-0.4.1/datalad/downloaders/tests/__init__.py000066400000000000000000000000001301115270200222520ustar00rootroot00000000000000datalad-0.4.1/datalad/downloaders/tests/test_base.py000066400000000000000000000007061301115270200225010ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for downloaders base""" def test_docstring(): pass datalad-0.4.1/datalad/downloaders/tests/test_credentials.py000066400000000000000000000110251301115270200240600ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for credentials""" from mock import patch from datalad.tests.utils import with_testsui from datalad.tests.utils import assert_equal from datalad.tests.utils import assert_true, assert_false from datalad.tests.utils import assert_raises from datalad.tests.utils import SkipTest from datalad.support.keyring_ import MemoryKeyring from datalad.support.keyring_ import Keyring from ..credentials import UserPassword from ..credentials import CompositeCredential from ..credentials import AWS_S3 @with_testsui(responses=['user1', 'password1']) def test_cred1_enter_new(): keyring = MemoryKeyring() cred = UserPassword("name", keyring=keyring) assert_false(cred.is_known) assert_equal(cred.enter_new(), None) assert_true(cred.is_known) assert_equal(keyring.get('name', 'user'), 'user1') assert_equal(keyring.get('name', 'password'), 'password1') keyring.delete('name') assert_raises(KeyError, keyring.delete, 'name', 'user') assert_raises(KeyError, keyring.delete, 'name') assert_equal(keyring.get('name', 'user'), None) @with_testsui(responses=['password1']) def test_cred1_call(): keyring = MemoryKeyring() cred = UserPassword("name", keyring=keyring) # we will set the name but not the password, expecting UI # requesting it assert_equal(keyring.set('name', 'user', 'user1'), None) assert_equal(keyring.get('name', 'user'), 'user1') assert_equal(cred(), {'user': 'user1', 'password': 'password1'}) assert_equal(keyring.get('name', 'password'), 'password1') def test_keyring(): # mock out keyring methods and test that we are providing correct values # with 'datalad-' prefix raise SkipTest("provide tests for Keyring which interfaces keyring module") def _cred1_adapter(user=None, password=None): """Just a sample adapter from one user/pw type to another""" return dict(user=user + "_1", password=password + "_2") class _CCred1(CompositeCredential): """A Simple composite credential which will do some entries transformation """ _CREDENTIAL_CLASSES = (UserPassword, UserPassword) _CREDENTIAL_ADAPTERS = (_cred1_adapter,) @with_testsui(responses=['user1', 'password1', 'user2', 'password2']) def test_composite_credential1(): # basic test of composite credential keyring = MemoryKeyring() cred = _CCred1("name", keyring=keyring) # When queried, does the chain assert_equal(cred(), {'user': 'user1_1', 'password': 'password1_2'}) # But the "Front" credential is exposed to the user assert_equal(cred.get('user'), 'user1') assert_equal(keyring.get('name', 'user'), 'user1') assert_raises(ValueError, cred.get, 'unknown_field') assert_equal(cred.get('password'), 'password1') assert_equal(keyring.get('name', 'password'), 'password1') # ATM composite credential stores "derived" ones unconditionally in the # keyring as well assert_equal(keyring.get('name:1', 'user'), 'user1_1') assert_equal(keyring.get('name:1', 'password'), 'password1_2') # and now enter new should remove "derived" entries cred.enter_new() assert_equal(keyring.get('name', 'user'), 'user2') assert_equal(keyring.get('name', 'password'), 'password2') assert_equal(keyring.get('name:1', 'user'), None) assert_equal(keyring.get('name:1', 'password'), None) # which would get reevaluated if requested assert_equal(keyring.entries, {'name:1': {}, 'name': {'user': 'user2', 'password': 'password2'}}) assert_equal(cred(), {'user': 'user2_1', 'password': 'password2_2'}) def test_credentials_from_env(): keyring = Keyring() cred = AWS_S3("test-s3", keyring=keyring) assert_false(cred.is_known) assert_equal(cred.get('key_id'), None) assert_equal(cred.get('secret_id'), None) with patch.dict('os.environ', {'DATALAD_test_s3_key_id': '1'}): assert_equal(cred.get('key_id'), '1') assert_false(cred.is_known) with patch.dict('os.environ', {'DATALAD_test_s3_secret_id': '2'}): assert_equal(cred.get('key_id'), '1') assert_equal(cred.get('secret_id'), '2') assert_true(cred.is_known) assert_false(cred.is_known) # no memory of the pastdatalad-0.4.1/datalad/downloaders/tests/test_http.py000066400000000000000000000445321301115270200225530ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for http downloader""" import time from calendar import timegm from six import PY3 import os import six.moves.builtins as __builtin__ from os.path import join as opj from datalad.downloaders.tests.utils import get_test_providers from ..base import DownloadError from ..base import IncompleteDownloadError from ..base import BaseDownloader from ..credentials import UserPassword from ..http import HTMLFormAuthenticator from ..http import HTTPDownloader from ...support.network import get_url_straight_filename from ...tests.utils import with_fake_cookies_db from ...tests.utils import skip_if_no_network from ...tests.utils import with_testsui from ...tests.utils import with_memory_keyring # BTW -- mock_open is not in mock on wheezy (Debian 7.x) try: if PY3: raise ImportError("Not yet ready apparently: https://travis-ci.org/datalad/datalad/jobs/111659666") import httpretty except ImportError: class NoHTTPPretty(object): __bool__ = __nonzero__ = lambda s: False activate = lambda s, t: t httpretty = NoHTTPPretty() from mock import patch from ...tests.utils import SkipTest from ...tests.utils import assert_in from ...tests.utils import assert_not_in from ...tests.utils import assert_equal from ...tests.utils import assert_greater from ...tests.utils import assert_false from ...tests.utils import assert_raises from ...tests.utils import ok_file_has_content from ...tests.utils import serve_path_via_http, with_tree from ...tests.utils import swallow_logs from ...tests.utils import swallow_outputs from ...tests.utils import with_tempfile from ...tests.utils import use_cassette from ...tests.utils import skip_if from ...tests.utils import without_http_proxy from ...support.status import FileStatus from ...support.network import get_url_disposition_filename def test_docstring(): doc = HTTPDownloader.__init__.__doc__ assert_in("\ncredential: Credential", doc) # XXX doesn't quite work as it should since doesn't provide context handling # I guess... but at least causes the DownloadError ;) def fake_open(write_=None): class myfile(object): """file which does nothing""" if write_: def write(self, *args, **kwargs): write_(*args, **kwargs) def close(self): pass def myopen(*args, **kwargs): return myfile return myopen def _raise_IOError(*args, **kwargs): raise IOError("Testing here") @with_tree(tree=[('file.dat', 'abc')]) @serve_path_via_http def test_HTTPDownloader_basic(toppath, topurl): furl = "%sfile.dat" % topurl tfpath = opj(toppath, "file-downloaded.dat") downloader = HTTPDownloader() # no auth/credentials needed download = downloader.download download(furl, tfpath) ok_file_has_content(tfpath, 'abc') # see if fetch works correctly assert_equal(downloader.fetch(furl), 'abc') # By default should not overwrite the file assert_raises(DownloadError, download, furl, tfpath) # but be able to redownload whenever overwrite==True downloaded_path = download(furl, tfpath, overwrite=True) assert_equal(downloaded_path, tfpath) ok_file_has_content(tfpath, 'abc') # Some errors handling # XXX obscure mocking since impossible to mock write alone # and it still results in some warning being spit out with swallow_logs(), \ patch.object(__builtin__, 'open', fake_open(write_=_raise_IOError)): assert_raises(DownloadError, download, furl, tfpath, overwrite=True) # incomplete download scenario - should have 3 tries def _fail_verify_download(try_to_fail): try_ = [0] _orig_verify_download = BaseDownloader._verify_download def _verify_download(self, *args, **kwargs): try_[0] += 1 if try_[0] >= try_to_fail: return _orig_verify_download(self, *args, **kwargs) raise IncompleteDownloadError() return _verify_download with patch.object(BaseDownloader, '_verify_download', _fail_verify_download(6)), \ swallow_logs(): # how was before the "fix": #assert_raises(DownloadError, downloader.fetch, furl) #assert_raises(DownloadError, downloader.fetch, furl) # now should download just fine assert_equal(downloader.fetch(furl), 'abc') # but should fail if keeps failing all 5 times and then on 11th should raise DownloadError with patch.object(BaseDownloader, '_verify_download', _fail_verify_download(7)), \ swallow_logs(): assert_raises(DownloadError, downloader.fetch, furl) # TODO: access denied scenario # TODO: access denied detection @with_tempfile(mkdir=True) def check_download_external_url(url, failed_str, success_str, d, url_final=None): fpath = opj(d, get_url_straight_filename(url)) providers = get_test_providers(url) # url for check of credentials provider = providers.get_provider(url) downloader = provider.get_downloader(url) # Download way with swallow_outputs() as cmo: downloaded_path = downloader.download(url, path=d) assert_equal(fpath, downloaded_path) with open(fpath) as f: content = f.read() if success_str is not None: assert_in(success_str, content) if failed_str is not None: assert_false(failed_str in content) # And if we specify size for s in [1, 2]: with swallow_outputs() as cmo: downloaded_path_ = downloader.download(url, path=d, size=s, overwrite=True) # should not be affected assert_equal(downloaded_path, downloaded_path_) with open(fpath) as f: content_ = f.read() assert_equal(len(content_), s) assert_equal(content_, content[:s]) # Fetch way content = downloader.fetch(url) if success_str is not None: assert_in(success_str, content) if failed_str is not None: assert_false(failed_str in content) # And if we specify size for s in [1, 2]: with swallow_outputs() as cmo: content_ = downloader.fetch(url, size=s) assert_equal(len(content_), s) assert_equal(content_, content[:s]) # Verify status status = downloader.get_status(url) assert(isinstance(status, FileStatus)) if not url.startswith('ftp://'): # TODO introduce support for mtime into requests_ftp? assert(status.mtime) assert(status.size) # Verify possible redirections if url_final is None: url_final = url assert_equal(downloader.get_target_url(url), url_final) # TODO -- more and more specific @skip_if_no_network @use_cassette('test_authenticate_external_portals', record_mode='once') def test_authenticate_external_portals(): yield check_download_external_url, \ "https://portal.nersc.gov/project/crcns/download/alm-1/checksums.md5", \ " different checksum, despit # same content assert_not_equal(md5sum(default_outname), custom1_md5) # should really sleep so if they stop using time.time - we know time.sleep(1.1) ds.export('tarball', output=custom_outname) # should not encode mtime, so should be identical assert_equal(md5sum(custom_outname), custom1_md5) def check_contents(outname, prefix): with tarfile.open(outname) as tf: nfiles = 0 for ti in tf: # any annex links resolved assert_false(ti.issym()) ok_startswith(ti.name, prefix + '/') assert_equal(ti.mtime, committed_date) if '.datalad' not in ti.name: # ignore any files in .datalad for this test to not be # susceptible to changes in how much we generate a meta info nfiles += 1 # we have exactly three files, and expect no content for any directory assert_equal(nfiles, 3) check_contents(default_outname, 'datalad_%s' % ds.id) check_contents(custom_outname, 'myexport') datalad-0.4.1/datalad/interface/000077500000000000000000000000001301115270200164305ustar00rootroot00000000000000datalad-0.4.1/datalad/interface/__init__.py000066400000000000000000000053461301115270200205510ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface definition """ __docformat__ = 'restructuredtext' # the following should be series of import definitions for interface implementations # that shall be exposed in the Python API and the cmdline interface # all interfaces should be associated with (at least) one of the groups below _group_dataset = ( 'Commands for dataset operations', [ # source module, source object[, dest. cmdline name[, dest python name]] # src module can be relative, but has to be relative to the main 'datalad' package ('datalad.distribution.create', 'Create'), ('datalad.distribution.install', 'Install'), ('datalad.distribution.get', 'Get'), ('datalad.distribution.add', 'Add'), ('datalad.distribution.publish', 'Publish'), ('datalad.distribution.uninstall', 'Uninstall'), # N/I ATM # ('datalad.distribution.move', 'Move'), ('datalad.distribution.update', 'Update'), ('datalad.distribution.create_sibling', 'CreateSibling', 'create-sibling'), ('datalad.distribution.create_sibling_github', 'CreateSiblingGithub', 'create-sibling-github'), ('datalad.distribution.add_sibling', 'AddSibling', 'add-sibling'), ('datalad.distribution.rewrite_urls', 'RewriteURLs', 'rewrite-urls'), ('datalad.interface.unlock', 'Unlock', 'unlock'), ('datalad.interface.save', 'Save', 'save'), ('datalad.export', 'Export', 'export'), ]) _group_metadata = ( 'Commands for meta data handling', [ ('datalad.metadata.search', 'Search', 'search', 'search'), ('datalad.metadata.aggregate', 'AggregateMetaData', 'aggregate-metadata', 'aggregate_metadata'), ]) _group_misc = ( 'Miscellaneous commands', [ ('datalad.interface.test', 'Test'), ('datalad.interface.crawl', 'Crawl'), ('datalad.interface.crawl_init', 'CrawlInit', 'crawl-init'), ('datalad.interface.ls', 'Ls'), ('datalad.interface.clean', 'Clean'), ('datalad.interface.add_archive_content', 'AddArchiveContent', 'add-archive-content'), ('datalad.interface.download_url', 'DownloadURL', 'download-url'), # very optional ones ('datalad.distribution.create_test_dataset', 'CreateTestDataset', 'create-test-dataset'), ]) datalad-0.4.1/datalad/interface/add_archive_content.py000066400000000000000000000536111301115270200227730ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface for adding content of an archive under annex control """ from __future__ import print_function __docformat__ = 'restructuredtext' import re import os import shlex import tempfile from os.path import join as opj, realpath, curdir, exists, lexists, relpath, basename from os.path import commonprefix from os.path import sep as opsep from os.path import islink from os.path import isabs from os.path import dirname from os.path import normpath from .base import Interface from .common_opts import allow_dirty from ..consts import ARCHIVES_SPECIAL_REMOTE from ..support.param import Parameter from ..support.constraints import EnsureStr, EnsureNone from ..support.annexrepo import AnnexRepo from ..support.strings import apply_replacement_rules from ..support.stats import ActivityStats from ..cmdline.helpers import get_repo_instance from ..utils import getpwd, rmtree, file_basename from ..utils import md5sum from ..utils import assure_tuple_or_list from six import string_types from ..log import logging lgr = logging.getLogger('datalad.interfaces.add_archive_content') # Shortcut note _KEY_OPT = "[PY: `key=True` PY][CMD: --key CMD]" _KEY_OPT_NOTE = "Note that it will be of no effect if %s is given" % _KEY_OPT # TODO: may be we could enable separate logging or add a flag to enable # all but by default to print only the one associated with this given action class AddArchiveContent(Interface): """Add content of an archive under git annex control. This results in the files within archive (which should be under annex control itself) added under annex referencing original archive via custom special remotes mechanism Example: annex-repo$ datalad add-archive-content my_big_tarball.tar.gz """ _params_ = dict( delete=Parameter( args=("-d", "--delete"), action="store_true", doc="""flag to delete original archive from the filesystem/git in current tree. %s""" % _KEY_OPT_NOTE), add_archive_leading_dir=Parameter( args=("--add-archive-leading-dir",), action="store_true", doc="""flag to place extracted content under a directory which would correspond to archive name with suffix stripped. E.g. for archive `example.zip` its content will be extracted under a directory `example/`"""), strip_leading_dirs=Parameter( args=("--strip-leading-dirs",), action="store_true", doc="""flag to move all files directories up, from how they were stored in an archive, if that one contained a number (possibly more than 1 down) single leading directories"""), leading_dirs_depth=Parameter( args=("--leading-dirs-depth",), action="store", type=int, doc="""maximal depth to strip leading directories to. If not specified (None), no limit"""), leading_dirs_consider=Parameter( args=("--leading-dirs-consider",), action="append", doc="""regular expression(s) for directories to consider to strip away""", constraints=EnsureStr() | EnsureNone(), ), use_current_dir=Parameter( args=("--use-current-dir",), action="store_true", doc="""flag to extract archive under the current directory, not the directory where archive is located. %s""" % _KEY_OPT_NOTE), # TODO: add option to extract under archive's original directory. Currently would extract in curdir existing=Parameter( args=("--existing",), choices=('fail', 'overwrite', 'archive-suffix', 'numeric-suffix'), default="fail", doc="""what operation to perform a file from archive tries to overwrite an existing file with the same name. 'fail' (default) leads to RuntimeError exception. 'overwrite' silently replaces existing file. 'archive-suffix' instructs to add a suffix (prefixed with a '-') matching archive name from which file gets extracted, and if that one present, 'numeric-suffix' is in effect in addition, when incremental numeric suffix (prefixed with a '.') is added until no name collision is longer detected""" ), exclude=Parameter( args=("-e", "--exclude"), action='append', doc="""regular expressions for filenames which to exclude from being added to annex. Applied after --rename if that one is specified. For exact matching, use anchoring""", constraints=EnsureStr() | EnsureNone() ), rename=Parameter( args=("-r", "--rename"), action='append', doc="""regular expressions to rename files before being added under git. First letter defines how to split provided string into two parts: Python regular expression (with groups), and replacement string""", constraints=EnsureStr(min_len=2) | EnsureNone() ), annex_options=Parameter( args=("-o", "--annex-options"), doc="""additional options to pass to git-annex""", constraints=EnsureStr() | EnsureNone() ), # TODO: Python only??? annex=Parameter( doc="""annex instance to use""" #constraints=EnsureStr() | EnsureNone() ), # TODO: Python only! stats=Parameter( doc="""ActivityStats instance for global tracking""", ), key=Parameter( args=("--key",), action="store_true", doc="""flag to signal if provided archive is not actually a filename on its own but an annex key"""), copy=Parameter( args=("--copy",), action="store_true", doc="""flag to copy the content of the archive instead of moving"""), allow_dirty=allow_dirty, commit=Parameter( args=("--no-commit",), action="store_false", dest="commit", doc="""flag to not commit upon completion"""), drop_after=Parameter( args=("--drop-after",), action="store_true", doc="""drop extracted files after adding to annex""", ), delete_after=Parameter( args=("--delete-after",), action="store_true", doc="""extract under a temporary directory, git-annex add, and delete after. To be used to "index" files within annex without actually creating corresponding files under git. Note that `annex dropunused` would later remove that load"""), # TODO: interaction with archives cache whenever we make it persistent across runs archive=Parameter( doc="archive file or a key (if %s specified)" % _KEY_OPT, constraints=EnsureStr()), ) # use-case from openfmri pipeline # ExtractArchives( # # will do the merge of 'replace' strategy # source_branch="incoming", # regex="\.(tgz|tar\..*)$", # renames=[ # ("^[^/]*/(.*)", "\1") # e.g. to strip leading dir, or could prepend etc # ], # #exclude="license.*", # regexp # ), @staticmethod def __call__(archive, annex=None, add_archive_leading_dir=False, strip_leading_dirs=False, leading_dirs_depth=None, leading_dirs_consider=None, use_current_dir=False, delete=False, key=False, exclude=None, rename=None, existing='fail', annex_options=None, copy=False, commit=True, allow_dirty=False, stats=None, drop_after=False, delete_after=False): """ Returns ------- annex """ if exclude: exclude = assure_tuple_or_list(exclude) if rename: rename = assure_tuple_or_list(rename) # TODO: actually I see possibly us asking user either he wants to convert # his git repo into annex archive_path = archive pwd = getpwd() if annex is None: annex = get_repo_instance(pwd, class_=AnnexRepo) if not isabs(archive): # if not absolute -- relative to wd and thus archive_path = normpath(opj(pwd, archive)) # abspath(archive) is not "good" since dereferences links in the path # archive_path = abspath(archive) elif not isabs(archive): # if we are given an annex, then assume that given path is within annex, not # relative to PWD archive_path = opj(annex.path, archive) annex_path = annex.path # _rpath below should depict paths relative to the top of the annex archive_rpath = relpath(archive_path, annex_path) # TODO: somewhat too cruel -- may be an option or smth... if not allow_dirty and annex.dirty: # already saved me once ;) raise RuntimeError("You better commit all the changes and untracked files first") if not key: # we were given a file which must exist if not exists(archive_path): raise ValueError("Archive {} does not exist".format(archive)) # TODO: support adding archives content from outside the annex/repo origin = 'archive' key = annex.get_file_key(archive_rpath) archive_dir = dirname(archive_path) else: origin = 'key' key = archive archive_dir = None # We must not have anything to do with the location under .git/annex archive_basename = file_basename(archive) if not key: # TODO: allow for it to be under git??? how to reference then? raise NotImplementedError( "Provided file %s is not under annex. We don't support yet adding everything " "straight to git" % archive ) # are we in a subdirectory of the repository? pwd_under_annex = commonprefix([pwd, annex_path]) == annex_path # then we should add content under that # subdirectory, # get the path relative to the repo top if use_current_dir: # if outside -- extract to the top of repo extract_rpath = relpath(pwd, annex_path) \ if pwd_under_annex \ else None else: extract_rpath = relpath(archive_dir, annex_path) # relpath might return '.' as the relative path to curdir, which then normalize_paths # would take as instructions to really go from cwd, so we need to sanitize if extract_rpath == curdir: extract_rpath = None # no special relpath from top of the repo # and operate from now on the key or whereever content available "canonically" try: key_rpath = annex.get_contentlocation(key) # , relative_to_top=True) except: raise RuntimeError("Content of %s seems to be N/A. Fetch it first" % key) # now we simply need to go through every file in that archive and lgr.info("Adding content of the archive %s into annex %s", archive, annex) from datalad.customremotes.archives import ArchiveAnnexCustomRemote # TODO: shouldn't we be able just to pass existing AnnexRepo instance? # TODO: we will use persistent cache so we could just (ab)use possibly extracted archive annexarchive = ArchiveAnnexCustomRemote(path=annex_path, persistent_cache=True) # We will move extracted content so it must not exist prior running annexarchive.cache.allow_existing = True earchive = annexarchive.cache[key_rpath] # TODO: check if may be it was already added if ARCHIVES_SPECIAL_REMOTE not in annex.get_remotes(): lgr.debug("Adding new special remote {}".format(ARCHIVES_SPECIAL_REMOTE)) annex.init_remote( ARCHIVES_SPECIAL_REMOTE, ['encryption=none', 'type=external', 'externaltype=%s' % ARCHIVES_SPECIAL_REMOTE, 'autoenable=true']) else: lgr.debug("Special remote {} already exists".format(ARCHIVES_SPECIAL_REMOTE)) try: old_always_commit = annex.always_commit annex.always_commit = False if annex_options: if isinstance(annex_options, string_types): annex_options = shlex.split(annex_options) leading_dir = earchive.get_leading_directory( depth=leading_dirs_depth, exclude=exclude, consider=leading_dirs_consider) \ if strip_leading_dirs else None leading_dir_len = len(leading_dir) + len(opsep) if leading_dir else 0 # we need to create a temporary directory at the top level which would later be # removed prefix_dir = basename(tempfile.mkdtemp(prefix=".datalad", dir=annex_path)) \ if delete_after \ else None # dedicated stats which would be added to passed in (if any) outside_stats = stats stats = ActivityStats() for extracted_file in earchive.get_extracted_files(): stats.files += 1 extracted_path = opj(earchive.path, extracted_file) if islink(extracted_path): link_path = realpath(extracted_path) if not exists(link_path): # TODO: config addarchive.symlink-broken='skip' lgr.warning("Path %s points to non-existing file %s" % (extracted_path, link_path)) stats.skipped += 1 continue # TODO: check if points outside of the archive -- warning and skip # preliminary target name which might get modified by renames target_file_orig = target_file = extracted_file # strip leading dirs target_file = target_file[leading_dir_len:] if add_archive_leading_dir: target_file = opj(archive_basename, target_file) if rename: target_file = apply_replacement_rules(rename, target_file) # continue to next iteration if extracted_file in excluded if exclude: try: # since we need to skip outside loop from inside loop for regexp in exclude: if re.search(regexp, extracted_file): lgr.debug( "Skipping {extracted_file} since contains {regexp} pattern".format(**locals())) stats.skipped += 1 raise StopIteration except StopIteration: continue if prefix_dir: target_file = opj(prefix_dir, target_file) url = annexarchive.get_file_url(archive_key=key, file=extracted_file, size=os.stat(extracted_path).st_size) # lgr.debug("mv {extracted_path} {target_file}. URL: {url}".format(**locals())) if lexists(target_file): if md5sum(target_file) == md5sum(extracted_path): # must be having the same content, we should just add possibly a new extra URL pass elif existing == 'fail': raise RuntimeError( "File {} already exists, but new (?) file {} was instructed " "to be placed there while overwrite=False".format(target_file, extracted_file)) elif existing == 'overwrite': stats.overwritten += 1 # to make sure it doesn't conflict -- might have been a tree rmtree(target_file) else: target_file_orig_ = target_file # To keep extension intact -- operate on the base of the filename p, fn = os.path.split(target_file) ends_with_dot = fn.endswith('.') fn_base, fn_ext = file_basename(fn, return_ext=True) if existing == 'archive-suffix': fn_base += '-%s' % archive_basename elif existing == 'numeric-suffix': pass # archive-suffix will have the same logic else: raise ValueError(existing) # keep incrementing index in the suffix until file doesn't collide suf, i = '', 0 while True: target_file_new = opj(p, fn_base + suf + ('.' if (fn_ext or ends_with_dot) else '') + fn_ext) if not lexists(target_file_new): break lgr.debug("File %s already exists" % target_file_new) i += 1 suf = '.%d' % i target_file = target_file_new lgr.debug("Original file %s will be saved into %s" % (target_file_orig_, target_file)) # TODO: should we reserve smth like # stats.clobbed += 1 if target_file != target_file_orig: stats.renamed += 1 #target_path = opj(getpwd(), target_file) if copy: raise NotImplementedError("Not yet copying from 'persistent' cache") else: # os.renames(extracted_path, target_path) # addurl implementation relying on annex'es addurl below would actually copy pass lgr.debug("Adding %s to annex pointing to %s and with options %r", target_file, url, annex_options) target_file_rpath = opj(extract_rpath, target_file) if extract_rpath else target_file out_json = annex.add_url_to_file( target_file_rpath, url, options=annex_options, batch=True) if 'key' in out_json and out_json['key'] is not None: # annex.is_under_annex(target_file, batch=True): # due to http://git-annex.branchable.com/bugs/annex_drop_is_not___34__in_effect__34___for_load_which_was___34__addurl_--batch__34__ed_but_not_yet_committed/?updated # we need to maintain a list of those to be dropped files if drop_after: annex.drop_key(out_json['key'], batch=True) stats.dropped += 1 stats.add_annex += 1 else: lgr.debug("File {} was added to git, not adding url".format(target_file)) stats.add_git += 1 if delete_after: # forcing since it is only staged, not yet committed annex.remove(target_file_rpath, force=True) # TODO: batch! stats.removed += 1 # # chaining 3 annex commands, 2 of which not batched -- less efficient but more bullet proof etc # annex.add(target_path, options=annex_options) # # above action might add to git or to annex # if annex.file_has_content(target_path): # # if not -- it was added to git, if in annex, it is present and output is True # annex.add_url_to_file(target_file, url, options=['--relaxed'], batch=True) # stats.add_annex += 1 # else: # lgr.debug("File {} was added to git, not adding url".format(target_file)) # stats.add_git += 1 # # TODO: actually check if it is anyhow different from a previous version. If not # # then it wasn't really added del target_file # Done with target_file -- just to have clear end of the loop if delete and archive and origin != 'key': lgr.debug("Removing the original archive {}".format(archive)) # force=True since some times might still be staged and fail annex.remove(archive_rpath, force=True) lgr.info("Finished adding %s: %s" % (archive, stats.as_str(mode='line'))) if outside_stats: outside_stats += stats if commit: commit_stats = outside_stats if outside_stats else stats annex.commit( "Added content extracted from %s %s\n\n%s" % (origin, archive, commit_stats.as_str(mode='full')), _datalad_msg=True ) commit_stats.reset() finally: # since we batched addurl, we should close those batched processes annex.precommit() if delete_after: prefix_path = opj(annex_path, prefix_dir) if exists(prefix_path): # probably would always be there lgr.info("Removing temporary directory under which extracted files were annexed: %s", prefix_path) rmtree(prefix_path) annex.always_commit = old_always_commit # remove what is left and/or everything upon failure earchive.clean(force=True) return annex datalad-0.4.1/datalad/interface/base.py000066400000000000000000000245761301115270200177320ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """High-level interface generation """ __docformat__ = 'restructuredtext' import sys import re import textwrap from ..ui import ui from ..dochelpers import exc_str def get_api_name(intfspec): """Given an interface specification return an API name for it""" if len(intfspec) > 3: name = intfspec[3] else: name = intfspec[0].split('.')[-1] return name def get_cmdline_command_name(intfspec): """Given an interface specification return a cmdline command name""" if len(intfspec) > 2: name = intfspec[2] else: name = intfspec[0].split('.')[-1].replace('_', '-') return name def get_interface_groups(): from .. import interface as _interfaces grps = [] # auto detect all available interfaces and generate a function-based # API from them for _item in _interfaces.__dict__: if not _item.startswith('_group_'): continue grp_name = _item[7:] grp = getattr(_interfaces, _item) grps.append((grp_name,) + grp) return grps def dedent_docstring(text): """Remove uniform indentation from a multiline docstring""" # Problem is that first line might often have no offset, so might # need to be ignored from dedent call if text is None: return None if not text.startswith(' '): lines = text.split('\n') if len(lines) == 1: # single line, no indentation, nothing to do return text text2 = '\n'.join(lines[1:]) return lines[0] + "\n" + textwrap.dedent(text2) else: return textwrap.dedent(text) def alter_interface_docs_for_api(docs): """Apply modifications to interface docstrings for Python API use.""" # central place to alter the impression of docstrings, # like removing cmdline specific sections if not docs: return docs docs = dedent_docstring(docs) # clean cmdline sections docs = re.sub( '\|\| CMDLINE \>\>.*\<\< CMDLINE \|\|', '', docs, flags=re.MULTILINE | re.DOTALL) # clean cmdline in-line bits docs = re.sub( '\[CMD:\s[^\[\]]*\sCMD\]', '', docs, flags=re.MULTILINE | re.DOTALL) docs = re.sub( '\[PY:\s([^\[\]]*)\sPY\]', lambda match: match.group(1), docs, flags=re.MULTILINE) docs = re.sub( '\|\| PYTHON \>\>(.*)\<\< PYTHON \|\|', lambda match: match.group(1), docs, flags=re.MULTILINE | re.DOTALL) docs = re.sub( '\|\| REFLOW \>\>\n(.*)\<\< REFLOW \|\|', lambda match: textwrap.fill(match.group(1)), docs, flags=re.MULTILINE | re.DOTALL) return docs def alter_interface_docs_for_cmdline(docs): """Apply modifications to interface docstrings for cmdline doc use.""" # central place to alter the impression of docstrings, # like removing Python API specific sections, and argument markup if not docs: return docs docs = dedent_docstring(docs) # clean cmdline sections docs = re.sub( '\|\| PYTHON \>\>.*\<\< PYTHON \|\|', '', docs, flags=re.MULTILINE | re.DOTALL) # clean cmdline in-line bits docs = re.sub( '\[PY:\s[^\[\]]*\sPY\]', '', docs, flags=re.MULTILINE | re.DOTALL) docs = re.sub( '\[CMD:\s([^\[\]]*)\sCMD\]', lambda match: match.group(1), docs, flags=re.MULTILINE) docs = re.sub( '\|\| CMDLINE \>\>(.*)\<\< CMDLINE \|\|', lambda match: match.group(1), docs, flags=re.MULTILINE | re.DOTALL) # remove :role:`...` RST markup for cmdline docs docs = re.sub( r':\S+:`[^`]*`[\\]*', lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\'), docs, flags=re.MULTILINE | re.DOTALL) # remove None constraint. In general, `None` on the cmdline means don't # give option at all, but specifying `None` explicitly is practically # impossible docs = re.sub( ',\sor\svalue\smust\sbe\s`None`', '', docs, flags=re.MULTILINE | re.DOTALL) # capitalize variables and remove backticks to uniformize with # argparse output docs = re.sub( '`\S*`', lambda match: match.group(0).strip('`').upper(), docs) # clean up sphinx API refs docs = re.sub( '\~datalad\.api\.\S*', lambda match: "`{0}`".format(match.group(0)[13:]), docs) # Remove RST paragraph markup docs = re.sub( r'^.. \S+::', lambda match: match.group(0)[3:-2].upper(), docs, flags=re.MULTILINE) docs = re.sub( '\|\| REFLOW \>\>\n(.*)\<\< REFLOW \|\|', lambda match: textwrap.fill(match.group(1)), docs, flags=re.MULTILINE | re.DOTALL) return docs def is_api_arg(arg): """Return True if argument is our API argument or self or used for internal purposes """ return arg != 'self' and not arg.startswith('_') def update_docstring_with_parameters(func, params, prefix=None, suffix=None): """Generate a useful docstring from a parameter spec Amends any existing docstring of a callable with a textual description of its parameters. The Parameter spec needs to match the number and names of the callables arguments. """ from inspect import getargspec # get the signature ndefaults = 0 args, varargs, varkw, defaults = getargspec(func) if not defaults is None: ndefaults = len(defaults) # start documentation with what the callable brings with it doc = prefix if prefix else u'' if len(args) > 1: if len(doc): doc += '\n' doc += "Parameters\n----------\n" for i, arg in enumerate(args): if not is_api_arg(arg): continue # we need a parameter spec for each argument if not arg in params: raise ValueError("function has argument '%s' not described as a parameter" % arg) param = params[arg] # validate the default -- to make sure that the parameter description is # somewhat OK defaults_idx = ndefaults - len(args) + i if defaults_idx >= 0: if not param.constraints is None: param.constraints(defaults[defaults_idx]) orig_docs = param._doc param._doc = alter_interface_docs_for_api(param._doc) doc += param.get_autodoc( arg, default=defaults[defaults_idx] if defaults_idx >= 0 else None, has_default=defaults_idx >= 0) param._doc = orig_docs doc += '\n' doc += suffix if suffix else u"" # assign the amended docs func.__doc__ = doc return func class Interface(object): """Base class for interface implementations""" @classmethod def setup_parser(cls, parser): # XXX needs safety check for name collisions # XXX allow for parser kwargs customization parser_kwargs = {} from inspect import getargspec # get the signature ndefaults = 0 args, varargs, varkw, defaults = getargspec(cls.__call__) if not defaults is None: ndefaults = len(defaults) for i, arg in enumerate(args): if not is_api_arg(arg): continue param = cls._params_[arg] defaults_idx = ndefaults - len(args) + i cmd_args = param.cmd_args if cmd_args is None: cmd_args = [] if not len(cmd_args): if defaults_idx >= 0: # dealing with a kwarg template = '--%s' else: # positional arg template = '%s' # use parameter name as default argument name parser_args = (template % arg.replace('_', '-'),) else: parser_args = [c.replace('_', '-') for c in cmd_args] parser_kwargs = param.cmd_kwargs if defaults_idx >= 0: parser_kwargs['default'] = defaults[defaults_idx] help = alter_interface_docs_for_cmdline(param._doc) if help and help[-1] != '.': help += '.' if param.constraints is not None: parser_kwargs['type'] = param.constraints # include value constraint description and default # into the help string cdoc = alter_interface_docs_for_cmdline( param.constraints.long_description()) if cdoc[0] == '(' and cdoc[-1] == ')': cdoc = cdoc[1:-1] help += ' Constraints: %s' % cdoc if defaults_idx >= 0: help += " [Default: %r]" % (defaults[defaults_idx],) # create the parameter, using the constraint instance for type # conversion parser.add_argument(*parser_args, help=help, **parser_kwargs) @classmethod def call_from_parser(cls, args): # XXX needs safety check for name collisions from inspect import getargspec argspec = getargspec(cls.__call__) if argspec[2] is None: # no **kwargs in the call receiver, pull argnames from signature argnames = getargspec(cls.__call__)[0] else: # common options # XXX define or better get from elsewhere common_opts = ('change_path', 'common_debug', 'common_idebug', 'func', 'help', 'log_level', 'logger', 'pbs_runner', 'result_renderer', 'subparser') argnames = [name for name in dir(args) if not (name.startswith('_') or name in common_opts)] kwargs = {k: getattr(args, k) for k in argnames if is_api_arg(k)} try: return cls.__call__(**kwargs) except KeyboardInterrupt as exc: ui.error("\nInterrupted by user while doing magic: %s" % exc_str(exc)) sys.exit(1) datalad-0.4.1/datalad/interface/clean.py000066400000000000000000000041111301115270200200610ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## __docformat__ = 'restructuredtext' from os.path import join as opj from glob import glob from .base import Interface from ..ui import ui from ..utils import rmtree, getpwd from ..support.param import Parameter from ..support.annexrepo import AnnexRepo from ..cmdline.helpers import get_repo_instance from ..consts import ARCHIVES_TEMP_DIR from logging import getLogger lgr = getLogger('datalad.api.clean') class Clean(Interface): """Clean up after DataLad (possible temporary files etc.) Removes extracted temporary archives, etc. Examples: $ datalad clean """ _params_ = dict( # TODO: --info -- which performs dry run just summarizing what is to be cleaned up # TODO: Python only??? annex=Parameter( doc="""annex instance to use""" #constraints=EnsureStr() | EnsureNone() ), ) @staticmethod def __call__(annex=None): if annex is None: annex = get_repo_instance(class_=AnnexRepo) topdir = opj(annex.path, ARCHIVES_TEMP_DIR) temp_archives_dirs = glob(opj(annex.path, ARCHIVES_TEMP_DIR, '*')) if temp_archives_dirs: pl = len(temp_archives_dirs) > 1 pl1, pl2 = ('s', 'ies') if pl else ('', 'y') pwd = getpwd() # relative version if possible rtopdir = topdir[len(pwd) + 1:] \ if topdir.startswith(pwd) else topdir ui.message("Removing %d temporary archive director%s under %s: %s" % (len(temp_archives_dirs), pl2, rtopdir, ", ".join(sorted([x[len(topdir) + 1:] for x in temp_archives_dirs])))) rmtree(topdir) datalad-0.4.1/datalad/interface/common_cfg.py000066400000000000000000000154471301115270200211240ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Common configuration options """ __docformat__ = 'restructuredtext' from appdirs import AppDirs from datalad.support.constraints import EnsureBool dirs = AppDirs("datalad", "datalad.org") definitions = { # this is actually used in downloaders, but kept cfg name original 'datalad.crawl.cache': { 'ui': ('yesno', { 'title': 'Crawler download caching', 'text': 'Should the crawler cache downloaded files?'}), 'destination': 'local', 'type': bool, }, 'datalad.crawl.default_backend': { 'ui': ('question', { 'title': 'Default annex backend', # XXX we could add choices... but might get out of sync 'text': 'Content hashing method to be used by git-annex'}), 'destination': 'dataset', }, 'datalad.crawl.dryrun': { 'ui': ('yesno', { 'title': 'Crawler dry-run', 'text': 'Should the crawler ... I AM NOT QUITE SURE WHAT?'}), 'destination': 'local', 'type': EnsureBool(), }, 'datalad.crawl.init_direct': { 'ui': ('question', { 'title': 'Default annex repository mode', 'text': 'Should dataset be initialized in direct mode?'}), 'destination': 'global', }, 'datalad.crawl.pipeline.housekeeping': { 'ui': ('yesno', { 'title': 'Crawler pipeline house keeping', 'text': 'Should the crawler tidy up datasets (git gc, repack, clean)?'}), 'destination': 'global', 'type': EnsureBool(), }, 'datalad.externals.nda.dbserver': { 'ui': ('question', { 'title': 'NDA database server', 'text': 'Hostname of the database server'}), 'destination': 'global', }, 'datalad.locations.cache': { 'ui': ('question', { 'title': 'Cache directory', 'text': 'Where should datalad cache files?'}), 'destination': 'global', 'default': dirs.user_cache_dir, }, 'datalad.exc.str.tblimit': { 'ui': ('question', { 'title': 'This flag is used by the datalad extract_tb function which extracts and formats stack-traces. It caps the number of lines to DATALAD_EXC_STR_TBLIMIT of pre-processed entries from traceback.'}), }, 'datalad.tests.nonetwork': { 'ui': ('yesno', { 'title': 'Skips network tests completely if this flag is set Examples include test for s3, git_repositories, openfmri etc'}), 'type': EnsureBool(), }, 'datalad.tests.nonlo': { 'ui': ('question', { 'title': 'Specifies network interfaces to bring down/up for testing. Currently used by travis.'}), }, 'datalad.tests.noteardown': { 'ui': ('yesno', { 'title': 'Does not execute teardown_package which cleans up temp files and directories created by tests if this flag is set'}), 'type': EnsureBool(), }, 'datalad.tests.protocolremote': { 'ui': ('yesno', { 'title': 'Binary flag to specify whether to test protocol interactions of custom remote with annex'}), 'type': EnsureBool(), }, 'datalad.tests.runcmdline': { 'ui': ('yesno', { 'title': 'Binary flag to specify if shell testing using shunit2 to be carried out'}), 'type': EnsureBool(), }, 'datalad.tests.ssh': { 'ui': ('yesno', { 'title': 'Skips SSH tests if this flag is **not** set'}), 'type': EnsureBool(), }, 'datalad.tests.temp.dir': { 'ui': ('question', { 'title': 'Create a temporary directory at location specified by this flag. It is used by tests to create a temporary git directory while testing git annex archives etc'}), }, 'datalad.tests.temp.keep': { 'ui': ('yesno', { 'title': 'Function rmtemp will not remove temporary file/directory created for testing if this flag is set'}), 'type': EnsureBool(), }, 'datalad.tests.temp.fs': { 'ui': ('question', { 'title': 'Specify the temporary file system to use as loop device for testing DATALAD_TESTS_TEMP_DIR creation'}), }, 'datalad.tests.temp.fssize': { 'ui': ('question', { 'title': 'Specify the size of temporary file system to use as loop device for testing DATALAD_TESTS_TEMP_DIR creation'}), }, 'datalad.tests.usecassette': { 'ui': ('question', { 'title': 'Specifies the location of the file to record network transactions by the VCR module. Currently used by when testing custom special remotes'}), }, 'datalad.api.alwaysrender': { 'ui': ('yesno', { 'title': 'Python API output rendering', 'text': 'Should the high-level API functions be altered to render output like the command line interface would do, in order to make interactive use less cumbersome?'}), 'default': False, 'type': EnsureBool(), }, 'datalad.log.level': { 'ui': ('question', { 'title': 'Used for control the verbosity of logs printed to ' 'stdout while running datalad commands/debugging'}), }, 'datalad.log.outputs': { 'ui': ('question', { 'title': 'Used to control either both stdout and stderr of external commands execution are logged in detail (at DEBUG level)'}), }, 'datalad.log.timestamp': { 'ui': ('yesno', { 'title': 'Used to add timestamp to datalad logs'}), 'default': False, 'type': EnsureBool(), }, 'datalad.log.traceback': { 'ui': ('question', { 'title': 'Runs TraceBack function with collide set to True, if this flag is set to "collide". This replaces any common prefix between current traceback log and previous invocation with "..."'}), }, 'datalad.cmd.protocol': { 'ui': ('question', { 'title': 'Specifies the protocol number used by the Runner to note shell command or python function call times and allows for dry runs. "externals-time" for ExecutionTimeExternalsProtocol, "time" for ExecutionTimeProtocol and "null" for NullProtocol. Any new DATALAD_CMD_PROTOCOL has to implement datalad.support.protocol.ProtocolInterface'}), }, 'datalad.cmd.protocol.prefix': { 'ui': ('question', { 'title': 'Sets a prefix to add before the command call times are noted by DATALAD_CMD_PROTOCOL.'}), }, } datalad-0.4.1/datalad/interface/common_opts.py000066400000000000000000000124551301115270200213460ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Common interface options """ __docformat__ = 'restructuredtext' from datalad.support.param import Parameter from datalad.support.constraints import EnsureInt, EnsureNone, EnsureStr dataset_description = Parameter( args=("-D", "--description",), constraints=EnsureStr() | EnsureNone(), doc="""short description of this dataset instance that humans can use to identify the repository/location, e.g. "Precious data on my laptop.""") recursion_flag = Parameter( args=("-r", "--recursive",), action="store_true", doc="""if set, recurse into potential subdataset""") recursion_limit = Parameter( args=("--recursion-limit",), metavar="LEVELS", constraints=EnsureInt() | EnsureNone(), doc="""limit recursion into subdataset to the given number of levels""") shared_access_opt = Parameter( args=('--shared-access',), metavar='MODE', doc="""configure shared access to a dataset, see `git init --shared` documentation for complete details on the supported scenarios. Possible values include: 'false', 'true', 'group', and 'all'""") super_datasets_flag = Parameter( args=("-S", "--super-datasets",), action="store_true", doc="""if set, traverse and save stats corresponding to the change within super-datasets""") git_opts = Parameter( args=("--git-opts",), metavar='STRING', constraints=EnsureStr() | EnsureNone(), doc="""option string to be passed to :command:`git` calls""") git_clone_opts = Parameter( args=("--git-clone-opts",), metavar='STRING', constraints=EnsureStr() | EnsureNone(), doc="""option string to be passed to :command:`git clone` calls""") annex_opts = Parameter( args=("--annex-opts",), metavar='STRING', constraints=EnsureStr() | EnsureNone(), doc="""option string to be passed to :command:`git annex` calls""") annex_init_opts = Parameter( args=("--annex-init-opts",), metavar='STRING', constraints=EnsureStr() | EnsureNone(), doc="""option string to be passed to :command:`git annex init` calls""") annex_add_opts = Parameter( args=("--annex-add-opts",), metavar='STRING', constraints=EnsureStr() | EnsureNone(), doc="""option string to be passed to :command:`git annex add` calls""") annex_get_opts = Parameter( args=("--annex-get-opts",), metavar='STRING', constraints=EnsureStr() | EnsureNone(), doc="""option string to be passed to :command:`git annex get` calls""") annex_copy_opts = Parameter( args=("--annex-copy-opts",), metavar='STRING', constraints=EnsureStr() | EnsureNone(), doc="""option string to be passed to :command:`git annex copy` calls""") allow_dirty = Parameter( args=("--allow-dirty",), action="store_true", doc="""flag that operating on a dirty repository (uncommitted or untracked content) is ok""") if_dirty_opt = Parameter( args=("--if-dirty",), choices=('fail', 'save-before', 'ignore'), doc="""desired behavior if a dataset with unsaved changes is discovered: 'fail' will trigger an error and further processing is aborted; 'save-before' will save all changes prior any further action; 'ignore' let's datalad proceed as if the dataset would not have unsaved changes.""") nosave_opt = Parameter( args=("--nosave",), dest='save', action="store_false", doc="""by default all modifications to a dataset are immediately saved. Given this option will disable this behavior.""") reckless_opt = Parameter( args=("--reckless",), action="store_true", doc="""Set up the dataset to be able to obtain content in the cheapest/fastest possible way, even if this poses a potential risk the data integrity (e.g. hardlink files from a local clone of the dataset). Use with care, and limit to "read-only" use cases. With this flag the installed dataset will be marked as untrusted.""") jobs_opt = Parameter( args=("-J", "--jobs"), metavar="NJOBS", constraints=EnsureInt() | EnsureNone(), doc="""how many parallel jobs (where possible) to use.""") verbose = Parameter( args=("-v", "--verbose",), action="store_true", doc="""print out more detailed information while executing a command""") as_common_datasrc = Parameter( args=("--as-common-datasrc",), metavar='NAME', doc="""configure the created sibling as a common data source of the dataset that can be automatically used by all consumers of the dataset (technical: git-annex auto-enabled special remote)""") publish_depends = Parameter( args=("--publish-depends",), metavar='SIBLINGNAME', doc="""add a dependency such that the given exsiting sibling is always published prior to the new sibling. This equals setting a configuration item 'remote.SIBLINGNAME.datalad-publish-depends'. [PY: Multiple dependencies can be given as a list of sibling names PY][CMD: This option can be given more than once to configure multiple dependencies CMD]""", action='append') datalad-0.4.1/datalad/interface/crawl.py000066400000000000000000000173111301115270200201150ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Interface for crawling a webpage and push extracted data into a dataset""" __docformat__ = 'restructuredtext' from os.path import exists from .base import Interface from datalad.support.param import Parameter from datalad.support.constraints import EnsureStr, EnsureNone from datalad.crawler.pipeline import initiate_pipeline_config from datalad.support.stats import ActivityStats from datalad import utils from logging import getLogger lgr = getLogger('datalad.api.crawl') from .. import cfg class Crawl(Interface): """Crawl online resource to create or update a dataset. Examples: $ datalad crawl # within a dataset having .datalad/crawl/crawl.cfg """ _params_ = dict( # Dry run is untested and largely probably not working in this implementation # so let's not expose it for now at all # dry_run=Parameter( # args=("-n", "--dry-run"), # action="store_true", # doc="""flag if file manipulations to be invoked (e.g., adding to git/annex). # If not, commands are only printed to the stdout"""), is_pipeline=Parameter( args=("--is-pipeline",), action="store_true", doc="""flag if provided file is a Python script which defines pipeline()"""), is_template=Parameter( args=("-t", "--is-template"), action="store_true", doc="""flag if provided value is the name of the template to use"""), recursive=Parameter( args=("-r", "--recursive"), action="store_true", doc="""flag to crawl subdatasets as well (for now serially)"""), chdir=Parameter( args=("-C", "--chdir"), constraints=EnsureStr() | EnsureNone(), doc="""directory to chdir to for crawling"""), path=Parameter( args=('path',), metavar='file', nargs='?', constraints=EnsureStr() | EnsureNone(), doc="""configuration (or pipeline if --is-pipeline) file defining crawling, or a directory of a dataset on which to perform crawling using its standard crawling specification"""), ) @staticmethod def __call__(path=None, is_pipeline=False, is_template=False, recursive=False, chdir=None): # dry_run=False, dry_run = False from datalad.crawler.pipeline import ( load_pipeline_from_config, load_pipeline_from_module, get_repo_pipeline_config_path, get_repo_pipeline_script_path ) from datalad.crawler.pipeline import run_pipeline from datalad.utils import chpwd # import late so we could mock during tests with chpwd(chdir): assert not (is_pipeline and is_template), "it is either a pipeline or a template name, can't be both" if is_template: # generate a config and overload path with its filename path = initiate_pipeline_config(template=path, # kwargs=TODO, commit=True) # TODO: centralize via _params_ handling if dry_run: dryrun_optlabel = 'datalad.crawl.dryrun' if dryrun_optlabel in cfg: cfg.unset(dryrun_optlabel, where='local', reload=False) cfg.add(dryrun_optlabel, "True", where='local') if path is None: # get config from the current repository/dataset if is_pipeline: raise ValueError("You must specify the file if --pipeline") # Let's see if there is a config or pipeline in this repo path = get_repo_pipeline_config_path() if not path or not exists(path): # Check if there may be the pipeline provided path = get_repo_pipeline_script_path() if path and exists(path): is_pipeline = True stats = ActivityStats() if not path: raise RuntimeError("Cannot locate crawler config or pipeline file") if is_pipeline: lgr.info("Loading pipeline definition from %s" % path) pipeline = load_pipeline_from_module(path) else: lgr.info("Loading pipeline specification from %s" % path) pipeline = load_pipeline_from_config(path) lgr.info("Running pipeline %s" % str(pipeline)) # TODO: capture the state of all branches so in case of crash # we could gracefully reset back try: output = run_pipeline(pipeline, stats=stats) except Exception as exc: # TODO: config.crawl.failure = full-reset | last-good-master # probably ask via ui which action should be performed unless # explicitly specified raise stats.datasets_crawled += 1 # TODO: Move gc/clean over here! stats_total = stats.get_total() if recursive: # get all subdatasets, and crawl them too! ## ? assert path_orig is None, "Otherwise not sure what to do with path=%r in subdatasets" % path import os from ..distribution.dataset import Dataset from ..api import crawl from ..utils import swallow_logs from ..dochelpers import exc_str # Note: we could collect all datasets to be crawled here or pass recursive=True # into the subdatasets' crawl. We will collect all of them here so we might later # also introduce automatic commits when super-dataset got successfully updated subdatasets = Dataset(os.curdir).get_subdatasets(recursive=recursive) lgr.info("Crawling %d subdatasets", len(subdatasets)) output = [output] # TODO: parallelize # TODO: assumes that all sub-datasets are 'crawllable', and if not # just adds them to crawl_failed count. But may be we should make it more # explicit, that some sub-datasets might not need to be crawled, so they get # skipped explicitly? for ds_ in subdatasets: ds_logfile = utils.get_logfilename(ds_, 'crawl') try: # TODO: might be cool to be able to report a 'heart beat' from the swallow into pbar or smth with swallow_logs(file_=ds_logfile) as cml: output_, stats_ = crawl(chdir=ds_) stats_total += stats_ output.append(output_) lgr.info("Crawled %s: %s (log: %s)", ds_, stats_.as_str(mode='line'), ds_logfile) except Exception as exc: stats_total.datasets_crawl_failed += 1 stats_total.datasets_crawled += 1 output += [None] lgr.warning("Crawling of %s has failed (more in %s): %s.", # Log output: %s", ds_, ds_logfile, exc_str(exc)) # , cml.out) lgr.info("Total stats: %s", stats_total.as_str(mode='line')) return output, stats_total datalad-0.4.1/datalad/interface/crawl_init.py000066400000000000000000000072661301115270200211500ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Interface for a generic template in which arguments are specified by the user""" __docformat__ = 'restructuredtext' from os.path import curdir from .base import Interface from collections import OrderedDict from datalad.distribution.dataset import Dataset from ..dochelpers import exc_str from ..support.param import Parameter from ..support.constraints import EnsureStr, EnsureNone from ..utils import get_func_kwargs_doc from ..crawler.pipeline import load_pipeline_from_template, initiate_pipeline_config from logging import getLogger lgr = getLogger('datalad.api.crawl_init') CRAWLER_PIPELINE_SECTION = 'crawl:pipeline' class CrawlInit(Interface): """Initialize crawling configuration Allows to specify template and function to generate a crawling pipeline Examples: $ datalad crawl-init \ --template openfmri \ --template-func superdataset_pipeline $ datalad crawl-init \ --template fcptable \ dataset=Baltimore tarballs=True """ _params_ = dict( template=Parameter( args=("-t", "--template"), action="store", constraints=EnsureStr() | EnsureNone(), doc="""the name of the template"""), template_func=Parameter( args=("-f", "--template-func"), action="store", doc="""the name of the function"""), args=Parameter( args=("args",), nargs="*", constraints=EnsureStr() | EnsureNone(), doc="""keyword arguments to pass into the template function generating actual pipeline, organized in [PY: a dict PY][CMD: key=value pairs CMD]"""), save=Parameter( args=("--save",), action="store_true", doc="""flag to save file into git repo"""), ) @staticmethod def __call__(args=None, template=None, template_func=None, save=False): if args: if isinstance(args, str): args = [args] if isinstance(args, list): args = OrderedDict(map(str, it.split('=', 1)) for it in args) elif isinstance(args, dict): pass else: raise ValueError( "args entered must be given in a list or dict, were given as %s", type(args)) elif not template: raise TypeError("crawl-init needs a template") else: args = {} pipeline_func = load_pipeline_from_template(template, template_func, kwargs=args, return_only=True) try: pipeline = pipeline_func(**args) except Exception as exc: raise RuntimeError( "Running the pipeline function resulted in %s." "FYI this pipeline only takes the following args: %s" % (exc_str(exc), get_func_kwargs_doc(pipeline_func))) if not pipeline: raise ValueError("returned pipeline is empty") if not isinstance(pipeline, list): raise ValueError("pipeline should be represented as a list. Got: %r" % pipeline) configfile = initiate_pipeline_config(template, template_func, args) if save: from datalad.api import save ds = Dataset(curdir) ds.save("committing crawl config file", files=[configfile]) datalad-0.4.1/datalad/interface/download_url.py000066400000000000000000000074131301115270200215000ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Interface to DataLad downloaders """ # TODO: or may be it should be converted to 'addurl' thus with option for # adding into annex or git, depending on the largefiles option, and # --download-only to only download... I see myself using it in other projects # as well I think. __docformat__ = 'restructuredtext' from os.path import isdir, curdir from .base import Interface from ..ui import ui from ..utils import assure_list_from_str from ..dochelpers import exc_str from ..support.param import Parameter from ..support.constraints import EnsureStr, EnsureNone from logging import getLogger lgr = getLogger('datalad.api.download-url') class DownloadURL(Interface): """Download content It allows for a uniform download interface to various supported URL schemes, re-using or asking for authentication detail maintained by datalad. Examples: $ datalad download http://example.com/file.dat s3://bucket/file2.dat """ _params_ = dict( urls=Parameter( doc="URL(s) to be downloaded", constraints=EnsureStr(), # TODO: EnsureURL metavar='url', nargs='+'), overwrite=Parameter( args=("-o", "--overwrite"), action="store_true", doc="""flag to overwrite it if target file exists"""), stop_on_failure=Parameter( args=("-x", "--stop-on-failure"), action="store_true", doc="""flag to stop subsequent downloads upon first failure to download"""), path=Parameter( args=("-O", "--path"), doc="path (filename or directory path) where to store downloaded file(s). " "In case of multiple URLs provided, must point to a directory. Otherwise current " "directory is used", constraints=EnsureStr() | EnsureNone()) ) @staticmethod def __call__(urls, path=None, overwrite=False, stop_on_failure=False): """ Returns ------- list of str downloaded successfully files """ from ..downloaders.providers import Providers urls = assure_list_from_str(urls) if len(urls) > 1: if path: if not(isdir(path)): raise ValueError( "When specifying multiple urls, --path should point to " "an existing directory. Got %r" % path) if not path: path = curdir # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress # in % of urls which were already downloaded providers = Providers.from_config_files() downloaded_paths, failed_urls = [], [] for url in urls: # somewhat "ugly" # providers.get_provider(url).get_downloader(url).download(url, path=path) # for now -- via sugaring try: downloaded_path = providers.download(url, path=path, overwrite=overwrite) downloaded_paths.append(downloaded_path) # ui.message("%s -> %s" % (url, downloaded_path)) except Exception as e: failed_urls.append(url) ui.error(exc_str(e)) if stop_on_failure: break if failed_urls: raise RuntimeError("%d url(s) failed to download" % len(failed_urls)) return downloaded_paths datalad-0.4.1/datalad/interface/ls.py000066400000000000000000001014371301115270200174260ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil; coding: utf-8 -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Helper utility to list things. ATM list content of S3 bucket """ __docformat__ = 'restructuredtext' import sys import time from os.path import exists, lexists, join as opj, abspath, isabs, getmtime from os.path import curdir, isfile, islink, isdir, dirname, basename, split, realpath from os import listdir, lstat, remove, makedirs import json as js import hashlib from six.moves.urllib.request import urlopen, Request from six.moves.urllib.error import HTTPError from ..utils import auto_repr from .base import Interface from ..ui import ui from ..utils import swallow_logs from ..consts import METADATA_DIR from ..consts import METADATA_FILENAME from ..dochelpers import exc_str from ..support.param import Parameter from ..support import ansi_colors from ..support.constraints import EnsureStr, EnsureNone from ..distribution.dataset import Dataset from datalad.support.annexrepo import AnnexRepo from datalad.support.annexrepo import GitRepo import string import humanize from datalad.utils import is_interactive from logging import getLogger lgr = getLogger('datalad.api.ls') class Ls(Interface): """List summary information about URLs and dataset(s) ATM only s3:// URLs and datasets are supported Examples -------- $ datalad ls s3://openfmri/tarballs/ds202 # to list S3 bucket $ datalad ls # to list current dataset """ _params_ = dict( loc=Parameter( doc="URL or path to list, e.g. s3://...", metavar='PATH/URL', nargs="*", constraints=EnsureStr() | EnsureNone(), ), recursive=Parameter( args=("-r", "--recursive"), action="store_true", doc="recurse into subdirectories", ), fast=Parameter( args=("-F", "--fast"), action="store_true", doc="only perform fast operations. Would be overridden by --all", ), all=Parameter( args=("-a", "--all"), action="store_true", doc="list all (versions of) entries, not e.g. only latest entries " "in case of S3", ), long=Parameter( args=("-L", "--long"), action="store_true", doc="list more information on entries (e.g. acl, urls in s3, annex " "sizes etc)", ), config_file=Parameter( doc="""path to config file which could help the 'ls'. E.g. for s3:// URLs could be some ~/.s3cfg file which would provide credentials""", constraints=EnsureStr() | EnsureNone() ), list_content=Parameter( choices=(None, 'first10', 'md5', 'full'), doc="""list also the content or only first 10 bytes (first10), or md5 checksum of an entry. Might require expensive transfer and dump binary output to your screen. Do not enable unless you know what you are after""", default=None ), json=Parameter( choices=('file', 'display', 'delete'), doc="""metadata json of dataset for creating web user interface. display: prints jsons to stdout or file: writes each subdir metadata to json file in subdir of dataset or delete: deletes all metadata json files in dataset""", ), ) @staticmethod def __call__(loc, recursive=False, fast=False, all=False, long=False, config_file=None, list_content=False, json=None): if isinstance(loc, list) and not len(loc): # nothing given, CWD assumed -- just like regular ls loc = '.' kw = dict(fast=fast, recursive=recursive, all=all, long=long) if isinstance(loc, list): return [Ls.__call__(loc_, config_file=config_file, list_content=list_content, json=json, **kw) for loc_ in loc] # TODO: do some clever handling of kwargs as to remember what were defaults # and what any particular implementation actually needs, and then issuing # warning if some custom value/option was specified which doesn't apply to the # given url # rename to not angry Python gods who took all good words kw['all_'] = kw.pop('all') kw['long_'] = kw.pop('long') if loc.startswith('s3://'): return _ls_s3(loc, config_file=config_file, list_content=list_content, **kw) elif lexists(loc) and Dataset(loc).is_installed(): return _ls_json(loc, json=json, **kw) if json else _ls_dataset(loc, **kw) else: #raise ValueError("ATM supporting only s3:// URLs and paths to local datasets") # TODO: unify all the output here -- _ls functions should just return something # to be displayed ui.message( "{} {}".format( ansi_colors.color_word(loc, ansi_colors.DATASET), ansi_colors.color_word("unknown", ansi_colors.RED) ) ) # # Dataset listing # @auto_repr class AbsentRepoModel(object): """Just a base for those where repo wasn't installed yet""" def __init__(self, path): self.path = path self.repo = None @property def type(self): return "N/A" @auto_repr class GitModel(object): """A base class for models which have some .repo available""" __slots__ = ['_branch', 'repo', '_path'] def __init__(self, repo): self.repo = repo # lazy evaluation variables self._branch = None self._path = None @property def path(self): return self.repo.path if self._path is None else self._path @path.setter def path(self, v): self._path = v @property def branch(self): if self._branch is None: try: self._branch = self.repo.get_active_branch() except: # MIH: InvalidGitRepositoryError? return None return self._branch @property def clean(self): return not self.repo.dirty @property def describe(self): try: with swallow_logs(): describe, outerr = self.repo._git_custom_command([], ['git', 'describe', '--tags']) return describe.strip() except: return None @property def date(self): """Date of the last commit """ return self.repo.get_committed_date() @property def count_objects(self): return self.repo.count_objects @property def git_local_size(self): count_objects = self.count_objects return count_objects['size'] if count_objects else None @property def type(self): return {False: 'git', True: 'annex'}[isinstance(self.repo, AnnexRepo)] @auto_repr class AnnexModel(GitModel): __slots__ = ['_info'] + GitModel.__slots__ def __init__(self, *args, **kwargs): super(AnnexModel, self).__init__(*args, **kwargs) self._info = None @property def info(self): if self._info is None and self.type == 'annex': self._info = self.repo.repo_info() return self._info @property def annex_worktree_size(self): info = self.info return info['size of annexed files in working tree'] if info else 0.0 @property def annex_local_size(self): info = self.info return info['local annex size'] if info else 0.0 @auto_repr class FsModel(AnnexModel): __slots__ = AnnexModel.__slots__ def __init__(self, path, *args, **kwargs): super(FsModel, self).__init__(*args, **kwargs) self._path = path @property def path(self): return self._path @property def symlink(self): """if symlink returns path the symlink points to else returns None""" if islink(self._path): # if symlink target_path = realpath(self._path) # find link target # convert to absolute path if not return target_path if exists(target_path) else None return None @property def date(self): """Date of last modification""" if self.type_ is not ['git', 'annex']: return lstat(self._path).st_mtime else: super(self.__class__, self).date @property def size(self): """Size of the node computed based on its type""" type_ = self.type_ sizes = {'total': 0.0, 'ondisk': 0.0, 'git': 0.0, 'annex': 0.0, 'annex_worktree': 0.0} if type_ in ['file', 'link', 'link-broken']: # if node is under annex, ask annex for node size, ondisk_size if isinstance(self.repo, AnnexRepo) and self.repo.is_under_annex(self._path): size = self.repo.info(self._path, batch=True)['size'] ondisk_size = size \ if self.repo.file_has_content(self._path) \ else 0 # else ask fs for node size (= ondisk_size) else: size = ondisk_size = 0 \ if type_ == 'link-broken' \ else lstat(self.symlink or self._path).st_size sizes.update({'total': size, 'ondisk': ondisk_size}) if self.repo.path == self._path: sizes.update({'git': self.git_local_size, 'annex': self.annex_local_size, 'annex_worktree': self.annex_worktree_size}) return sizes @property def type_(self): """outputs the node type Types: link, link-broken, file, dir, annex-repo, git-repo""" if islink(self.path): return 'link' if self.symlink else 'link-broken' elif isfile(self.path): return 'file' elif exists(opj(self.path, ".git", "annex")): return 'annex' elif exists(opj(self.path, ".git")): return 'git' elif isdir(self.path): return 'dir' else: return None class LsFormatter(string.Formatter): # condition by interactive if is_interactive(): BLUE = ansi_colors.COLOR_SEQ % ansi_colors.BLUE RED = ansi_colors.COLOR_SEQ % ansi_colors.RED GREEN = ansi_colors.COLOR_SEQ % ansi_colors.GREEN RESET = ansi_colors.RESET_SEQ DATASET = ansi_colors.COLOR_SEQ % ansi_colors.UNDERLINE else: BLUE = RED = GREEN = RESET = DATASET = u"" # http://stackoverflow.com/questions/9932406/unicodeencodeerror-only-when-running-as-a-cron-job # reveals that Python uses ascii encoding when stdout is a pipe, so we shouldn't force it to be # unicode then # TODO: we might want to just ignore and force utf8 while explicitly .encode()'ing output! if sys.getdefaultencoding() == 'ascii': OK = 'OK' # u"✓" NOK = 'X' # u"✗" NONE = '-' # u"✗" else: # unicode versions which look better but which blow during tests etc OK = u"✓" NOK = u"✗" NONE = u"✗" def convert_field(self, value, conversion): #print("%r->%r" % (value, conversion)) if conversion == 'D': # Date if value is not None: return time.strftime(u"%Y-%m-%d/%H:%M:%S", time.localtime(value)) else: return u'-' elif conversion == 'S': # Human size #return value if value is not None: return humanize.naturalsize(value) else: return u'-' elif conversion == 'X': # colored bool chr, col = (self.OK, self.GREEN) if value else (self.NOK, self.RED) return u"%s%s%s" % (col, chr, self.RESET) elif conversion == 'N': # colored Red - if None if value is None: # return "%s✖%s" % (self.RED, self.RESET) return u"%s%s%s" % (self.RED, self.NONE, self.RESET) return value elif conversion in {'B', 'R', 'U'}: return u"%s%s%s" % ({'B': self.BLUE, 'R': self.RED, 'U': self.DATASET}[conversion], value, self.RESET) return super(LsFormatter, self).convert_field(value, conversion) def format_field(self, value, format_spec): # TODO: move all the "coloring" into formatting, so we could correctly indent # given the format and only then color it up # print "> %r, %r" % (value, format_spec) return super(LsFormatter, self).format_field(value, format_spec) def format_ds_model(formatter, ds_model, format_str, format_exc): try: #print("WORKING ON %s" % ds_model.path) if not exists(ds_model.path) or not ds_model.repo: return formatter.format(format_exc, ds=ds_model, msg=u"not installed") ds_formatted = formatter.format(format_str, ds=ds_model) #print("FINISHED ON %s" % ds_model.path) return ds_formatted except Exception as exc: return formatter.format(format_exc, ds=ds_model, msg=exc_str(exc)) # from joblib import Parallel, delayed def _ls_dataset(loc, fast=False, recursive=False, all_=False, long_=False): isabs_loc = isabs(loc) topdir = '' if isabs_loc else abspath(curdir) topds = Dataset(loc) dss = [topds] + ( [Dataset(opj(loc, sm)) for sm in topds.get_subdatasets(recursive=recursive)] if recursive else []) dsms = [] for ds in dss: if not ds.is_installed(): dsm = AbsentRepoModel(ds.path) elif isinstance(ds.repo, AnnexRepo): dsm = AnnexModel(ds.repo) elif isinstance(ds.repo, GitRepo): dsm = GitModel(ds.repo) else: raise RuntimeError("Got some dataset which don't know how to handle %s" % ds) dsms.append(dsm) # adjust path strings for ds_model in dsms: path = ds_model.path[len(topdir) + 1 if topdir else 0:] if not path: path = '.' ds_model.path = path dsms = sorted(dsms, key=lambda m: m.path) maxpath = max(len(ds_model.path) for ds_model in dsms) path_fmt = u"{ds.path!U:<%d}" % (maxpath + (11 if is_interactive() else 0)) # + to accommodate ansi codes pathtype_fmt = path_fmt + u" [{ds.type}]" full_fmt = pathtype_fmt + u" {ds.branch!N} {ds.describe!N} {ds.date!D}" if (not fast) or long_: full_fmt += u" {ds.clean!X}" fmts = { AbsentRepoModel: pathtype_fmt, GitModel: full_fmt, AnnexModel: full_fmt } if long_: fmts[AnnexModel] += u" {ds.annex_local_size!S}/{ds.annex_worktree_size!S}" formatter = LsFormatter() # weird problems happen in the parallel run -- TODO - figure it out # for out in Parallel(n_jobs=1)( # delayed(format_ds_model)(formatter, dsm, full_fmt, format_exc=path_fmt + " {msg!R}") # for dsm in dss): # print(out) for dsm in dsms: fmt = fmts[dsm.__class__] ds_str = format_ds_model(formatter, dsm, fmt, format_exc=path_fmt + u" {msg!R}") print(ds_str) def machinesize(humansize): """convert human-size string to machine-size""" try: size_str, size_unit = humansize.split(" ") except AttributeError: return float(humansize) unit_converter = {'Byte': 0, 'Bytes': 0, 'kB': 1, 'MB': 2, 'GB': 3, 'TB': 4, 'PB': 5} machinesize = float(size_str) * (1000 ** unit_converter[size_unit]) return machinesize def leaf_name(path): """takes a relative or absolute path and returns name of node at that location""" head, tail = split(abspath(path)) return tail or basename(head) def ignored(path, only_hidden=False): """if path is in the ignorelist return True ignore list includes hidden files and git or annex maintained folders when only_hidden set, only ignores hidden files and folders not git or annex maintained folders """ if isdir(opj(path, ".git")) and not only_hidden: return True return '.' == leaf_name(path)[0] or leaf_name(path) == 'index.html' def metadata_locator(fs_metadata=None, path=None, ds_path=None, metadata_path=None): """path to metadata file of node associated with the fs_metadata dictionary Parameters ---------- fs_metadata: dict Metadata json of a node path: str Path to directory of metadata to be rendered ds_path: str Path to dataset root metadata_path: str Path to metadata root. Calculated relative to ds_path Returns ------- str path to metadata of current node """ # use implicit paths unless paths explicitly specified ds_path = ds_path or fs_metadata['repo'] path = path or fs_metadata['path'] metadata_path = metadata_path or '.git/datalad/metadata' # directory metadata directory tree location metadata_dir = opj(ds_path, metadata_path) # relative path of current directory wrt dataset root dir_path = path.split(ds_path)[1][1:] or '/' # create md5 hash of current directory's relative path metadata_hash = hashlib.md5(dir_path.encode('utf-8')).hexdigest() # construct final path to metadata file metadata_file = opj(metadata_dir, metadata_hash) return metadata_file def fs_extract(nodepath, repo): """extract required info of nodepath with its associated parent repository and returns it as a dictionary Parameters ---------- nodepath : str Full path to the location we are exploring (must be a directory within `repo` repo : GitRepo Is the repository nodepath belongs to """ # Create FsModel from filesystem nodepath and its associated parent repository node = FsModel(nodepath, repo) pretty_size = {stype: humanize.naturalsize(svalue) for stype, svalue in node.size.items()} pretty_date = time.strftime(u"%Y-%m-%d %H:%M:%S", time.localtime(node.date)) name = leaf_name(node._path) if leaf_name(node._path) != "" else leaf_name(node.repo.path) rec = { "name": name, "path": node._path, "repo": node.repo.path, "type": node.type_, "size": pretty_size, "date": pretty_date, } # if there is meta-data for the dataset (done by aggregate-metadata) # we include it metadata_path = opj(nodepath, METADATA_DIR, METADATA_FILENAME) if exists(metadata_path): rec["metadata"] = metadata = js.load(open(metadata_path)) return rec def fs_render(fs_metadata, json=None, **kwargs): """takes node to render and based on json option passed renders to file, stdout or deletes json at root Parameters ---------- fs_metadata: dict Metadata json to be rendered json: str ('file', 'display', 'delete') Render to file, stdout or delete json """ metadata_file = metadata_locator(fs_metadata, **kwargs) if json == 'file': # create metadata_root directory if it doesn't exist metadata_dir = dirname(metadata_file) if not exists(metadata_dir): makedirs(metadata_dir) # write directory metadata to json with open(metadata_file, 'w') as f: js.dump(fs_metadata, f) # else if json flag set to delete, remove .dir.json of current directory elif json == 'delete' and exists(metadata_file): remove(metadata_file) # else dump json to stdout elif json == 'display': print(js.dumps(fs_metadata) + '\n') def fs_traverse(path, repo, parent=None, render=True, recursive=False, json=None): """Traverse path through its nodes and returns a dictionary of relevant attributes attached to each node Parameters ---------- path: str Path to the directory to be traversed repo: AnnexRepo or GitRepo Repo object the directory belongs too parent: dict Extracted info about parent directory recursive: bool Recurse into subdirectories (note that subdatasets are not traversed) render: bool To render from within function or not. Set to false if results to be manipulated before final render Returns ------- list of dict extracts and returns a (recursive) list of directory info at path does not traverse into annex, git or hidden directories """ fs = fs_extract(path, repo) if isdir(path): # if node is a directory children = [fs.copy()] # store its info in its children dict too (Yarik is not sure why, but I guess for .?) # ATM seems some pieces still rely on having this duplication, so left as is # TODO: strip away for node in listdir(path): nodepath = opj(path, node) # TODO: it might be a subdir which is non-initialized submodule! # if not ignored, append child node info to current nodes dictionary if not ignored(nodepath): # if recursive, create info dictionary of each child node too if recursive: subdir = fs_traverse(nodepath, repo, parent=None, # children[0], recursive=recursive, json=json) else: # read child metadata from its metadata file if it exists subdir_json = metadata_locator(path=nodepath, ds_path=fs["repo"]) if exists(subdir_json): with open(subdir_json) as data_file: subdir = js.load(data_file) subdir.pop('nodes', None) # else extract whatever information you can about the child else: subdir = fs_extract(nodepath, repo) # append child metadata to list children.extend([subdir]) # sum sizes of all 1st level children children_size = {} for node in children[1:]: for size_type, child_size in node['size'].items(): children_size[size_type] = children_size.get(size_type, 0) + machinesize(child_size) # update current node sizes to the humanized aggregate children size fs['size'] = children[0]['size'] = \ {size_type: humanize.naturalsize(child_size) for size_type, child_size in children_size.items()} children[0]['name'] = '.' # replace current node name with '.' to emulate unix syntax if parent: parent['name'] = '..' # replace parent node name with '..' to emulate unix syntax children.insert(1, parent) # insert parent info after current node info in children dict fs['nodes'] = children # add children info to main fs dictionary if render: # render directory node at location(path) fs_render(fs, json=json) lgr.info('Directory: %s' % path) return fs def ds_traverse(rootds, parent=None, json=None, recursive=False, all_=False, long_=False): """Hierarchical dataset traverser Parameters ---------- rootds: Dataset Root dataset to be traversed parent: Dataset Parent dataset of the current rootds recursive: bool Recurse into subdirectories of the current dataset all_: bool Recurse into subdatasets of the root dataset Returns ------- list of dict extracts and returns a (recursive) list of dataset(s) info at path """ # extract parent info to pass to traverser fsparent = fs_extract(parent.path, parent.repo) if parent else None # (recursively) traverse file tree of current dataset fs = fs_traverse(rootds.path, rootds.repo, render=False, parent=fsparent, recursive=all_, json=json) size_list = [fs['size']] # (recursively) traverse each subdataset children = [] for subds_path in rootds.get_subdatasets(): subds_path = opj(rootds.path, subds_path) subds = Dataset(subds_path) subds_json = metadata_locator(path=subds_path, ds_path=subds_path) def handle_not_installed(): # for now just traverse as fs lgr.warning("%s is either not installed or lacks meta-data", subds) subfs = fs_extract(subds_path, rootds) # but add a custom type that it is a not installed subds subfs['type'] = 'uninitialized' # we need to kick it out from 'children' # TODO: this is inefficient and cruel -- "ignored" should be made # smarted to ignore submodules for the repo if fs['nodes']: fs['nodes'] = [c for c in fs['nodes'] if c['path'] != subds_path] return subfs if not subds.is_installed(): subfs = handle_not_installed() elif recursive: subfs = ds_traverse(subds, json=json, recursive=recursive, all_=all_, parent=rootds) subfs.pop('nodes', None) size_list.append(subfs['size']) # else just pick the data from metadata_file of each subdataset else: lgr.info(subds_path) if exists(subds_json): with open(subds_json) as data_file: subfs = js.load(data_file) subfs.pop('nodes', None) size_list.append(subfs['size']) else: # the same drill as if not installed lgr.warning("%s is installed but no meta-data yet", subds) subfs = handle_not_installed() children.extend([subfs]) # sum sizes of all 1st level children dataset children_size = {} for subdataset_size in size_list: for size_type, subds_size in subdataset_size.items(): children_size[size_type] = children_size.get(size_type, 0) + machinesize(subds_size) # update current dataset sizes to the humanized aggregate subdataset sizes fs['size'] = {size_type: humanize.naturalsize(size) for size_type, size in children_size.items()} fs['nodes'][0]['size'] = fs['size'] # update self's updated size in nodes sublist too! # add dataset specific entries to its dict rootds_model = GitModel(rootds.repo) fs['tags'] = rootds_model.describe fs['branch'] = rootds_model.branch index_file = opj(rootds.path, '.git', 'index') fs['index-mtime'] = time.strftime( u"%Y-%m-%d %H:%M:%S", time.localtime(getmtime(index_file))) if exists(index_file) else '' # append children datasets info to current dataset fs['nodes'].extend(children) # render current dataset lgr.info('Dataset: %s' % rootds.path) fs_render(fs, json=json) return fs def _ls_json(loc, fast=False, **kwargs): # hierarchically traverse file tree of (sub-)dataset(s) under path passed(loc) return ds_traverse(Dataset(loc), parent=None, **kwargs) # # S3 listing # def _ls_s3(loc, fast=False, recursive=False, all_=False, long_=False, config_file=None, list_content=False): """List S3 bucket content""" if loc.startswith('s3://'): bucket_prefix = loc[5:] else: raise ValueError("passed location should be an s3:// url") import boto from hashlib import md5 from boto.s3.key import Key from boto.s3.prefix import Prefix from boto.exception import S3ResponseError from ..support.configparserinc import SafeConfigParser # provides PY2,3 imports if '/' in bucket_prefix: bucket_name, prefix = bucket_prefix.split('/', 1) else: bucket_name, prefix = bucket_prefix, None if prefix and '?' in prefix: ui.message("We do not care about URL options ATM, they get stripped") prefix = prefix[:prefix.index('?')] ui.message("Connecting to bucket: %s" % bucket_name) if config_file: config = SafeConfigParser() config.read(config_file) access_key = config.get('default', 'access_key') secret_key = config.get('default', 'secret_key') # TODO: remove duplication -- reuse logic within downloaders/s3.py to get connected conn = boto.connect_s3(access_key, secret_key) try: bucket = conn.get_bucket(bucket_name) except S3ResponseError as e: ui.message("E: Cannot access bucket %s by name" % bucket_name) all_buckets = conn.get_all_buckets() all_bucket_names = [b.name for b in all_buckets] ui.message("I: Found following buckets %s" % ', '.join(all_bucket_names)) if bucket_name in all_bucket_names: bucket = all_buckets[all_bucket_names.index(bucket_name)] else: raise RuntimeError("E: no bucket named %s thus exiting" % bucket_name) else: # TODO: expose credentials # We don't need any provider here really but only credentials from datalad.downloaders.providers import Providers providers = Providers.from_config_files() provider = providers.get_provider(loc) if not provider: raise ValueError( "Don't know how to deal with this url %s -- no provider defined for %s. " "Define a new provider (DOCS: TODO) or specify just s3cmd config file instead for now." % loc ) downloader = provider.get_downloader(loc) # should authenticate etc, and when ready we will ask for a bucket ;) bucket = downloader.access(lambda url: downloader.bucket, loc) info = [] for iname, imeth in [ ("Versioning", bucket.get_versioning_status), (" Website", bucket.get_website_endpoint), (" ACL", bucket.get_acl), ]: try: ival = imeth() except Exception as e: ival = str(e).split('\n')[0] info.append(" {iname}: {ival}".format(**locals())) ui.message("Bucket info:\n %s" % '\n '.join(info)) kwargs = {} if recursive else {'delimiter': '/'} ACCESS_METHODS = [ bucket.list_versions, bucket.list ] prefix_all_versions = None for acc in ACCESS_METHODS: try: prefix_all_versions = list(acc(prefix, **kwargs)) break except Exception as exc: lgr.debug("Failed to access via %s: %s", acc, exc_str(exc)) if not prefix_all_versions: ui.error("No output was provided for prefix %r" % prefix) else: max_length = max((len(e.name) for e in prefix_all_versions)) max_size_length = max((len(str(getattr(e, 'size', 0))) for e in prefix_all_versions)) results = [] for e in prefix_all_versions: results.append(e) if isinstance(e, Prefix): ui.message("%s" % (e.name, ),) continue base_msg = ("%%-%ds %%s" % max_length) % (e.name, e.last_modified) if isinstance(e, Key): if not (e.is_latest or all_): # Skip this one continue ui.message(base_msg + " %%%dd" % max_size_length % e.size, cr=' ') # OPT: delayed import from ..support.s3 import get_key_url url = get_key_url(e, schema='http') try: _ = urlopen(Request(url)) urlok = "OK" except HTTPError as err: urlok = "E: %s" % err.code try: acl = e.get_acl() except S3ResponseError as err: acl = err.message content = "" if list_content: # IO intensive, make an option finally! try: # _ = e.next()[:5] if we are able to fetch the content kwargs = dict(version_id=e.version_id) if list_content in {'full', 'first10'}: if list_content in 'first10': kwargs['headers'] = {'Range': 'bytes=0-9'} content = repr(e.get_contents_as_string(**kwargs)) elif list_content == 'md5': digest = md5() digest.update(e.get_contents_as_string(**kwargs)) content = digest.hexdigest() else: raise ValueError(list_content) # content = "[S3: OK]" except S3ResponseError as err: content = err.message finally: content = " " + content if long_: ui.message("ver:%-32s acl:%s %s [%s]%s" % (e.version_id, acl, url, urlok, content)) else: ui.message('') else: ui.message(base_msg + " " + str(type(e)).split('.')[-1].rstrip("\"'>")) return resultsdatalad-0.4.1/datalad/interface/save.py000066400000000000000000000354471301115270200177550ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """For now just a wrapper for Dataset.save() """ __docformat__ = 'restructuredtext' import logging import os from os.path import join as opj, isdir, realpath, relpath, pardir from os.path import isabs from datalad.support.constraints import EnsureStr from datalad.support.constraints import EnsureNone from datalad.support.param import Parameter from datalad.distribution.dataset import Dataset from datalad.distribution.dataset import EnsureDataset from datalad.distribution.dataset import datasetmethod from datalad.distribution.dataset import require_dataset from datalad.distribution.dataset import resolve_path from datalad.distribution.utils import _install_subds_inplace from datalad.interface.common_opts import recursion_limit, recursion_flag from datalad.interface.common_opts import super_datasets_flag from datalad.utils import assure_list from datalad.utils import with_pathsep as _with_sep from .base import Interface lgr = logging.getLogger('datalad.interface.commit') def untracked_subdatasets_to_submodules(ds, consider_paths): # treat special case of still untracked subdatasets. # those need to become submodules now, as they are otherwise added # without an entry in .gitmodules, and subsequently break Git's # submodule functionality completely new_modules = [] if not consider_paths: # nothing to test return new_modules for utf in ds.repo.repo.untracked_files: utf_abspath = opj(ds.path, utf) if not isdir(utf_abspath): # this cannot be a repository continue # test whether the potential submodule is scheduled for saving utf_realpath = realpath(utf_abspath) if any([utf_realpath.startswith(_with_sep(realpath(f))) for f in consider_paths]): # matches at least one path -> turn into submodule _install_subds_inplace( ds=ds, path=utf_abspath, # can be ignored, we don't need the return value relativepath=utf.rstrip(os.sep), name=None) new_modules.append(utf.rstrip(os.sep)) return new_modules def deinit_deleted_submodules(ds): # helper to inspect a repo and `deinit` all submodules that are reported # as present, but the mountpoint doesn't exist deleted = ds.repo.get_deleted_files() deinited = [] for subdspath in ds.get_subdatasets(absolute=False, recursive=False): if subdspath in deleted: lgr.debug('deinit deleted subdataset {} in {}'.format(subdspath, ds)) ds.repo.deinit_submodule(subdspath) deinited.append(subdspath) return deinited class Save(Interface): """Save the current state of a dataset Saving the state of a dataset records all changes that have been made to it. This change record is annotated with a user-provided description. Optionally, an additional tag, such as a version, can be assigned to the saved state. Such tag enables straightforward retrieval of past versions at a later point in time. || PYTHON >> Returns ------- commit or None `None` if nothing was saved, the resulting commit otherwise. << PYTHON || """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc=""""specify the dataset to save. If no dataset is given, an attempt is made to identify the dataset based on the current working directory.""", constraints=EnsureDataset() | EnsureNone()), files=Parameter( args=("files",), metavar='FILES', doc="""list of files to consider. If given, only changes made to those files are recorded in the new state.""", nargs='*', constraints=EnsureStr() | EnsureNone()), message=Parameter( args=("-m", "--message",), metavar='MESSAGE', doc="""a message to annotate the saved state.""", constraints=EnsureStr() | EnsureNone()), auto_add_changes=Parameter( args=("-a", "--auto-add-changes"), doc="""automatically include all changes in the entire dataset, independent of the current working directory.""", action="store_true"), version_tag=Parameter( args=("--version-tag",), metavar='ID', doc="""an additional marker for that state.""", constraints=EnsureStr() | EnsureNone()), super_datasets=super_datasets_flag, recursive=recursion_flag, recursion_limit=recursion_limit, ) @staticmethod @datasetmethod(name='save') def __call__(message=None, files=None, dataset=None, auto_add_changes=False, version_tag=None, recursive=False, recursion_limit=None, super_datasets=False): # shortcut ds = require_dataset(dataset, check_installed=True, purpose='saving') if not ds.repo.repo.is_dirty( index=True, working_tree=True, untracked_files=True, submodules=True): # if we cannot see anything dirty at all, the only things we could # do is tag if version_tag: ds.repo.tag(version_tag) # take the easy one out return # always yields list; empty if None files = assure_list(files) # track what to be committed, so it becomes # possible to decide when/what to save further down # and one level up orig_hexsha = ds.repo.get_hexsha() to_commit = [] # before anything, let's deal with missing submodules that may have # been rm'ed by the user # this will not alter/amend the history of the dataset deinit_deleted_submodules(ds) # XXX path resolution needs to happen on the input argument, not the # resolved dataset! # otherwise we will not be able to figure out, whether there was an # explicit dataset provided, or just a matching one resolved # automatically. # if files are provided but no dataset, we interpret them as # CWD-related if auto_add_changes: # use the dataset's base path to indicate that everything # should be saved if files: lgr.warning( "List of paths was provided to save but auto_add_changes " "was specified, so list of paths was ignored") files = [ds.path] else: # make sure we apply the usual path interpretation logic files = [resolve_path(p, dataset) for p in files] new_submodules = untracked_subdatasets_to_submodules(ds, files) if new_submodules: # make sure that .gitmodules is added to the list of files # to be committed. Adding to index might not be enough iff # custom files was provided to_commit.append('.gitmodules') to_commit.extend(new_submodules) # now we should have a complete list of submodules to potentially # recurse into if recursive and (recursion_limit is None or recursion_limit > 0): # what subdataset to touch? subdss = [] if auto_add_changes: # all installed 1st-level ones # we only want immediate subdatasets, higher depths will come # via recursion subdss = [Dataset(opj(ds.path, subds_path)) for subds_path in ds.get_subdatasets( recursive=False)] elif files is not None: # only subdatasets that contain any of the to-be-considered # paths # TODO: the same deductions will be redone later again # very inefficient. Should be just sorted into subds # once! subdss = [ds.get_containing_subdataset( p, recursion_limit=1) for p in files] # skip anything that isn't installed, or this dataset subdss = [d for d in subdss if d.is_installed() and d != ds] prop_recursion_limit = \ None if recursion_limit is None else max(recursion_limit - 1, 0) for subds in subdss: # TODO: just make use of get._sort_paths_into_datasets # currently it is very inefficient since for the same ds # it asks about subdatasets for every file! subds_files = [] # files belonging to the subds todo_files = [] # leftover files for f in files: if ds.get_containing_subdataset(f, recursion_limit=1) == subds: subds_files.append(f) else: todo_files.append(f) files = todo_files subds_modified = Save.__call__( message=message, files=subds_files, dataset=subds, auto_add_changes=auto_add_changes, version_tag=version_tag, recursive=recursive and (prop_recursion_limit is None or prop_recursion_limit > 0), recursion_limit=prop_recursion_limit, ) if subds_modified: # stage changes in this submodule subdspath = relpath(subds.path, ds.path) ds.repo.add(subdspath, git=True) to_commit.append(subdspath) if files: # could still be none without auto add changes ds_subdatasets = ds.get_subdatasets(recursive=False) subdatasets_paths = { opj(ds.path, f) for f in ds_subdatasets } # TODO: also use some centralized sorting into sub-datasets # e.g. one used in get ds_files = [ f for f in files if f in subdatasets_paths or ds.get_containing_subdataset(f, recursion_limit=1) == ds ] if len(ds_files): # XXX Is there a better way to handle files in mixed repos? ds.repo.add(ds_files) ds.repo.add(ds_files, git=True) to_commit.extend(ds_files) # it might be that the file itself is the submodule, so we might # need to commit .gitmodules for f in files: for subds in subdatasets_paths: if subds.rstrip('/') == f.rstrip('/'): to_commit.append('.gitmodules') break _datalad_msg = False if not message: message = 'Recorded existing changes' _datalad_msg = True # extend with files yet to be committed in this dataset to_commit.extend(files) # anything should be staged by now # however, staged submodule changes are not considered as # `index`, hence `submodules` needs to be True too # we can have an explicit list of stuff to save or (if no `files` # provided) have staged stuff if ds.repo.repo.is_dirty( index=True, working_tree=False, untracked_files=False, submodules=True): # Analyze list of known to be committed files/submodules, # see if nothing points outside, and then convert to relative paths to_commit_rel = [] if to_commit: repopath = ds.repo.path for f in to_commit: if isabs(f): frel = relpath(f, repopath) if frel.startswith(pardir): # XXX may be just a warning and skip? raise RuntimeError( "Path %s outside of the dataset %s. Can't commit" % (f, ds) ) f = frel to_commit_rel.append(f) to_commit_rel = sorted(set(to_commit_rel)) if '.' in to_commit_rel: # we need to commit everything to_commit_rel = [] ds.repo.commit(message, options=to_commit_rel, _datalad_msg=_datalad_msg) elif to_commit: lgr.warning( "Was instructed to commit %s files but repository is not dirty", to_commit) elif not auto_add_changes: lgr.info( 'Nothing to save, consider auto-detection of changes, ' 'if this is unexpected.') # MIH: let's tag even if there was nothing commit. I'd forget this # option too often... if version_tag: ds.repo.tag(version_tag) _was_modified = ds.repo.get_hexsha() != orig_hexsha # and now we could consider saving our changes within super-datasets # Let's float up until we get to a non-dataset if super_datasets: if _was_modified: if version_tag: lgr.info("Version tag %s will not be applied to super datasets", version_tag) superds = ds while True: supersubds = superds superds = superds.get_superdataset(datalad_only=True) if not superds: break Save.__call__( message=message + " [origin: %s]" % relpath(ds.path, superds.path), files=[relpath(supersubds.path, superds.path)], dataset=superds, auto_add_changes=False, version_tag=None, recursive=False, ) else: lgr.info( "Not trying to save super-datasets since no modifications") # TODO: figure out what we should return for recursive/super_datasets # shouldn't we return all commits??? return ds.repo.repo.head.commit if _was_modified else None @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui if res: ui.message('Saved state: "{0}" by {1} [{2}]'.format( res.message.splitlines()[0], res.committer, res.hexsha)) datalad-0.4.1/datalad/interface/test.py000066400000000000000000000013571301115270200177670ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Run internal DataLad (unit)tests to verify correct operation on the system""" __docformat__ = 'restructuredtext' import datalad from .base import Interface class Test(Interface): """Run internal DataLad (unit)tests. This can be used to verify correct operation on the system """ @staticmethod def __call__(): datalad.test() datalad-0.4.1/datalad/interface/tests/000077500000000000000000000000001301115270200175725ustar00rootroot00000000000000datalad-0.4.1/datalad/interface/tests/__init__.py000066400000000000000000000007021301115270200217020ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Interfaces tests """ __docformat__ = 'restructuredtext' datalad-0.4.1/datalad/interface/tests/test_add_archive_content.py000066400000000000000000000373651301115270200252040ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for add-archive-content command """ __docformat__ = 'restructuredtext' import logging from os import unlink from os.path import exists, join as opj, pardir, basename, lexists from glob import glob from ...tests.utils import ok_, eq_, assert_cwd_unchanged, assert_raises, \ with_tempfile, assert_in from ...tests.utils import assert_equal, assert_not_equal from ...tests.utils import assert_false from ...tests.utils import assert_true from ...tests.utils import ok_archives_caches from ...tests.utils import SkipTest from ...support.annexrepo import AnnexRepo from ...support.exceptions import FileNotInRepositoryError from ...support.exceptions import CommandError from ...tests.utils import with_tree, serve_path_via_http, ok_file_under_git, swallow_outputs from ...tests.utils import swallow_logs from ...utils import chpwd, getpwd, rmtemp from ...utils import find_files from ...utils import rmtree from datalad.log import lgr from ...api import add_archive_content, clean treeargs = dict( tree=( ('1.tar.gz', ( ('crcns_pfc-1_data', (('CR24A', ( ('behaving1', {'1 f.txt': '1 f load'}),)),)), ('crcns_pfc-1_data', (('CR24C', ( ('behaving3', {'3 f.txt': '3 f load'}),)),)), ('crcns_pfc-1_data', (('CR24D', ( ('behaving2', {'2 f.txt': '2 f load'}),)),)), ('__MACOSX', (('crcns_pfc-2_data', ( ('CR24B', ( ('behaving2', {'2 f.txt': '2 f load'}),)),) ),)), ('crcns_pfc-2_data', (('__MACOSX', ( ('CR24E', ( ('behaving2', {'2 f.txt': '2 f load'}),)),) ),)), )), ) ) @assert_cwd_unchanged(ok_to_chdir=True) @with_tree(**treeargs) @serve_path_via_http() @with_tempfile(mkdir=True) def test_add_archive_dirs(path_orig, url, repo_path): # change to repo_path chpwd(repo_path) # create annex repo repo = AnnexRepo(repo_path, create=True, direct=False) # add archive to the repo so we could test with swallow_outputs(): repo.add_urls([opj(url, '1.tar.gz')], options=["--pathdepth", "-1"]) repo.commit("added 1.tar.gz") # test with excludes and annex options add_archive_content('1.tar.gz', existing='archive-suffix', # Since inconsistent and seems in many cases no leading dirs to strip, keep them as provided strip_leading_dirs=True, delete=True, leading_dirs_consider=['crcns.*', '1'], leading_dirs_depth=2, use_current_dir=False, exclude='.*__MACOSX.*') # some junk penetrates all_files = sorted(find_files('.')) target_files = { './CR24A/behaving1/1 f.txt', './CR24C/behaving3/3 f.txt', './CR24D/behaving2/2 f.txt', } eq_(set(all_files), target_files) # regression test: the subdir in MACOSX wasn't excluded and its name was getting stripped by leading_dir_len assert_false(exists('__MACOSX')) # if stripping and exclude didn't work this fails assert_false(exists('c-1_data')) # if exclude doesn't work then name of subdir gets stripped by leading_dir_len assert_false(exists('CR24B')) # if exclude doesn't work but everything else works this fails # within top directory # archive is in subdirectory -- adding in the same (or different) directory tree1args = dict( tree=( ('1.tar.gz', ( ('1 f.txt', '1 f load'), ('d', (('1d', ''),)), )), ('1u', { '1.tar.gz': { # updated file content '1 f.txt': '1 f load1' }}), ('2u', { # updated file content '1.tar.gz': { '1 f.txt': '1 f load2' }}), ('3u', { # updated file content '1.tar.gz': { '1 f.txt': '1 f load3' }}), ('4u', { # updated file content '1.tar.gz': { '1 f.txt': '1 f load4' }}), ('d1', (('1.tar.gz', ( ('2 f.txt', '2 f load'), ('d2', ( ('2d', ''),) )),),),), ) ) tree4uargs = dict( tree=( ('4u', { # updated file content '1.tar.gz': { '1 f.txt': '1 f load4', 'sub.tar.gz': { '2 f.txt': '2 f' } }}), ) ) @assert_cwd_unchanged(ok_to_chdir=True) @with_tree(**tree1args) @serve_path_via_http() @with_tempfile(mkdir=True) def test_add_archive_content(path_orig, url, repo_path): direct = False # TODO: test on undirect, but too long ATM orig_pwd = getpwd() chpwd(repo_path) # TODO we need to be able to pass path into add_archive_content # We could mock but I mean for the API assert_raises(RuntimeError, add_archive_content, "nonexisting.tar.gz") # no repo yet repo = AnnexRepo(repo_path, create=True, direct=direct) assert_raises(ValueError, add_archive_content, "nonexisting.tar.gz") # we can't add a file from outside the repo ATM assert_raises(FileNotInRepositoryError, add_archive_content, opj(path_orig, '1.tar.gz')) # Let's add first archive to the repo so we could test with swallow_outputs(): repo.add_urls([opj(url, '1.tar.gz')], options=["--pathdepth", "-1"]) for s in range(1, 5): repo.add_urls([opj(url, '%du/1.tar.gz' % s)], options=["--pathdepth", "-2"]) repo.commit("added 1.tar.gz") key_1tar = repo.get_file_key('1.tar.gz') # will be used in the test later def d1_basic_checks(): ok_(exists('1')) ok_file_under_git('1', '1 f.txt', annexed=True) ok_file_under_git(opj('1', 'd', '1d'), annexed=True) ok_archives_caches(repo_path, 0) # and by default it just does it, everything goes to annex repo_ = add_archive_content('1.tar.gz') eq_(repo.path, repo_.path) d1_basic_checks() # If ran again, should proceed just fine since the content is the same so no changes would be made really add_archive_content('1.tar.gz') # But that other one carries updated file, so should fail due to overwrite with assert_raises(RuntimeError) as cme: add_archive_content(opj('1u', '1.tar.gz')) # TODO: somewhat not precise since we have two possible "already exists" # -- in caching and overwrite check assert_in("already exists", str(cme.exception)) # but should do fine if overrides are allowed add_archive_content(opj('1u', '1.tar.gz'), existing='overwrite', use_current_dir=True) d1_basic_checks() add_archive_content(opj('2u', '1.tar.gz'), existing='archive-suffix', use_current_dir=True) add_archive_content(opj('3u', '1.tar.gz'), existing='archive-suffix', use_current_dir=True) add_archive_content(opj('4u', '1.tar.gz'), existing='archive-suffix', use_current_dir=True) # rudimentary test assert_equal(sorted(map(basename, glob(opj(repo_path, '1', '1*')))), ['1 f-1.1.txt', '1 f-1.2.txt', '1 f-1.txt', '1 f.txt']) whereis = repo.whereis(glob(opj(repo_path, '1', '1*'))) # they all must be the same assert(all([x == whereis[0] for x in whereis[1:]])) # and we should be able to reference it while under subdirectory subdir = opj(repo_path, 'subdir') with chpwd(subdir, mkdir=True): add_archive_content(opj(pardir, '1.tar.gz'), use_current_dir=True) d1_basic_checks() # or we could keep relative path and also demand to keep the archive prefix # while extracting under original (annex root) dir add_archive_content(opj(pardir, '1.tar.gz'), add_archive_leading_dir=True) with chpwd('1'): d1_basic_checks() # test with excludes and renames and annex options add_archive_content( '1.tar.gz', exclude=['d'], rename=['/ /_', '/^1/2'], annex_options="-c annex.largefiles=exclude=*.txt", delete=True) # no conflicts since new name ok_file_under_git('2', '1_f.txt', annexed=False) assert_false(exists(opj('2', 'd'))) assert_false(exists('1.tar.gz')) # delete was in effect # now test ability to extract within subdir with chpwd('d1', mkdir=True): # Let's add first archive to the repo so we could test # named the same way but different content with swallow_outputs(): repo.add_urls([opj(url, 'd1', '1.tar.gz')], options=["--pathdepth", "-1"], cwd=getpwd()) # invoke under current subdir repo.commit("added 1.tar.gz in d1") def d2_basic_checks(): ok_(exists('1')) ok_file_under_git('1', '2 f.txt', annexed=True) ok_file_under_git(opj('1', 'd2', '2d'), annexed=True) ok_archives_caches(repo.path, 0) add_archive_content('1.tar.gz') d2_basic_checks() # in manual tests ran into the situation of inability to obtain on a single run # a file from an archive which was coming from a dropped key. I thought it was # tested in custom remote tests, but I guess not sufficiently well enough repo.drop(opj('1', '1 f.txt')) # should be all kosher repo.get(opj('1', '1 f.txt')) ok_archives_caches(repo.path, 1, persistent=True) ok_archives_caches(repo.path, 0, persistent=False) repo.drop(opj('1', '1 f.txt')) # should be all kosher repo.drop(key_1tar, options=['--key']) # is available from the URL -- should be kosher repo.get(opj('1', '1 f.txt')) # that what managed to not work # TODO: check if persistent archive is there for the 1.tar.gz # We should be able to drop everything since available online with swallow_outputs(): clean(annex=repo) repo.drop(key_1tar, options=['--key']) # is available from the URL -- should be kosher chpwd(orig_pwd) # just to avoid warnings ;) move below whenever SkipTest removed repo.drop(opj('1', '1 f.txt')) # should be all kosher repo.get(opj('1', '1 f.txt')) # and should be able to get it again # bug was that dropping didn't work since archive was dropped first repo._annex_custom_command([], ["git", "annex", "drop", "--all"]) # verify that we can't drop a file if archive key was dropped and online archive was removed or changed size! ;) repo.get(key_1tar, options=['--key']) unlink(opj(path_orig, '1.tar.gz')) with swallow_logs(new_level=logging.ERROR) as cml: assert_raises(CommandError, repo.drop, key_1tar, options=['--key']) assert exists(opj(repo.path, repo.get_contentlocation(key_1tar))) assert_in('Could only verify the existence of 0 out of 1 necessary copies', cml.out) @assert_cwd_unchanged(ok_to_chdir=True) @with_tree(**tree1args) @serve_path_via_http() @with_tempfile(mkdir=True) def test_add_archive_content_strip_leading(path_orig, url, repo_path): direct = False # TODO: test on undirect, but too long ATM orig_pwd = getpwd() chpwd(repo_path) repo = AnnexRepo(repo_path, create=True, direct=direct) # Let's add first archive to the repo so we could test with swallow_outputs(): repo.add_urls([opj(url, '1.tar.gz')], options=["--pathdepth", "-1"]) repo.commit("added 1.tar.gz") add_archive_content('1.tar.gz', strip_leading_dirs=True) ok_(not exists('1')) ok_file_under_git(repo.path, '1 f.txt', annexed=True) ok_file_under_git('d', '1d', annexed=True) ok_archives_caches(repo.path, 0) chpwd(orig_pwd) # just to avoid warnings ;) # looking for the future tagging of lengthy tests test_add_archive_content.tags = ['integration'] @assert_cwd_unchanged(ok_to_chdir=True) @with_tree(**tree4uargs) def test_add_archive_use_archive_dir(repo_path): direct = False # TODO: test on undirect, but too long ATM repo = AnnexRepo(repo_path, create=True, direct=direct) with chpwd(repo_path): # Let's add first archive to the repo with default setting archive_path = opj('4u', '1.tar.gz') with swallow_outputs(): repo.add(archive_path) repo.commit("added 1.tar.gz") ok_archives_caches(repo.path, 0) add_archive_content(archive_path, strip_leading_dirs=True, use_current_dir=True) ok_(not exists(opj('4u', '1 f.txt'))) ok_file_under_git(repo.path, '1 f.txt', annexed=True) ok_archives_caches(repo.path, 0) # and now let's extract under archive dir add_archive_content(archive_path, strip_leading_dirs=True) ok_file_under_git(repo.path, opj('4u', '1 f.txt'), annexed=True) ok_archives_caches(repo.path, 0) add_archive_content(opj('4u', 'sub.tar.gz')) ok_file_under_git(repo.path, opj('4u', 'sub', '2 f.txt'), annexed=True) ok_archives_caches(repo.path, 0) class TestAddArchiveOptions(): # few tests bundled with a common setup/teardown to minimize boiler plate @with_tree(tree={'1.tar': {'file.txt': 'load', '1.dat': 'load2'}}, delete=False) def setup(self, repo_path): self.pwd = getpwd() direct = False # TODO: test on undirect, but too long ATM self.annex = annex = AnnexRepo(repo_path, create=True, direct=direct) # Let's add first archive to the annex so we could test annex.add('1.tar') annex.commit(msg="added 1.tar") def teardown(self): assert_equal(self.pwd, getpwd()) rmtemp(self.annex.path) def test_add_delete(self): # To test that .tar gets removed add_archive_content('1.tar', annex=self.annex, strip_leading_dirs=True, delete=True) assert_false(lexists(opj(self.annex.path, '1.tar'))) def test_add_archive_leading_dir(self): import os os.mkdir(opj(self.annex.path, 'sub')) f123 = opj('sub', '123.tar') os.rename(opj(self.annex.path, '1.tar'), opj(self.annex.path, f123)) self.annex.remove('1.tar', force=True) self.annex.add(f123) self.annex.commit(msg="renamed") add_archive_content(f123, annex=self.annex, add_archive_leading_dir=True, strip_leading_dirs=True) ok_file_under_git(self.annex.path, opj('sub', '123', 'file.txt'), annexed=True) def test_add_delete_after_and_drop(self): # To test that .tar gets removed # but that new stuff was added to annex repo. We know the key since default # backend and content remain the same key1 = 'SHA256E-s5--16d3ad1974655987dd7801d70659990b89bfe7e931a0a358964e64e901761cc0.dat' # previous state of things: prev_files = list(find_files('.*', self.annex.path)) with assert_raises(Exception), \ swallow_logs(): self.annex.whereis(key1, key=True, output='full') add_archive_content('1.tar', annex=self.annex, strip_leading_dirs=True, delete_after=True) assert_equal(prev_files, list(find_files('.*', self.annex.path))) w = self.annex.whereis(key1, key=True, output='full') assert_equal(len(w), 2) # in archive, and locally since we didn't drop # Let's now do the same but also drop content add_archive_content('1.tar', annex=self.annex, strip_leading_dirs=True, delete_after=True, drop_after=True) assert_equal(prev_files, list(find_files('.*', self.annex.path))) w = self.annex.whereis(key1, key=True, output='full') assert_equal(len(w), 1) # in archive datalad-0.4.1/datalad/interface/tests/test_clean.py000066400000000000000000000033771301115270200222770ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Primarily a smoke test for ls """ __docformat__ = 'restructuredtext' from os import makedirs from os.path import join as opj from ...api import clean from ...consts import ARCHIVES_TEMP_DIR from ...support.annexrepo import AnnexRepo from ...tests.utils import with_tempfile from ...utils import swallow_outputs from ...utils import chpwd from ...tests.utils import assert_equal, assert_in from ...tests.utils import use_cassette @with_tempfile(mkdir=True) def test_ls(d): repo = AnnexRepo(d, create=True) with swallow_outputs() as cmo: assert_equal(clean(annex=repo), None) # no output ATM assert_equal(cmo.out, '') # no output ATM # if we create some temp archives directory makedirs(opj(d, ARCHIVES_TEMP_DIR, 'somebogus')) with swallow_outputs() as cmo: assert_equal(clean(annex=repo), None) # no output ATM assert_equal("Removing 1 temporary archive directory under %s: somebogus" % opj(d, ARCHIVES_TEMP_DIR), cmo.out.rstrip()) # relative path makedirs(opj(d, ARCHIVES_TEMP_DIR, 'somebogus')) makedirs(opj(d, ARCHIVES_TEMP_DIR, 'somebogus2')) with chpwd(d), \ swallow_outputs() as cmo: assert_equal(clean(), None) # no output ATM assert_equal("Removing 2 temporary archive directories under %s: somebogus, somebogus2" % ARCHIVES_TEMP_DIR, cmo.out.rstrip()) datalad-0.4.1/datalad/interface/tests/test_crawl.py000066400000000000000000000074501301115270200223210ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for crawl command """ __docformat__ = 'restructuredtext' from mock import patch from mock import call from nose import SkipTest from datalad.support.external_versions import external_versions from ...api import crawl from ...tests.utils import assert_cwd_unchanged from ...tests.utils import assert_equal from ...tests.utils import with_tempfile from ...support.stats import ActivityStats from ...utils import chpwd from ...utils import getpwd from ...utils import find_files from ...utils import _path_ @assert_cwd_unchanged(ok_to_chdir=True) @patch('datalad.utils.chpwd') @patch('datalad.crawler.pipeline.load_pipeline_from_config', return_value=['pipeline']) @patch('datalad.crawler.pipeline.run_pipeline', return_value=None) # Note that order of patched things as args is reverse for some reason :-/ def test_crawl_api_chdir(run_pipeline_, load_pipeline_from_config_, chpwd_): output, stats = crawl('some_path_not_checked', chdir='somedir') assert_equal(stats, ActivityStats(datasets_crawled=1)) # nothing was done but we got it assert_equal(output, None) chpwd_.assert_called_with('somedir') load_pipeline_from_config_.assert_called_with('some_path_not_checked') run_pipeline_.assert_called_with(['pipeline'], stats=ActivityStats(datasets_crawled=1)) # XXX we could also mock run_pipeline to adjust stats etc more so we #def run_pipeline_se(output, stats): # stats.add_git += 1 # return output, stats @assert_cwd_unchanged(ok_to_chdir=True) @patch('datalad.utils.chpwd') @patch('datalad.utils.get_logfilename', return_value="some.log") @patch('datalad.crawler.pipeline.get_repo_pipeline_script_path', return_value='script_path') @patch('datalad.crawler.pipeline.load_pipeline_from_config', return_value=['pipeline']) @patch('datalad.crawler.pipeline.run_pipeline', side_effect=[ [], [], [], [], Exception("crawling failed") ] ) @patch('datalad.distribution.dataset.Dataset.get_subdatasets', # return_value=['path1', 'path2']) side_effect=[ ['path1', 'path1/path1_1', 'path2', 'path_to_fail'], # ATM we will not recursively crawl within sub-datasets # ['path1_1'], # sub-dataset of the first sub-dataset #[], # so it would get crawled and have no further sub-datasets # [], # no sub-datasets in path2 # will fail to return anything for path_to_fail ] ) # Note that order of patched things as args is reverse for some reason :-/ @with_tempfile(mkdir=True) def test_crawl_api_recursive(get_subdatasets_, run_pipeline_, load_pipeline_from_config_, get_repo_pipeline_script_path_, get_lofilename_, chpwd_, tdir): pwd = getpwd() with chpwd(tdir): output, stats = crawl(recursive=True) assert_equal(pwd, getpwd()) if external_versions['mock'] < '1.0.1': raise SkipTest("needs a more recent mock which throws exceptions in side_effects") assert_equal(output, [[]]*4 + [None]) # for now output is just a list of outputs assert_equal(stats, ActivityStats(datasets_crawled=5, datasets_crawl_failed=1)) # nothing was done but we got it crawled chpwd_.assert_has_calls( [ call(None), call('path1'), call('path1/path1_1'), call('path2'), ], any_order=True ) assert_equal(list(find_files('.*', tdir, exclude_vcs=False)), [_path_(tdir, 'some.log')]) # no files were generated besides the logdatalad-0.4.1/datalad/interface/tests/test_crawl_init.py000066400000000000000000000066371301115270200233520ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from nose.tools import eq_, assert_raises, assert_in from mock import patch from ...api import crawl_init from collections import OrderedDict from os.path import exists from datalad.support.annexrepo import AnnexRepo from datalad.tests.utils import with_tempfile, chpwd from datalad.tests.utils import ok_clean_git from datalad.consts import CRAWLER_META_CONFIG_PATH, CRAWLER_META_DIR from datalad.distribution.dataset import Dataset @with_tempfile(mkdir=True) def _test_crawl_init(args, template, template_func, save, target_value, tmpdir): ar = AnnexRepo(tmpdir, create=True) with chpwd(tmpdir): crawl_init(args=args, template=template, template_func=template_func, save=save) eq_(exists(CRAWLER_META_DIR), True) eq_(exists(CRAWLER_META_CONFIG_PATH), True) f = open(CRAWLER_META_CONFIG_PATH, 'r') contents = f.read() eq_(contents, target_value) if save: ds = Dataset(tmpdir) ok_clean_git(tmpdir, annex=isinstance(ds.repo, AnnexRepo)) def test_crawl_init(): yield _test_crawl_init, None, 'openfmri', 'superdataset_pipeline', False, \ '[crawl:pipeline]\ntemplate = openfmri\nfunc = superdataset_pipeline\n\n' yield _test_crawl_init, {'dataset': 'ds000001'}, 'openfmri', None, False, \ '[crawl:pipeline]\ntemplate = openfmri\n_dataset = ds000001\n\n' yield _test_crawl_init, ['dataset=ds000001', 'versioned_urls=True'], 'openfmri', None, False, \ '[crawl:pipeline]\ntemplate = openfmri\n_dataset = ds000001\n_versioned_urls = True\n\n' yield _test_crawl_init, None, 'openfmri', 'superdataset_pipeline', True, \ '[crawl:pipeline]\ntemplate = openfmri\nfunc = superdataset_pipeline\n\n' @with_tempfile(mkdir=True) def _test_crawl_init_error(args, template, template_func, target_value, tmpdir): ar = AnnexRepo(tmpdir) with chpwd(tmpdir): assert_raises(target_value, crawl_init, args=args, template=template, template_func=template_func) def test_crawl_init_error(): yield _test_crawl_init_error, 'tmpdir', None, None, ValueError yield _test_crawl_init_error, ['dataset=Baltimore', 'pie=True'], 'openfmri', None, RuntimeError yield _test_crawl_init_error, None, None, None, TypeError @with_tempfile(mkdir=True) def _test_crawl_init_error_patch(return_value, exc, exc_msg, d): ar = AnnexRepo(d, create=True) with patch('datalad.interface.crawl_init.load_pipeline_from_template', return_value=lambda dataset: return_value) as cm: with chpwd(d): with assert_raises(exc) as cm2: crawl_init(args=['dataset=Baltimore'], template='openfmri') assert_in(exc_msg, str(cm2.exception)) cm.assert_called_with('openfmri', None, return_only=True, kwargs=OrderedDict([('dataset', 'Baltimore')])) def test_crawl_init_error_patch(): yield _test_crawl_init_error_patch, [], ValueError, "returned pipeline is empty" yield _test_crawl_init_error_patch, {1: 2}, ValueError, "pipeline should be represented as a list. Got: {1: 2}" datalad-0.4.1/datalad/interface/tests/test_docs.py000066400000000000000000000077101301115270200221400ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for interface doc wranglers. """ __docformat__ = 'restructuredtext' from datalad.interface.base import dedent_docstring from datalad.interface.base import alter_interface_docs_for_api from datalad.interface.base import alter_interface_docs_for_cmdline from datalad.tests.utils import assert_true, assert_false, assert_in, \ assert_not_in, eq_ demo_doc = """\ Bla bla summary Generic intro blurb. Ping pong ping pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping pong ping pong ping pong ping pong. || CMDLINE >> || REFLOW >> Something for the cmdline only Multiline! << REFLOW || << CMDLINE || || PYTHON >> || REFLOW >> Some Python-only bits Multiline! << REFLOW || << PYTHON || And an example for in-line markup: [PY: just for Python PY] and the other one [CMD: just for the command line CMD]. End of demo. Generic appendix. Ding dong ding dong ding dong. Ding dong ding dong ding dong. Ding dong ding dong ding dong. Ding dong ding dong ding dong. Ding dong ding dong ding dong. """ demo_paramdoc = """\ Parameters ---------- dataset : Dataset or None, optional something [PY: python only PY] inbetween [CMD: cmdline only CMD] appended [PY: more python PY] dataset is given, an attempt is made to identify the dataset based Dataset (e.g. a path), or value must be `None`. [Default: None] """ demo_argdoc = """\ specify the dataset to perform the install operation on. If no dataset is given, an attempt is made to identify the dataset based on the current working directory and/or the `path` given. Constraints: Value must be a Dataset or a valid identifier of a Dataset (e.g. a path), or value must be `None`. [Default: None] """ def test_dedent(): assert_false(dedent_docstring("one liner").endswith("\n")) def test_alter_interface_docs_for_api(): alt = alter_interface_docs_for_api(demo_doc) alt_l = alt.split('\n') # dedented assert_false(alt_l[0].startswith(' ')) assert_false(alt_l[-1].startswith(' ')) assert_not_in('CMD', alt) assert_not_in('PY', alt) assert_not_in('REFLOW', alt) assert_in("Some Python-only bits Multiline!", alt) altpd = alter_interface_docs_for_api(demo_paramdoc) assert_in('python', altpd) assert_in('inbetween', altpd) assert_in('appended', altpd) assert_not_in('cmdline', altpd) def test_alter_interface_docs_for_cmdline(): alt = alter_interface_docs_for_cmdline(demo_doc) alt_l = alt.split('\n') # dedented assert_false(alt_l[0].startswith(' ')) assert_false(alt_l[-1].startswith(' ')) assert_not_in('PY', alt) assert_not_in('CMD', alt) assert_not_in('REFLOW', alt) assert_in("Something for the cmdline only Multiline!", alt) # args altarg = alter_interface_docs_for_cmdline(demo_argdoc) # RST role markup eq_(alter_interface_docs_for_cmdline(':murks:`me and my buddies`'), 'me and my buddies') # spread across lines eq_(alter_interface_docs_for_cmdline(':term:`Barbara\nStreisand`'), 'Barbara\nStreisand') # multiple on one line eq_(alter_interface_docs_for_cmdline( ':term:`one` bla bla :term:`two` bla'), 'one bla bla two bla') altpd = alter_interface_docs_for_cmdline(demo_paramdoc) assert_not_in('python', altpd) assert_in('inbetween', altpd) assert_in('appended', altpd) assert_in('cmdline', altpd) datalad-0.4.1/datalad/interface/tests/test_download_url.py000066400000000000000000000036011301115270200236740ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Tests for install-dataset command """ __docformat__ = 'restructuredtext' from os.path import join as opj from ...api import download_url from ...tests.utils import eq_, assert_cwd_unchanged, assert_raises, \ with_tempfile from ...tests.utils import with_tree from ...tests.utils import serve_path_via_http from ...tests.utils import swallow_outputs def test_download_url_exceptions(): assert_raises(ValueError, download_url, ['url1', 'url2'], path=__file__) # is not in effect somehow :-/ TODO: investigate! with swallow_outputs() as cmo: # bogus urls can't be downloaded any ways with assert_raises(RuntimeError) as cm: download_url('http://example.com/bogus') eq_(str(cm.exception), "1 url(s) failed to download") @assert_cwd_unchanged @with_tree(tree=[ ('file1.txt', 'abc'), ('file2.txt', 'abc'), ]) @serve_path_via_http @with_tempfile(mkdir=True) def test_download_url_return(toppath, topurl, outdir): files = ['file1.txt', 'file2.txt'] urls = [topurl + f for f in files] outfiles = [opj(outdir, f) for f in files] with swallow_outputs() as cmo: out1 = download_url(urls[0], path=outdir) eq_(out1, outfiles[:1]) # can't overwrite with assert_raises(RuntimeError), \ swallow_outputs() as cmo: out2 = download_url(urls, path=outdir) eq_(out2, outfiles[1:]) # only 2nd one with swallow_outputs() as cmo: out3 = download_url(urls, path=outdir, overwrite=True) eq_(out3, outfiles) datalad-0.4.1/datalad/interface/tests/test_ls.py000066400000000000000000000256351301115270200216340ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Primarily a smoke test for ls """ __docformat__ = 'restructuredtext' import logging import hashlib from glob import glob from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from datalad.distribution.dataset import Dataset from ...api import ls from ...utils import swallow_outputs, swallow_logs, chpwd from ...tests.utils import assert_equal, assert_in, assert_raises, assert_not_equal from ...tests.utils import use_cassette from ...tests.utils import with_tempfile from ...tests.utils import with_tree from ...tests.utils import skip_if_no_network from datalad.interface.ls import ignored, fs_traverse, _ls_json, machinesize from os.path import exists, join as opj from datalad.downloaders.tests.utils import get_test_providers @skip_if_no_network @use_cassette('test_ls_s3') def test_ls_s3(): url = 's3://datalad-test0-versioned/' with swallow_outputs(): # just to skip if no credentials get_test_providers(url) with swallow_outputs() as cmo: assert_equal(ls(url), None) # not output ATM assert_in('Bucket info:', cmo.out) test_ls_s3.tags = ['network'] @with_tempfile def test_ls_repos(toppath): # smoke test pretty much GitRepo(toppath + '1', create=True) AnnexRepo(toppath + '2', create=True) repos = glob(toppath + '*') for args in (repos, repos + ["/some/bogus/file"]): for recursive in [False, True]: # in both cases shouldn't fail with swallow_outputs() as cmo: ls(args, recursive=recursive) assert_equal(len(cmo.out.rstrip().split('\n')), len(args)) assert_in('[annex]', cmo.out) assert_in('[git]', cmo.out) assert_in('master', cmo.out) if "bogus" in args: assert_in('unknown', cmo.out) def test_machinesize(): assert_equal(1.0, machinesize(1)) for key, value in {'Byte': 0, 'Bytes': 0, 'kB': 1, 'MB': 2, 'GB': 3, 'TB': 4, 'PB': 5}.items(): assert_equal(1.0*(1000**value), machinesize('1 ' + key)) assert_raises(ValueError, machinesize, 't byte') @with_tree( tree={'dir': {'file1.txt': '123', 'file2.txt': '456'}, '.hidden': {'.hidden_file': '121'}}) def test_ignored(topdir): # create annex, git repos AnnexRepo(opj(topdir, 'annexdir'), create=True) GitRepo(opj(topdir, 'gitdir'), create=True) # non-git or annex should not be ignored assert_equal(ignored(topdir), False) # git, annex and hidden nodes should be ignored for subdir in ["annexdir", "gitdir", ".hidden"]: assert_equal(ignored(opj(topdir, subdir)), True) # ignore only hidden nodes(not git or annex repos) flag should work assert_equal(ignored(opj(topdir, "annexdir"), only_hidden=True), False) @with_tree( tree={'dir': {'.fgit': {'ab.txt': '123'}, 'subdir': {'file1.txt': '124', 'file2.txt': '123'}, 'subgit': {'fgit.txt': '123'}}, 'topfile.txt': '123', '.hidden': {'.hidden_file': '123'}}) def test_fs_traverse(topdir): # setup temp directory tree for testing annex = AnnexRepo(topdir) AnnexRepo(opj(topdir, 'annexdir'), create=True) GitRepo(opj(topdir, 'gitdir'), create=True) GitRepo(opj(topdir, 'dir', 'subgit'), create=True) annex.add(opj(topdir, 'dir'), commit=True) annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'), options=['--force']) # traverse file system in recursive and non-recursive modes for recursive in [True, False]: # test fs_traverse in display mode with swallow_logs(new_level=logging.INFO) as log, swallow_outputs() as cmo: fs = fs_traverse(topdir, AnnexRepo(topdir), recursive=recursive, json='display') if recursive: # fs_traverse logs should contain all not ignored subdirectories for subdir in [opj(topdir, 'dir'), opj(topdir, 'dir', 'subdir')]: assert_in('Directory: ' + subdir, log.out) # fs_traverse stdout contains subdirectory assert_in(('file2.txt' and 'dir'), cmo.out) # extract info of the top-level child directory child = [item for item in fs['nodes'] if item['name'] == 'dir'][0] # size of dir type child in non-recursive modes should be 0 Bytes(default) as # dir type child's size currently has no metadata file for traverser to pick its size from # and would require a recursive traversal w/ write to child metadata file mode assert_equal(child['size']['total'], {True: '6 Bytes', False: '0 Bytes'}[recursive]) for recursive in [True, False]: # run fs_traverse in write to json 'file' mode fs = fs_traverse(topdir, AnnexRepo(topdir), recursive=recursive, json='file') # fs_traverse should return a dictionary assert_equal(isinstance(fs, dict), True) # not including git and annex folders assert_equal([item for item in fs['nodes'] if ('gitdir' or 'annexdir') == item['name']], []) # extract info of the top-level child directory child = [item for item in fs['nodes'] if item['name'] == 'dir'][0] # verify node type assert_equal(child['type'], 'dir') # same node size on running fs_traversal in recursive followed by non-recursive mode # verifies child's metadata file being used to find its size # running in reverse order (non-recursive followed by recursive mode) will give (0, actual size) assert_equal(child['size']['total'], '6 Bytes') # verify subdirectory traversal if run in recursive mode if recursive: # sub-dictionary should not include git and hidden directory info assert_equal([item for item in child['nodes'] if ('subgit' or '.fgit') == item['name']], []) # extract subdirectory dictionary, else fail subchild = [subitem for subitem in child["nodes"] if subitem['name'] == 'subdir'][0] # extract info of file1.txts, else fail link = [subnode for subnode in subchild["nodes"] if subnode['name'] == 'file1.txt'][0] # verify node's sizes and type assert_equal(link['size']['total'], '3 Bytes') assert_equal(link['size']['ondisk'], link['size']['total']) assert_equal(link['type'], 'link') # extract info of file2.txt, else fail brokenlink = [subnode for subnode in subchild["nodes"] if subnode['name'] == 'file2.txt'][0] # verify node's sizes and type assert_equal(brokenlink['type'], 'link-broken') assert_equal(brokenlink['size']['ondisk'], '0 Bytes') assert_equal(brokenlink['size']['total'], '3 Bytes') @with_tree( tree={'dir': {'.fgit': {'ab.txt': '123'}, 'subdir': {'file1.txt': '123', 'file2.txt': '123'}, 'subgit': {'fgit.txt': '123'}}, '.hidden': {'.hidden_file': '123'}}) def test_ls_json(topdir): annex = AnnexRepo(topdir, create=True) ds = Dataset(topdir) # create some file and commit it open(opj(ds.path, 'subdsfile.txt'), 'w').write('123') ds.add(path='subdsfile.txt') ds.save("Hello!", version_tag=1) # add a subdataset ds.install('subds', source=topdir) git = GitRepo(opj(topdir, 'dir', 'subgit'), create=True) # create git repo git.add(opj(topdir, 'dir', 'subgit', 'fgit.txt'), commit=True) # commit to git to init git repo annex.add(opj(topdir, 'dir', 'subgit'), commit=True) # add the non-dataset git repo to annex annex.add(opj(topdir, 'dir'), commit=True) # add to annex (links) annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'), options=['--force']) # broken-link meta_dir = opj('.git', 'datalad', 'metadata') meta_path = opj(topdir, meta_dir) def get_metahash(*path): return hashlib.md5(opj(*path).encode('utf-8')).hexdigest() for all_ in [True, False]: for recursive in [True, False]: for state in ['file', 'delete']: with swallow_logs(), swallow_outputs(): ds = _ls_json(topdir, json=state, all_=all_, recursive=recursive) # subdataset should have its json created and deleted when all=True else not subds_metahash = get_metahash('/') subds_metapath = opj(topdir, 'subds', meta_dir, subds_metahash) assert_equal(exists(subds_metapath), (state == 'file' and recursive)) # root should have its json file created and deleted in all cases ds_metahash = get_metahash('/') ds_metapath = opj(meta_path, ds_metahash) assert_equal(exists(ds_metapath), state == 'file') # children should have their metadata json's created and deleted only when recursive=True child_metahash = get_metahash('dir', 'subdir') child_metapath = opj(meta_path, child_metahash) assert_equal(exists(child_metapath), (state == 'file' and all_)) # ignored directories should not have json files created in any case for subdir in [('.hidden'), ('dir', 'subgit')]: child_metahash = get_metahash(*subdir) assert_equal(exists(opj(meta_path, child_metahash)), False) # check if its updated in its nodes sublist too. used by web-ui json. regression test assert_equal(ds['nodes'][0]['size']['total'], ds['size']['total']) # check size of subdataset subds = [item for item in ds['nodes'] if item['name'] == ('subdsfile.txt' or 'subds')][0] assert_equal(subds['size']['total'], '3 Bytes') # run non-recursive dataset traversal after subdataset metadata already created # to verify sub-dataset metadata being picked up from its metadata file in such cases if state == 'file' and recursive and not all_: ds = _ls_json(topdir, json='file', all_=False) subds = [item for item in ds['nodes'] if item['name'] == ('subdsfile.txt' or 'subds')][0] assert_equal(subds['size']['total'], '3 Bytes') @with_tempfile def test_ls_noarg(toppath): # smoke test pretty much AnnexRepo(toppath, create=True) # this test is pointless for now and until ls() actually returns # something with swallow_outputs(): ls_out = ls(toppath) with chpwd(toppath): assert_equal(ls_out, ls([])) assert_equal(ls_out, ls('.')) datalad-0.4.1/datalad/interface/tests/test_save.py000066400000000000000000000125771301115270200221550ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """test command datalad save """ __docformat__ = 'restructuredtext' from os.path import join as opj from datalad.distribution.dataset import Dataset from datalad.support.annexrepo import AnnexRepo from datalad.tests.utils import ok_, assert_false, assert_true, assert_not_equal from datalad import api as _ from datalad.tests.utils import with_testrepos from datalad.tests.utils import with_tempfile from datalad.tests.utils import ok_clean_git from datalad.tests.utils import create_tree from datalad.tests.utils import assert_equal @with_testrepos('.*git.*', flavors=['clone']) def test_save(path): ds = Dataset(path) with open(opj(path, "new_file.tst"), "w") as f: f.write("something") ds.repo.add("new_file.tst", git=True) ok_(ds.repo.dirty) ds.save("add a new file", auto_add_changes=False) ok_clean_git(path, annex=isinstance(ds.repo, AnnexRepo)) with open(opj(path, "new_file.tst"), "w") as f: f.write("modify") ok_(ds.repo.dirty) # no need to git add before: ds.save("modified new_file.tst", auto_add_changes=True) ok_clean_git(path, annex=isinstance(ds.repo, AnnexRepo)) files = ['one.txt', 'two.txt'] for fn in files: with open(opj(path, fn), "w") as f: f.write(fn) ds.save("set of new files", files=[opj(path, f) for f in files]) ok_clean_git(path, annex=isinstance(ds.repo, AnnexRepo)) # create subdataset subds = ds.create('subds') ok_clean_git(path, annex=isinstance(ds.repo, AnnexRepo)) # modify subds with open(opj(subds.path, "some_file.tst"), "w") as f: f.write("something") subds.save(auto_add_changes=True) ok_clean_git(subds.path, annex=isinstance(ds.repo, AnnexRepo)) ok_(ds.repo.dirty) # ensure modified subds is commited ds.save(auto_add_changes=True) ok_clean_git(path, annex=isinstance(ds.repo, AnnexRepo)) @with_tempfile() def test_recursive_save(path): ds = Dataset(path).create() # nothing to save assert_false(ds.save()) subds = ds.create('sub') # subdataset presence already saved ok_clean_git(ds.path) subsubds = subds.create('subsub') with open(opj(subsubds.path, 'test'), 'w') as f: f.write('some') # does not save anything in the topdataset assert_false(ds.save()) # auto-add will save addition of subsubds to subds assert_true(ds.save(auto_add_changes=True)) # with recursive it will add the file in subsubds assert_true(ds.save(auto_add_changes=True, recursive=True)) # add content to subsub and try saving testfname = opj('sub', 'subsub', 'saveme') with open(opj(ds.path, testfname), 'w') as f: f.write('I am in here!') # the following should all do nothing # no auto_add assert_false(ds.save()) # no recursive assert_false(ds.save(auto_add_changes=True)) # no recursive and auto_add assert_false(ds.save(recursive=True)) # even with explicit target, no recursive safe assert_false(ds.save(files=[testfname])) # insufficient recursion depth for rlevel in (0, 1): assert_false(ds.save(files=[testfname], recursive=True, recursion_limit=rlevel)) # and finally with the right settings assert_true(ds.save(files=[testfname], recursive=True, recursion_limit=2)) # there is nothing else to save assert_false(ds.save(auto_add_changes=True, recursive=True)) # one more time and check that all datasets in the hierarchy get updated states = [d.repo.get_hexsha() for d in (ds, subds, subsubds)] testfname = opj('sub', 'subsub', 'saveme2') with open(opj(ds.path, testfname), 'w') as f: f.write('I am in here!') assert_true(ds.save(auto_add_changes=True, recursive=True)) newstates = [d.repo.get_hexsha() for d in (ds, subds, subsubds)] for old, new in zip(states, newstates): assert_not_equal(old, new) ## now let's check saving "upwards" assert not subds.repo.dirty create_tree(subds.path, {"testnew": 'smth', "testadded": "added"}) subds.repo.add("testadded") indexed_files = subds.repo.get_indexed_files() assert subds.repo.dirty assert ds.repo.dirty assert not subsubds.repo.dirty create_tree(subsubds.path, {"testnew2": 'smth'}) assert subsubds.repo.dirty # and indexed files didn't change assert_equal(indexed_files, subds.repo.get_indexed_files()) ok_clean_git(subds.repo, untracked=['testnew'], index_modified=['subsub'], head_modified=['testadded']) subsubds.save(message="saving", super_datasets=True, auto_add_changes=True) ok_clean_git(subsubds.repo) # but its super should have got only the subsub saved # not the file we created ok_clean_git(subds.repo, untracked=['testnew'], head_modified=['testadded']) # check commits to have correct messages assert_equal(next(ds.repo.get_branch_commits('master')).message.rstrip(), 'saving [origin: sub/subsub]') assert_equal(next(subds.repo.get_branch_commits('master')).message.rstrip(), 'saving [origin: subsub]') datalad-0.4.1/datalad/interface/tests/test_unlock.py000066400000000000000000000075241301115270200225060ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """test command datalad unlock """ __docformat__ = 'restructuredtext' from os import geteuid from os.path import join as opj from datalad.distribution.dataset import Dataset from datalad.api import create from datalad.api import unlock from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.exceptions import CommandError from datalad.support.annexrepo import AnnexRepo from datalad.tests.utils import with_tempfile from datalad.tests.utils import assert_raises from datalad.tests.utils import eq_, ok_ from datalad.tests.utils import getpwd from datalad.tests.utils import chpwd from datalad.tests.utils import assert_cwd_unchanged from datalad.tests.utils import with_testrepos from datalad.tests.utils import assert_in from datalad.tests.utils import on_windows, skip_if @assert_cwd_unchanged @with_tempfile(mkdir=True) @with_tempfile @with_tempfile def test_unlock_raises(path, path2, path3): # make sure, we are not within a dataset: _cwd = getpwd() chpwd(path) # no dataset and no path: assert_raises(InsufficientArgumentsError, unlock, dataset=None, path=None) # no dataset and path not within a dataset: assert_raises(InsufficientArgumentsError, unlock, dataset=None, path=path2) # no dataset and inconsistent paths: assert_raises(InsufficientArgumentsError, unlock, dataset=None, path=[path2, path3]) # let's be in a dataset,which is no annex: create(path=path, no_annex=True) ds = Dataset(path) assert_raises(ValueError, ds.unlock) # make it annex, but call unlock with invalid path: AnnexRepo(path, create=True) with assert_raises(CommandError) as cm: ds.unlock(path="notexistent.txt") assert_in("git-annex: notexistent.txt not found", str(cm.exception)) chpwd(_cwd) # Note: As root there is no actual lock/unlock. # Therefore don't know what to test for yet. @skip_if(cond=not on_windows and geteuid() == 0) # uid not available on windows @with_testrepos('.*annex.*', flavors=['clone']) def test_unlock(path): ds = Dataset(path) # file is currently locked: # TODO: use get_annexed_files instead of hardcoded filename assert_raises(IOError, open, opj(path, 'test-annex.dat'), "w") # cannot unlock without content (annex get wasn't called): assert_raises(CommandError, ds.unlock) ds.repo.get('test-annex.dat') result = ds.unlock() assert len(result) >= 1 assert_in('test-annex.dat', result) with open(opj(path, 'test-annex.dat'), "w") as f: f.write("change content") ds.repo.add('test-annex.dat') ds.repo.commit("edit 'test-annex.dat' via unlock and lock it again") # after commit, file is locked again: assert_raises(IOError, open, opj(path, 'test-annex.dat'), "w") # content was changed: with open(opj(path, 'test-annex.dat'), "r") as f: eq_("change content", f.read()) # unlock again, this time more specific: result = ds.unlock(path='test-annex.dat') eq_(['test-annex.dat'], result) with open(opj(path, 'test-annex.dat'), "w") as f: f.write("change content again") ds.repo.add('test-annex.dat') ds.repo.commit("edit 'test-annex.dat' via unlock and lock it again") # after commit, file is locked again: assert_raises(IOError, open, opj(path, 'test-annex.dat'), "w") # content was changed: with open(opj(path, 'test-annex.dat'), "r") as f: eq_("change content again", f.read()) datalad-0.4.1/datalad/interface/tests/test_utils.py000066400000000000000000000050741301115270200223510ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test dirty dataset handling """ __docformat__ = 'restructuredtext' from os.path import join as opj from nose.tools import assert_raises, assert_equal from datalad.tests.utils import with_tempfile, assert_not_equal from datalad.tests.utils import ok_clean_git from datalad.interface.utils import handle_dirty_dataset from datalad.distribution.dataset import Dataset from datalad.api import save _dirty_modes = ('fail', 'ignore', 'save-before') def _check_all_clean(ds, state): assert state is not None for mode in _dirty_modes: # nothing wrong, nothing saved handle_dirty_dataset(ds, mode) assert_equal(state, ds.repo.get_hexsha()) def _check_auto_save(ds, orig_state): handle_dirty_dataset(ds, 'ignore') assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail') handle_dirty_dataset(ds, 'save-before') state = ds.repo.get_hexsha() assert_not_equal(orig_state, state) _check_all_clean(ds, state) return state @with_tempfile(mkdir=True) def test_dirty(path): for mode in _dirty_modes: # does nothing without a dataset handle_dirty_dataset(None, mode) # placeholder, but not yet created ds = Dataset(path) # unknown mode assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP') # not yet created is very dirty assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail') handle_dirty_dataset(ds, 'ignore') assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before') # should yield a clean repo ds.create() orig_state = ds.repo.get_hexsha() _check_all_clean(ds, orig_state) # tainted: untracked with open(opj(ds.path, 'something'), 'w') as f: f.write('some') orig_state = _check_auto_save(ds, orig_state) # tainted: staged with open(opj(ds.path, 'staged'), 'w') as f: f.write('some') ds.repo.add('staged', git=True) orig_state = _check_auto_save(ds, orig_state) # tainted: submodule # not added to super on purpose! subds = ds.create('subds') _check_all_clean(subds, subds.repo.get_hexsha()) ok_clean_git(ds.path) # subdataset must be added as a submodule! assert_equal(ds.get_subdatasets(), ['subds']) datalad-0.4.1/datalad/interface/unlock.py000066400000000000000000000120011301115270200202670ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """For now just a proxy to git annex unlock """ __docformat__ = 'restructuredtext' import logging from os.path import commonprefix from os.path import abspath from datalad.support.constraints import EnsureStr from datalad.support.constraints import EnsureNone from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from datalad.support.param import Parameter from datalad.distribution.dataset import Dataset from datalad.distribution.dataset import EnsureDataset from datalad.distribution.dataset import datasetmethod from datalad.utils import getpwd from .base import Interface lgr = logging.getLogger('datalad.interface.unlock') class Unlock(Interface): """Unlock file(s) of a dataset Unlock files of a dataset in order to be able to edit the actual content """ _params_ = dict( path=Parameter( args=("path",), doc="""file(s) to unlock""", nargs="*", constraints=EnsureStr() | EnsureNone()), dataset=Parameter( args=("-d", "--dataset"), doc=""""specify the dataset to unlock files in. If no dataset is given, an attempt is made to identify the dataset based on the current working directory. If the latter fails, an attempt is made to identify the dataset based on `path` """, constraints=EnsureDataset() | EnsureNone()),) @staticmethod @datasetmethod(name='unlock') def __call__(path=None, dataset=None): # shortcut ds = dataset if isinstance(path, list): if not len(path): # normalize value to expected state when nothing was provided path = None elif len(path) == 1: # we can simply continue with the function as called with a # single argument path = path[0] if ds is not None and not isinstance(ds, Dataset): ds = Dataset(ds) if ds is None: # try CWD: dspath = GitRepo.get_toppath(getpwd()) if not dspath: if path is None: raise InsufficientArgumentsError( "insufficient arguments for unlocking: needs at least " "a dataset or a path to unlock.") # if we still have no dataset, try deriving it from path(s): if isinstance(path, list): # several paths and no dataset given; # paths have to be absolute and have to have common prefix # in order to be able to find a dataset # TODO: maybe consider realpath? prefix = commonprefix(path) if not prefix: raise InsufficientArgumentsError( "insufficient information for unlocking: no " "dataset given and paths don't have a common base " "to check for a dataset") dspath = GitRepo.get_toppath(abspath(prefix)) else: # single path dspath = GitRepo.get_toppath(abspath(path)) if dspath is None: raise InsufficientArgumentsError( "insufficient information for unlocking: no " "dataset given and none could be derived " "from given path(s) or current working directory") ds = Dataset(dspath) assert ds assert ds.repo if not isinstance(ds.repo, AnnexRepo): # TODO: Introduce NoAnnexError raise ValueError("No annex found in dataset (%s)." % ds.path) # TODO: AnnexRepo().unlock() with proper return value if not path: files = [] elif isinstance(path, list): files = path else: files = [path] std_out, std_err = ds.repo._annex_custom_command( files, ['git', 'annex', 'unlock']) return [line.split()[1] for line in std_out.splitlines() if line.strip().endswith('ok')] @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui if res is None: res = [] if not isinstance(res, list): res = [res] if not len(res): ui.message("Nothing was unlocked") return items = '\n'.join(map(str, res)) msg = "Unlocked {n} files:\n{items}".format( n=len(res), items=items) ui.message(msg) datalad-0.4.1/datalad/interface/utils.py000066400000000000000000000034351301115270200201470ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Interface utility functions """ __docformat__ = 'restructuredtext' # avoid import from API to not get into circular imports from datalad.interface.save import Save def handle_dirty_dataset(ds, mode, msg=None): """Detect and treat unsaved changes as instructed by `mode` Parameters ---------- ds : Dataset or None Dataset to be inspected. Does nothing if `None`. mode : {'fail', 'ignore', 'save-before'} How to act upon discovering unsaved changes. msg : str or None Custom message to use for a potential commit. Returns ------- None """ if ds is None: # nothing to be handled return if msg is None: msg = '[DATALAD] auto-saved changes' if mode == 'ignore': return elif mode == 'fail': if not ds.repo or ds.repo.repo.is_dirty(index=True, working_tree=True, untracked_files=True, submodules=True): raise RuntimeError('dataset {} has unsaved changes'.format(ds)) elif mode == 'save-before': if not ds.is_installed(): raise RuntimeError('dataset {} is not yet installed'.format(ds)) Save.__call__(dataset=ds, message=msg, auto_add_changes=True) else: raise ValueError("unknown if-dirty mode '{}'".format(mode)) datalad-0.4.1/datalad/log.py000066400000000000000000000220511301115270200156230ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import logging import os import sys import platform import logging.handlers from os.path import basename, dirname from .utils import is_interactive from .support import ansi_colors as colors __all__ = ['ColorFormatter'] # Snippets from traceback borrowed from duecredit which was borrowed from # PyMVPA upstream/2.4.0-39-g69ad545 MIT license (the same copyright as DataLad) def mbasename(s): """Custom function to include directory name if filename is too common Also strip .py at the end """ base = basename(s) if base.endswith('.py'): base = base[:-3] if base in set(['base', '__init__']): base = basename(dirname(s)) + '.' + base return base class TraceBack(object): """Customized traceback to be included in debug messages """ def __init__(self, limit=100, collide=False): """Initialize TraceBack metric Parameters ---------- collide : bool if True then prefix common with previous invocation gets replaced with ... """ self.__prev = "" self.limit = limit self.collide = collide # delayed imports and preparing the regex substitution if collide: import re self.__prefix_re = re.compile('>[^>]*$') else: self.__prefix_re = None import traceback self._extract_stack = traceback.extract_stack def __call__(self): ftb = self._extract_stack(limit=200)[:-2] entries = [[mbasename(x[0]), str(x[1])] for x in ftb if mbasename(x[0]) != 'logging.__init__'] entries = [e for e in entries if e[0] != 'unittest'] if len(entries) > self.limit: sftb = '...>' entries = entries[-self.limit:] else: sftb = '' if not entries: return "" # lets make it more consize entries_out = [entries[0]] for entry in entries[1:]: if entry[0] == entries_out[-1][0]: entries_out[-1][1] += ',%s' % entry[1] else: entries_out.append(entry) sftb += '>'.join( ['%s:%s' % (mbasename(x[0]), x[1]) for x in entries_out] ) if self.collide: # lets remove part which is common with previous invocation prev_next = sftb common_prefix = os.path.commonprefix((self.__prev, sftb)) common_prefix2 = self.__prefix_re.sub('', common_prefix) if common_prefix2 != "": sftb = '...' + sftb[len(common_prefix2):] self.__prev = prev_next return sftb # Recipe from http://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output # by Brandon Thomson # Adjusted for automagic determination either coloring is needed and # prefixing of multiline log lines class ColorFormatter(logging.Formatter): def __init__(self, use_color=None, log_name=False, log_pid=False): if use_color is None: # if 'auto' - use color only if all streams are tty use_color = is_interactive() self.use_color = use_color and platform.system() != 'Windows' # don't use color on windows msg = colors.format_msg(self._get_format(log_name, log_pid), self.use_color) log_env = os.environ.get('DATALAD_LOG_TRACEBACK', '') collide = log_env == 'collide' limit = 100 if collide else int(log_env) if log_env.isdigit() else 100 self._tb = TraceBack(collide=collide, limit=limit) if log_env else None logging.Formatter.__init__(self, msg) def _get_format(self, log_name=False, log_pid=False): from datalad import cfg from datalad.config import anything2bool show_timestamps = anything2bool(cfg.get('datalad.log.timestamp', False)) return (("" if not show_timestamps else "$BOLD%(asctime)-15s$RESET ") + ("%(name)-15s " if log_name else "") + ("{%(process)d}" if log_pid else "") + "[%(levelname)s] " "%(message)s ") def format(self, record): if record.msg.startswith('| '): # If we already log smth which supposed to go without formatting, like # output for running a command, just return the message and be done return record.msg levelname = record.levelname if self.use_color and levelname in colors.LOG_LEVEL_COLORS: fore_color = colors.LOG_LEVEL_COLORS[levelname] levelname_color = (colors.COLOR_SEQ % fore_color) + \ ("%-7s" % levelname) + colors.RESET_SEQ record.levelname = levelname_color record.msg = record.msg.replace("\n", "\n| ") if self._tb: record.msg = self._tb() + " " + record.msg return logging.Formatter.format(self, record) class LoggerHelper(object): """Helper to establish and control a Logger""" def __init__(self, name='datalad', logtarget=None): """ Parameters ---------- name : logtarget : string, optional If we want to use our logger for other log targets, while having a uniform control over them """ self.name = name self.logtarget = logtarget self.lgr = logging.getLogger(logtarget if logtarget is not None else name) def _get_environ(self, var, default=None): return os.environ.get(self.name.upper() + '_%s' % var.upper(), default) def set_level(self, level=None, default='INFO'): """Helper to set loglevel for an arbitrary logger By default operates for 'datalad'. TODO: deduce name from upper module name so it could be reused without changes """ if level is None: # see if nothing in the environment level = self._get_environ('LOG_LEVEL') if level is None: level = default try: # it might be a string which still represents an int log_level = int(level) except ValueError: # or a string which corresponds to a constant;) log_level = getattr(logging, level.upper()) self.lgr.setLevel(log_level) # and set other related/used loggers to the same level to prevent their # talkativity, if they are not yet known to this python session, so we # have little chance to "override" possibly set outside levels for dep in ('git',): if dep not in logging.Logger.manager.loggerDict: logging.getLogger(dep).setLevel(log_level) def get_initialized_logger(self, logtarget=None): """Initialize and return the logger Parameters ---------- target: string, optional Which log target to request logger for logtarget: { 'stdout', 'stderr', str }, optional Where to direct the logs. stdout and stderr stand for standard streams. Any other string is considered a filename. Multiple entries could be specified comma-separated Returns ------- logging.Logger """ # By default mimic previously talkative behavior logtarget = self._get_environ('LOG_TARGET', logtarget or 'stderr') # Allow for multiple handlers being specified, comma-separated if ',' in logtarget: for handler_ in logtarget.split(','): self.get_initialized_logger(logtarget=handler_) return self.lgr if logtarget.lower() in ('stdout', 'stderr'): loghandler = logging.StreamHandler(getattr(sys, logtarget.lower())) use_color = is_interactive() # explicitly decide here else: # must be a simple filename # Use RotatingFileHandler for possible future parametrization to keep # log succinct and rotating loghandler = logging.handlers.RotatingFileHandler(logtarget) use_color = False # I had decided not to guard this call and just raise exception to go # out happen that specified file location is not writable etc. # But now improve with colors and useful information such as time loghandler.setFormatter( ColorFormatter(use_color=use_color, # TODO: config log.name, pid log_name=self._get_environ("LOG_NAME", False), log_pid=self._get_environ("LOG_PID", False), )) # logging.Formatter('%(asctime)-15s %(levelname)-6s %(message)s')) self.lgr.addHandler(loghandler) self.set_level() # set default logging level return self.lgr lgr = LoggerHelper().get_initialized_logger() datalad-0.4.1/datalad/metadata/000077500000000000000000000000001301115270200162505ustar00rootroot00000000000000datalad-0.4.1/datalad/metadata/__init__.py000066400000000000000000000301671301115270200203700ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Metadata handling (parsing, storing, querying)""" import os from six import PY2 if PY2: import cPickle as pickle else: import pickle from hashlib import md5 from six.moves.urllib.parse import urlsplit from six import string_types from os.path import join as opj, exists from os.path import dirname from importlib import import_module from datalad.utils import swallow_logs from datalad.utils import assure_dir from datalad.support.json_py import load as jsonload from datalad.dochelpers import exc_str from datalad.log import lgr from datalad import cfg # common format metadata_filename = 'meta.json' metadata_basepath = opj('.datalad', 'meta') # XXX Could become dataset method def get_metadata_type(ds, guess=False): """Return the metadata type(s)/scheme(s) of a dataset Parameters ---------- ds : Dataset Dataset instance to be inspected guess : bool Whether to try to auto-detect the type if no metadata type setting is found. All supported metadata schemes are tested in alphanumeric order. Returns ------- list(str) Metadata type labels or an empty list if no type setting is found and optional auto-detection yielded no results """ cfg_ = ds.config # TODO give cfg name datalad prefix if cfg_ and cfg_.has_section('metadata'): if cfg_.has_option('metadata', 'nativetype'): return cfg_.get_value('metadata', 'nativetype').split() mtypes = [] if guess: # keep local, who knows what some parsers might pull in from . import parsers for mtype in sorted([p for p in parsers.__dict__ if not (p.startswith('_') or p in ('tests', 'base'))]): if mtype == 'aggregate': # skip, runs anyway, but later continue pmod = import_module('.%s' % (mtype,), package=parsers.__package__) if pmod.MetadataParser(ds).has_metadata(): lgr.debug('Predicted presence of "%s" meta data', mtype) mtypes.append(mtype) else: lgr.debug('No evidence for "%s" meta data', mtype) return mtypes def _get_base_dataset_metadata(ds_identifier): """Return base metadata as dict for a given ds_identifier """ meta = { "@context": { "@vocab": "http://schema.org/", "doap": "http://usefulinc.com/ns/doap#", }, # increment when changes to meta data representation are done "dcterms:conformsTo": "http://docs.datalad.org/metadata.html#v0-1", } if ds_identifier is not None: meta["@id"] = ds_identifier return meta def _get_implicit_metadata(ds, ds_identifier=None): """Convert git/git-annex info into metadata Anything that doesn't come as metadata in dataset **content**, but is encoded in the dataset repository itself. This does not include information on submodules, however. Meta data on subdatasets is provided by a dedicated parser implementation for "aggregate" meta data. Returns ------- dict """ if ds_identifier is None: ds_identifier = ds.id meta = _get_base_dataset_metadata(ds_identifier) if not ds.repo: # everything else comes from a repo return meta if ds.id: # it has an ID, so we consider it a proper dataset meta['type'] = "Dataset" # shortcut repo = ds.repo.repo if repo.head.is_valid(): meta['dcterms:modified'] = repo.head.commit.authored_datetime.isoformat() # maybe use something like git-describe instead -- but tag-references # might changes... meta['version'] = repo.head.commit.hexsha _add_annex_metadata(ds.repo, meta) return meta def _add_annex_metadata(repo, meta): # look for known remote annexes, doesn't need configured # remote to be meaningful # need annex repo instance # TODO refactor to use ds.uuid when #701 is addressed if hasattr(repo, 'repo_info'): # get all other annex ids, and filter out this one, origin and # non-specific remotes with swallow_logs(): # swallow logs, because git annex complains about every remote # for which no UUID is configured -- many special remotes... repo_info = repo.repo_info(fast=True) annex_meta = [] for src in ('trusted repositories', 'semitrusted repositories', 'untrusted repositories'): for anx in repo_info.get(src, []): anxid = anx.get('uuid', '00000000-0000-0000-0000-0000000000') if anxid.startswith('00000000-0000-0000-0000-000000000'): # ignore special continue anx_meta = {'@id': anxid} if 'description' in anx: anx_meta['description'] = anx['description'] # XXX maybe report which one is local? Available in anx['here'] # XXX maybe report the type of annex remote? annex_meta.append(anx_meta) if len(annex_meta) == 1: annex_meta = annex_meta[0] meta['availableFrom'] = annex_meta def is_implicit_metadata(meta): """Return whether a meta data set looks like our own implicit meta data""" std_spec = meta.get('dcterms:conformsTo', '') return isinstance(std_spec, string_types) \ and std_spec.startswith('http://docs.datalad.org/metadata.html#v') def _simplify_meta_data_structure(meta): # get a list of terms from any possible source if isinstance(meta, list) and len(meta) == 1: # simplify structure meta = meta[0] # XXX condition below is outdated (DOAP...), still needed? if isinstance(meta, dict) \ and sorted(meta.keys()) == ['@context', '@graph'] \ and meta.get('@context') == 'http://schema.org/': meta = meta['@graph'] elif isinstance(meta, dict): meta = [meta] elif isinstance(meta, list): # list this possibility explicitely to get 'else' to indicate that # we hit something unforseen pass else: raise NotImplementedError( 'got some unforseens meta data structure') return meta # XXX might become its own command def get_metadata(ds, guess_type=False, ignore_subdatasets=False, ignore_cache=False): # common identifier ds_identifier = ds.id # metadata receptacle meta = [] # where things are meta_path = opj(ds.path, metadata_basepath) main_meta_fname = opj(meta_path, metadata_filename) # from cache? if ignore_cache or not exists(main_meta_fname): # start with the implicit meta data, currently there is no cache for # this type of meta data, as it will change with every clone. # In contrast, native meta data is cached. implicit_meta = _get_implicit_metadata(ds, ds_identifier) meta.append(implicit_meta) # and any native meta data meta.extend( get_native_metadata( ds, guess_type=guess_type, ds_identifier=ds_identifier)) else: # from cache cached_meta = jsonload(main_meta_fname) if isinstance(cached_meta, list): meta.extend(cached_meta) else: meta.append(cached_meta) # cached meta data doesn't have version info for the top-level # dataset -> look for the item and update it for m in meta: if not is_implicit_metadata(m): continue if m.get('@id', None) == ds_identifier: m.update(_get_implicit_metadata(ds, ds_identifier)) break if ignore_subdatasets: # all done now return meta from datalad.metadata.parsers.aggregate import MetadataParser as AggregateParser agg_parser = AggregateParser(ds) if agg_parser.has_metadata(): agg_meta = agg_parser.get_metadata(ds_identifier) # try hard to keep things a simple non-nested list if isinstance(agg_meta, list): meta.extend(agg_meta) else: meta.append(agg_meta) return meta def _cached_load_document(url): """Loader of pyld document from a url, which caches loaded instance on disk """ doc_fname = _get_schema_url_cache_filename(url) doc = None if os.path.exists(doc_fname): try: lgr.debug("use cached request result to '%s' from %s", url, doc_fname) doc = pickle.load(open(doc_fname, 'rb')) except Exception as e: # it is OK to ignore any error and fall back on the true source lgr.warning( "cannot load cache from '%s', fall back on schema download: %s", doc_fname, exc_str(e)) if doc is None: from pyld.jsonld import load_document doc = load_document(url) assure_dir(dirname(doc_fname)) # use pickle to store the entire request result dict pickle.dump(doc, open(doc_fname, 'wb')) lgr.debug("stored result of request to '{}' in {}".format(url, doc_fname)) return doc def _get_schema_url_cache_filename(url): """Return a filename where to cache schema doc from a url""" cache_dir = opj(cfg.obtain('datalad.locations.cache'), 'schema') doc_fname = opj( cache_dir, '{}-{}.p{}'.format( urlsplit(url).netloc, md5(url.encode('utf-8')).hexdigest(), pickle.HIGHEST_PROTOCOL) ) return doc_fname def flatten_metadata_graph(obj): from pyld import jsonld # simplify graph into a sequence of one dict per known dataset, even # if multiple meta data set from different sources exist for the same # dataset. # cache schema requests; this also avoid the need for network access # for previously "visited" schemas jsonld.set_document_loader(_cached_load_document) # TODO cache entire graphs to prevent repeated term resolution for # subsequent calls return jsonld.flatten(obj, ctx={"@context": "http://schema.org/"}) def get_native_metadata(ds, guess_type=False, ds_identifier=None): """Parse a dataset to gather its native metadata Returns ------- List Each item in the list is a metadata dictionary (JSON-LD compliant). The first items corresponds to the annex-based metadata of the dataset. The last items contains the native metadata of the dataset content. Any additional items correspond to subdataset metadata sets. """ if ds_identifier is None: ds_identifier = ds.id # using a list, because we could get multiple sets of meta data per # dataset, and we want to quickly collect them without having to do potentially # complex graph merges meta = [] # get native metadata nativetypes = get_metadata_type(ds, guess=guess_type) if not nativetypes: return meta # keep local, who knows what some parsers might pull in from . import parsers for nativetype in nativetypes: if nativetype == 'aggregate': # this is special and needs to be ignored here, even if it was # configured. reason: this parser runs anyway in get_metadata() continue pmod = import_module('.{}'.format(nativetype), package=parsers.__package__) try: native_meta = pmod.MetadataParser(ds).get_metadata(ds_identifier) except Exception as e: lgr.error('failed to get native metadata ({}): {}'.format(nativetype, exc_str(e))) continue if native_meta: # TODO here we could apply a "patch" to the native metadata, if desired # try hard to keep things a simple non-nested list if isinstance(native_meta, list): meta.extend(native_meta) else: meta.append(native_meta) return meta datalad-0.4.1/datalad/metadata/aggregate.py000066400000000000000000000206321301115270200205530ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Interface for aggregating metadata """ __docformat__ = 'restructuredtext' import os from os.path import join as opj, exists, relpath, dirname from datalad.interface.base import Interface from datalad.interface.utils import handle_dirty_dataset from datalad.interface.common_opts import recursion_limit, recursion_flag from datalad.interface.common_opts import if_dirty_opt from datalad.interface.common_opts import nosave_opt from datalad.utils import with_pathsep as _with_sep from datalad.distribution.dataset import datasetmethod, EnsureDataset, \ Dataset, require_dataset from ..support.param import Parameter from ..support.constraints import EnsureNone from datalad.support.exceptions import CommandError from ..log import lgr from . import get_metadata, metadata_filename, metadata_basepath, is_implicit_metadata from datalad.support.json_py import dump as jsondump def _store_json(ds, path, meta): if not exists(path): os.makedirs(path) fname = opj(path, metadata_filename) jsondump(meta, fname) # stage potential changes ds.repo.add(fname, git=True) class AggregateMetaData(Interface): """Aggregate meta data of a dataset for later query. By default meta data is aggregated across all configured native meta data sources. Optionally, the type of available meta data can guessed, if no types are configures. Moreover, it is possible to aggregate meta data from any subdatasets into the superdataset, in order to facilitate data discovery without having to obtain any subdataset. Returns ------- List Any datasets where (updated) aggregated meta data was saved. """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc="""specify the dataset to perform the install operation on. If no dataset is given, an attempt is made to identify the dataset based on the current working directory and/or the `path` given""", constraints=EnsureDataset() | EnsureNone()), guess_native_type=Parameter( args=("--guess-native-type",), action="store_true", doc="""guess native meta data type of datasets, if none is configured. With a configured, or auto-detected meta data type, no native meta data will be aggregated."""), recursive=recursion_flag, recursion_limit=recursion_limit, save=nosave_opt, if_dirty=if_dirty_opt, ) @staticmethod @datasetmethod(name='aggregate_metadata') def __call__( dataset, guess_native_type=False, recursive=False, recursion_limit=None, save=True, if_dirty='save-before'): ds = require_dataset( dataset, check_installed=True, purpose='meta data aggregation') modified_ds = [] if ds.id is None: lgr.warning('%s has not configured ID, skipping.', dataset) return modified_ds # make sure we get to an expected state handle_dirty_dataset(ds, if_dirty) # if you want to modify the behavior of get_subdataset() make sure # there is a way to return the subdatasets DEPTH FIRST! ds_meta = {} for subds_path in ds.get_subdatasets( fulfilled=True, absolute=False, recursive=recursive, recursion_limit=recursion_limit): subds = Dataset(opj(ds.path, subds_path)) if subds.id is None: # nothing to worry about, any meta data from below this will be # injected upstairs lgr.debug('skipping non-dataset at %s', subds.path) continue else: lgr.info('aggregating meta data for %s', subds) metapath = opj(subds.path, metadata_basepath) handle_dirty_dataset(subds, if_dirty) # # Phase 1: aggregate the within-dataset meta data, and store # within the dataset # # pull out meta data from subds only (no subdatasets) _within_metadata_store( subds, guess_native_type, metapath) # # Phase 2: store everything that is in the look up and belongs into # this dataset # _dump_submeta(subds, ds_meta, subds_path, save, modified_ds) # save state of modified dataset, all we modified has been staged # already # we need to save before extracting to full metadata for upstairs # consumption to get the versions right modified_ds = _save_helper(subds, save, modified_ds) # # Phase 3: obtain all aggregated meta data from this dataset, and # keep in lookup to escalate it upstairs # ds_meta[subds_path] = get_metadata( subds, guess_type=False, ignore_subdatasets=False, ignore_cache=False) lgr.info('aggregating meta data for %s', ds) # pull out meta data from parent only (no subdatasets) _within_metadata_store( ds, guess_native_type, opj(ds.path, metadata_basepath)) # and lastly the subdatasets of the parent _dump_submeta(ds, ds_meta, '', save, modified_ds) # everything should be stored somewhere by now assert not len(ds_meta) # save the parent modified_ds = _save_helper(ds, save, modified_ds) def _within_metadata_store(ds, guess_native_type, metapath): meta = get_metadata( ds, guess_type=guess_native_type, ignore_subdatasets=True, ignore_cache=True) # strip git-based version info from the meta data that is cached # in the dataset itself -- this will be outdated the second we # commit below for m in meta: if not is_implicit_metadata(m): continue for prop in ('dcterms:modified', 'version'): if prop in m: del m[prop] _store_json(ds, metapath, meta) def _save_helper(ds, save, modified_ds): old_state = ds.repo.get_hexsha() if save and ds.repo.repo.is_dirty( index=True, working_tree=False, submodules=True): ds.save(message="[DATALAD] aggregated meta data") if ds.repo.get_hexsha() != old_state: modified_ds.append(ds) return modified_ds def _dump_submeta(ds, submetas, matchpath, save, modified_ds): known_subds = list(submetas.keys()) for p in known_subds: smeta = submetas[p] if matchpath and not p.startswith(_with_sep(matchpath)): continue subds_relpath = relpath(p, matchpath) # inject proper inter-dataset relationships for m in smeta: # skip non-implicit if not is_implicit_metadata(m): continue if 'dcterms:isPartOf' not in m and m.get('type', None) == 'Dataset': m['dcterms:isPartOf'] = ds.id sp = opj(ds.path, metadata_basepath, subds_relpath) _store_json(ds, sp, smeta) # stage potential changes in the subdataset try: ds.repo.add(subds_relpath, git=True) except CommandError: # it can blow if we skipped a non-dataset submodule # in this case we need to find the chain of submodules leading to it and # save then bottom-up testpath = dirname(subds_relpath) while testpath: repo = ds.get_containing_subdataset(testpath) repo.repo.add(relpath(subds_relpath, testpath), git=True) modified_ds = _save_helper(repo, save, modified_ds) # see if there is anything left... # IMPORTANT to go with relpath to actually get to an empty # string eventually testpath = dirname(relpath(repo.path, ds.path)) # removed stored item from lookup del submetas[p] return modified_ds datalad-0.4.1/datalad/metadata/parsers/000077500000000000000000000000001301115270200177275ustar00rootroot00000000000000datalad-0.4.1/datalad/metadata/parsers/__init__.py000066400000000000000000000011241301115270200220360ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Metadata parsers""" # the following imports are only needed for knowing which meta data types # we can try to guess from . import bids from . import frictionless_datapackage from . import datalad_rfc822 datalad-0.4.1/datalad/metadata/parsers/aggregate.py000066400000000000000000000072531301115270200222360ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Parser for datalad's own aggregated metadata""" from os.path import join as opj from datalad.utils import find_files from datalad.support.json_py import load as jsonload from datalad.metadata import _simplify_meta_data_structure from datalad.metadata import is_implicit_metadata from datalad.metadata.parsers.base import BaseMetadataParser from datalad.metadata import _get_base_dataset_metadata # XXX could be moved to aggregate parser... def _adjust_subdataset_location(meta, subds_relpath): # find implicit meta data for all contained subdatasets for m in meta: # skip non-implicit if not is_implicit_metadata(m): continue # prefix all subdataset location information with the relpath of this # subdataset if 'dcterms:hasPart' in m: parts = m['dcterms:hasPart'] if not isinstance(parts, list): parts = [parts] for p in parts: if 'location' not in p: continue loc = p.get('location', subds_relpath) if loc != subds_relpath: p['location'] = opj(subds_relpath, loc) class MetadataParser(BaseMetadataParser): def get_core_metadata_filenames(self): return list(find_files( 'meta\.json', topdir=opj(self.ds.path, '.datalad', 'meta'), exclude=None, exclude_vcs=False, exclude_datalad=False, dirs=False)) def get_metadata(self, dsid=None, full=False): base_meta = _get_base_dataset_metadata(dsid if dsid else self.ds.id) meta = [base_meta] basepath = opj(self.ds.path, '.datalad', 'meta') parts = [] for subds_meta_fname in self.get_core_metadata_filenames(): # get the part between the 'meta' dir and the filename # which is the subdataset mountpoint subds_path = subds_meta_fname[len(basepath) + 1:-10] if not subds_path: # this is a potentially existing cache of the native meta data # of the superdataset, not for us... continue submeta_info = { 'location': subds_path} # load aggregated meta data subds_meta = jsonload(subds_meta_fname) # we cannot simply append, or we get weired nested graphs # proper way would be to expand the JSON-LD, extend the list and # compact/flatten at the end. However assuming a single context # we can cheat. subds_meta = _simplify_meta_data_structure(subds_meta) _adjust_subdataset_location(subds_meta, subds_path) # sift through all meta data sets look for a meta data set that # knows about being part of this dataset, so we record its @id as # part for md in subds_meta: cand_id = md.get('dcterms:isPartOf', None) if cand_id == dsid and '@id' in md: submeta_info['@id'] = md['@id'] break if subds_meta: meta.extend(subds_meta) parts.append(submeta_info) if len(parts): if len(parts) == 1: parts = parts[0] base_meta['dcterms:hasPart'] = parts return meta datalad-0.4.1/datalad/metadata/parsers/base.py000066400000000000000000000044671301115270200212260ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Metadata parser base class""" from os.path import exists, join as opj from datalad.metadata import _get_base_dataset_metadata class BaseMetadataParser(object): # subclass can use this to provide a simple list of candidate files # to check _core_metadata_filenames = [] def __init__(self, ds): """ Parameters ---------- ds : dataset instance Dataset to extract metadata from. """ self.ds = ds def has_metadata(self): """Returns whether a dataset provides this kind meta data""" # default implementation, override with more efficient, if possible return len(self.get_core_metadata_filenames()) > 0 def get_core_metadata_filenames(self): """List of absolute filenames making up the core meta data source""" # default implementation, override if _core_metadata_filenames is not # used dspath = self.ds.path fnames = [opj(dspath, f) for f in self._core_metadata_filenames] return [f for f in fnames if exists(f)] def get_metadata_filenames(self): """List of absolute filenames making up the full meta data source""" # default implementation: core == full return self.get_core_metadata_filenames() def get_metadata(self, dsid=None, full=False): """Returns JSON-LD compliant meta data structure Parameter --------- full : bool If True, all intelligible meta data is return. Otherwise only meta data deemed essential by the author is returned. Returns ------- dict JSON-LD compliant """ if dsid is None: dsid = self.ds.id meta = _get_base_dataset_metadata(dsid) if self.has_metadata(): meta = self._get_metadata(dsid, meta, full) return meta def _get_metadata(self, dsid, basemeta, full): raise NotImplementedError datalad-0.4.1/datalad/metadata/parsers/bids.py000066400000000000000000000034101301115270200212200ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """BIDS metadata parser (http://bids.neuroimaging.io)""" from datalad.support.json_py import load as jsonload from datalad.metadata.parsers.base import BaseMetadataParser class MetadataParser(BaseMetadataParser): _core_metadata_filenames = ['dataset_description.json'] def _get_metadata(self, ds_identifier, meta, full): bids = jsonload( self.get_core_metadata_filenames()[0]) # TODO maybe normalize labels of standard licenses to definition URIs # perform mapping for bidsterm, dataladterm in (('Name', 'name'), ('License', 'license'), ('Authors', 'author'), ('ReferencesAndLinks', 'citation'), ('Funding', 'foaf:fundedBy'), ('Description', 'description')): if bidsterm in bids: meta[dataladterm] = bids[bidsterm] compliance = ["http://docs.datalad.org/metadata.html#v0-1"] # special case if bids.get('BIDSVersion'): compliance.append( 'http://bids.neuroimaging.io/bids_spec{}.pdf'.format( bids['BIDSVersion'].strip())) else: compliance.append('http://bids.neuroimaging.io') meta['dcterms:conformsTo'] = compliance return meta datalad-0.4.1/datalad/metadata/parsers/datalad_rfc822.py000066400000000000000000000067621301115270200227740ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Parser for RFC822-based metadata specifications This is inspired by (and very similiar to) Debian's package meta data format. The main difference is that information spread across multiple files in Debian packages, is concentrated in one file. The main advantage of this format is that it is proven to be hand-editable, i.e. can be composed from scratch, by hand, in an editor -- with a good chance of producing syntax-compliant content with the first attempt. """ import email from os.path import join as opj from datalad.metadata.parsers.base import BaseMetadataParser from datalad.interface.base import dedent_docstring def _split_list_field(content): return [i.strip() for i in content.split(',') if i.strip()] def _beautify_multiline_field(content): content = dedent_docstring(content) lines = content.split('\n') title = '' if len(lines): title = lines[0] if len(lines) > 1: content = '' for l in lines[1:]: l = l.strip() content = '{}{}{}'.format( content, ' ' if len(content) and l != '.' and content[-1] != '\n' else '', l if l != '.' else '\n') return title, content class MetadataParser(BaseMetadataParser): _metadata_compliance = "http://docs.datalad.org/metadata.html#v0-1" _core_metadata_filenames = [opj('.datalad', 'meta.rfc822')] def _get_metadata(self, ds_identifier, meta, full): spec = email.parser.Parser().parse( open(self.get_core_metadata_filenames()[0]), headersonly=True) # loop over all recognized headers and translate them for header, dataladterm in \ (('name', 'name'), ('license', 'license'), ('author', 'author'), ('maintainer', 'doap:maintainer'), ('audience', 'doap:audience'), ('homepage', 'doap:homepage'), ('version', 'doap:Version'), ('funding', 'foaf:fundedBy'), ('issue-tracker', 'bug-database'), ('cite-as', 'citation'), ('doi', 'sameAs'), ('description', None)): if not header in spec: continue content = spec[header] if header == 'description': short, long = _beautify_multiline_field(content) meta['doap:shortdesc'] = short meta['description'] = long elif header == 'license': # TODO if title looks like a URL, use it as @id label, desc = _beautify_multiline_field(content) if label: meta[dataladterm] = [label, desc] else: meta[dataladterm] = desc elif header in ('maintainer', 'author'): meta[dataladterm] = _split_list_field(content) elif header == 'doi': meta[dataladterm] = 'http://dx.doi.org/{}'.format(content) else: meta[dataladterm] = content meta['dcterms:conformsTo'] = self._metadata_compliance return meta datalad-0.4.1/datalad/metadata/parsers/frictionless_datapackage.py000066400000000000000000000043471301115270200253220ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Parser for friction-less data packages (http://specs.frictionlessdata.io/data-packages) """ from datalad.support.json_py import load as jsonload from datalad.metadata.parsers.base import BaseMetadataParser def _compact_author(obj): if isinstance(obj, dict): bits = [] if 'name' in obj: bits.append(obj['name']) if 'email' in obj: bits.append('<{}>'.format(obj['email'])) if 'web' in obj: bits.append('({})'.format(obj['web'])) return ' '.join(bits) else: return obj def _compact_license(obj): if isinstance(obj, dict): # With obj itself if no url or type return obj.get('url', obj.get('type', obj)) else: return obj class MetadataParser(BaseMetadataParser): _core_metadata_filenames = ['datapackage.json'] def _get_metadata(self, ds_identifier, meta, full): foreign = jsonload( self.get_core_metadata_filenames()[0]) for term in ( 'name', 'title', 'description', 'keywords', 'version', 'homepage'): if term in foreign: meta[term] = foreign[term] if 'author' in foreign: meta['author'] = _compact_author(foreign['author']) if 'contributors' in foreign: meta['contributors'] = [_compact_author(c) for c in foreign['contributors']] # two license terms were supported at some point if 'license' in foreign: meta['license'] = _compact_license(foreign['license']) if 'licenses' in foreign: meta['license'] = [_compact_license(l) for l in foreign['licenses']] meta['dcterms:conformsTo'] = [ 'http://specs.frictionlessdata.io/data-packages', 'http://docs.datalad.org/metadata.html#v0-1'] return meta datalad-0.4.1/datalad/metadata/parsers/tests/000077500000000000000000000000001301115270200210715ustar00rootroot00000000000000datalad-0.4.1/datalad/metadata/parsers/tests/__init__.py000066400000000000000000000006341301115270200232050ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Metadata parsers""" datalad-0.4.1/datalad/metadata/parsers/tests/test_aggregate.py000066400000000000000000000025761301115270200244420ustar00rootroot00000000000000# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test aggregate meta data parser """ import os from os.path import join as opj from datalad.distribution.dataset import Dataset from datalad.api import create from datalad.metadata.parsers.aggregate import MetadataParser from nose.tools import assert_true, assert_false, assert_equal from datalad.tests.utils import with_tempfile @with_tempfile(mkdir=True) def test_basic(path): ds = Dataset(path).create() p = MetadataParser(ds) assert_false(p.has_metadata()) mpath = opj(ds.path, '.datalad', 'meta', 'something', 'deep') os.makedirs(mpath) assert_false(p.has_metadata()) with open(opj(mpath, 'meta.json'), 'w') as fp: fp.write('{"name": "testmonkey", "dcterms:isPartOf": "%s", "@id": "unique"}' % ds.id) assert_true(p.has_metadata()) #from datalad.metadata import get_metadata #from json import dumps #print(dumps(get_metadata(ds), indent=1)) #assert_equal( # p.get_metadata(), # [{'dcterms:hasPart': {'location': 'something/deep'}}]) datalad-0.4.1/datalad/metadata/parsers/tests/test_bids.py000066400000000000000000000043431301115270200234270ustar00rootroot00000000000000# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test BIDS meta data parser """ from os.path import join as opj from simplejson import dumps from datalad.distribution.dataset import Dataset from datalad.metadata.parsers.bids import MetadataParser from nose.tools import assert_true, assert_false, assert_equal from datalad.tests.utils import with_tree, with_tempfile @with_tree(tree={'dataset_description.json': '{}'}) def test_has_metadata(path): ds = Dataset(path) p = MetadataParser(ds) assert_true(p.has_metadata()) assert_equal(p.get_core_metadata_filenames(), [opj(path, 'dataset_description.json')]) @with_tempfile(mkdir=True) def test_has_no_metadata(path): ds = Dataset(path) p = MetadataParser(ds) assert_false(p.has_metadata()) assert_equal(p.get_core_metadata_filenames(), []) @with_tree(tree={'dataset_description.json': """ { "Name": "studyforrest_phase2", "BIDSVersion": "1.0.0-rc3", "Description": "Some description", "License": "PDDL", "Authors": [ "Mike One", "Anna Two" ], "Funding": "We got money from collecting plastic bottles", "ReferencesAndLinks": [ "http://studyforrest.org" ] } """}) def test_get_metadata(path): ds = Dataset(path) meta = MetadataParser(ds).get_metadata('ID') assert_equal( dumps(meta, sort_keys=True, indent=2), """\ { "@context": { "@vocab": "http://schema.org/", "doap": "http://usefulinc.com/ns/doap#" }, "@id": "ID", "author": [ "Mike One", "Anna Two" ], "citation": [ "http://studyforrest.org" ], "dcterms:conformsTo": [ "http://docs.datalad.org/metadata.html#v0-1", "http://bids.neuroimaging.io/bids_spec1.0.0-rc3.pdf" ], "description": "Some description", "foaf:fundedBy": "We got money from collecting plastic bottles", "license": "PDDL", "name": "studyforrest_phase2" }""") datalad-0.4.1/datalad/metadata/parsers/tests/test_frictionless_datapackage.py000066400000000000000000000054621301115270200275220ustar00rootroot00000000000000# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test frictionless datapackage meta data parser """ from os.path import join as opj from simplejson import dumps from datalad.distribution.dataset import Dataset from datalad.metadata.parsers.frictionless_datapackage import MetadataParser from nose.tools import assert_true, assert_false, assert_equal from datalad.tests.utils import with_tree, with_tempfile @with_tree(tree={'datapackage.json': '{"name": "some"}'}) def test_has_metadata(path): ds = Dataset(path) p = MetadataParser(ds) assert_true(p.has_metadata()) assert_equal(p.get_metadata_filenames(), [opj(path, 'datapackage.json')]) @with_tempfile(mkdir=True) def test_has_no_metadata(path): ds = Dataset(path) p = MetadataParser(ds) assert_false(p.has_metadata()) assert_equal(p.get_core_metadata_filenames(), []) # bits from examples and the specs @with_tree(tree={'datapackage.json': """ { "name": "cpi", "title": "Annual Consumer Price Index (CPI)", "description": "Annual Consumer Price Index (CPI) for most countries in the world. Reference year is 2005.", "license" : { "type": "odc-pddl", "url": "http://opendatacommons.org/licenses/pddl/" }, "keywords": [ "CPI", "World", "Consumer Price Index", "Annual Data", "The World Bank" ], "version": "2.0.0", "last_updated": "2014-09-22", "contributors": [ { "name": "Joe Bloggs", "email": "joe@example.com", "web": "http://www.example.com" } ], "author": "Jane Doe " } """}) def test_get_metadata(path): ds = Dataset(path) p = MetadataParser(ds) meta = p.get_metadata('ID') assert_equal( dumps(meta, sort_keys=True, indent=2), """\ { "@context": { "@vocab": "http://schema.org/", "doap": "http://usefulinc.com/ns/doap#" }, "@id": "ID", "author": "Jane Doe ", "contributors": [ "Joe Bloggs (http://www.example.com)" ], "dcterms:conformsTo": [ "http://specs.frictionlessdata.io/data-packages", "http://docs.datalad.org/metadata.html#v0-1" ], "description": "Annual Consumer Price Index (CPI) for most countries in the world. Reference year is 2005.", "keywords": [ "CPI", "World", "Consumer Price Index", "Annual Data", "The World Bank" ], "license": "http://opendatacommons.org/licenses/pddl/", "name": "cpi", "title": "Annual Consumer Price Index (CPI)", "version": "2.0.0" }""") datalad-0.4.1/datalad/metadata/parsers/tests/test_rfc822.py000066400000000000000000000066011301115270200235130ustar00rootroot00000000000000# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test BIDS meta data parser """ from os.path import join as opj from simplejson import dumps from datalad.distribution.dataset import Dataset from datalad.metadata.parsers.datalad_rfc822 import MetadataParser from nose.tools import assert_true, assert_false, assert_equal from datalad.tests.utils import with_tree, with_tempfile @with_tree(tree={'.datalad': {'meta.rfc822': ''}}) def test_has_metadata(path): ds = Dataset(path) p = MetadataParser(ds) assert_true(p.has_metadata()) assert_equal(p.get_core_metadata_filenames(), [opj(path, '.datalad', 'meta.rfc822')]) @with_tempfile(mkdir=True) def test_has_no_metadata(path): ds = Dataset(path) p = MetadataParser(ds) assert_false(p.has_metadata()) assert_equal(p.get_core_metadata_filenames(), []) @with_tree(tree={'.datalad': {'meta.rfc822': """\ Name: studyforrest_phase2 Version: 1.0.0-rc3 Description: Basic summary A text with arbitrary length and content that can span multiple . paragraphs (this is a new one) License: CC0 The person who associated a work with this deed has dedicated the work to the public domain by waiving all of his or her rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law. . You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission. Maintainer: Mike One , Anna Two , Homepage: http://studyforrest.org Funding: BMBFGQ1411, NSF 1429999 Issue-Tracker: https://github.com/psychoinformatics-de/studyforrest-data-phase2/issues Cite-As: Cool (2016) DOI: 10.5281/zenodo.48421 """}}) def test_get_metadata(path): ds = Dataset(path) meta = MetadataParser(ds).get_metadata('ID') assert_equal( dumps(meta, sort_keys=True, indent=2), """\ { "@context": { "@vocab": "http://schema.org/", "doap": "http://usefulinc.com/ns/doap#" }, "@id": "ID", "bug-database": "https://github.com/psychoinformatics-de/studyforrest-data-phase2/issues", "citation": "Cool (2016)", "dcterms:conformsTo": "http://docs.datalad.org/metadata.html#v0-1", "description": "A text with arbitrary length and content that can span multiple\\nparagraphs (this is a new one)", "doap:Version": "1.0.0-rc3", "doap:homepage": "http://studyforrest.org", "doap:maintainer": [ "Mike One ", "Anna Two " ], "doap:shortdesc": "Basic summary", "foaf:fundedBy": "BMBFGQ1411, NSF 1429999", "license": [ "CC0", "The person who associated a work with this deed has dedicated the work to the public domain by waiving all of his or her rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law.\\nYou can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission." ], "name": "studyforrest_phase2", "sameAs": "http://dx.doi.org/10.5281/zenodo.48421" }""") datalad-0.4.1/datalad/metadata/search.py000066400000000000000000000412721301115270200200750ustar00rootroot00000000000000# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Interface for managing metadata """ __docformat__ = 'restructuredtext' import os import re import sys from operator import itemgetter from os.path import join as opj, exists from six import string_types from six import text_type from six import iteritems from six import reraise from six import PY3 from datalad.interface.base import Interface from datalad.distribution.dataset import Dataset from datalad.distribution.dataset import datasetmethod, EnsureDataset, \ require_dataset from datalad.distribution.utils import get_git_dir from ..support.param import Parameter from ..support.constraints import EnsureNone from ..support.constraints import EnsureChoice from ..log import lgr from . import get_metadata, flatten_metadata_graph, pickle from datalad.consts import LOCAL_CENTRAL_PATH from datalad.utils import assure_list from datalad.utils import get_path_prefix from datalad.support.exceptions import NoDatasetArgumentFound from datalad.support import ansi_colors from datalad.ui import ui class Search(Interface): """Search within available in datasets' meta data Yields ------ location : str (relative) path to the dataset report : dict fields which were requested by `report` option """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc="""specify the dataset to perform the query operation on. If no dataset is given, an attempt is made to identify the dataset based on the current working directory and/or the `path` given""", constraints=EnsureDataset() | EnsureNone()), match=Parameter( args=("match",), metavar='STRING', nargs="+", doc="a string (or a regular expression if " "[PY: `regex=True` PY][CMD: --regex CMD]) to search for " "in all meta data values. If multiple provided, all must have " "a match among some fields of a dataset"), #match=Parameter( # args=('-m', '--match',), # metavar='REGEX', # action='append', # nargs=2, # doc="""Pair of two regular expressions to match a property and its # value.[CMD: This option can be given multiple times CMD]"""), search=Parameter( args=('-s', '--search'), metavar='PROPERTY', action='append', # could also be regex doc="""name of the property to search for any match.[CMD: This option can be given multiple times. CMD] By default, all properties are searched."""), report=Parameter( args=('-r', '--report'), metavar='PROPERTY', action='append', # could also be regex doc="""name of the property to report for any match.[CMD: This option can be given multiple times. CMD] If '*' is given, all properties are reported."""), report_matched=Parameter( args=('-R', '--report-matched',), action="store_true", doc="""flag to report those fields which have matches. If `report` option values are provided, union of matched and those in `report` will be output"""), # Theoretically they should be CMDLINE specific I guess? format=Parameter( args=('-f', '--format'), constraints=EnsureChoice('custom', 'json', 'yaml'), doc="""format for output.""" ), regex=Parameter( args=("--regex",), action="store_true", doc="flag for STRING to be used as a (Python) regular expression " "which should match the value"), ) @staticmethod @datasetmethod(name='search') def __call__(match, dataset=None, search=None, report=None, report_matched=False, format='custom', regex=False): lgr.debug("Initiating search for match=%r and dataset %r", match, dataset) try: ds = require_dataset(dataset, check_installed=True, purpose='dataset search') if ds.id is None: raise NoDatasetArgumentFound( "This does not seem to be a dataset (no DataLad dataset ID " "found). 'datalad create --force %s' can initialize " "this repository as a DataLad dataset" % ds.path) except NoDatasetArgumentFound: exc_info = sys.exc_info() if dataset is None: if not ui.is_interactive: raise NoDatasetArgumentFound( "No DataLad dataset found. Specify a dataset to be " "searched, or run interactively to get assistance " "installing a queriable superdataset." ) # none was provided so we could ask user either he possibly wants # to install our beautiful mega-duper-super-dataset? # TODO: following logic could possibly benefit other actions. if os.path.exists(LOCAL_CENTRAL_PATH): central_ds = Dataset(LOCAL_CENTRAL_PATH) if central_ds.is_installed(): if ui.yesno( title="No DataLad dataset found at current location", text="Would you like to search the DataLad " "superdataset at %r?" % LOCAL_CENTRAL_PATH): pass else: reraise(*exc_info) else: raise NoDatasetArgumentFound( "No DataLad dataset found at current location. " "The DataLad superdataset location %r exists, " "but does not contain an dataset." % LOCAL_CENTRAL_PATH) elif ui.yesno( title="No DataLad dataset found at current location", text="Would you like to install the DataLad " "superdataset at %r?" % LOCAL_CENTRAL_PATH): from datalad.api import install central_ds = install(LOCAL_CENTRAL_PATH, source='///') ui.message( "From now on you can refer to this dataset using the " "label '///'" ) else: reraise(*exc_info) lgr.info( "Performing search using DataLad superdataset %r", central_ds.path ) for res in central_ds.search( match, search=search, report=report, report_matched=report_matched, format=format, regex=regex): yield res return else: raise cache_dir = opj(opj(ds.path, get_git_dir(ds.path)), 'datalad', 'cache') mcache_fname = opj(cache_dir, 'metadata.p%d' % pickle.HIGHEST_PROTOCOL) meta = None if os.path.exists(mcache_fname): lgr.debug("use cached metadata of '{}' from {}".format(ds, mcache_fname)) meta, checksum = pickle.load(open(mcache_fname, 'rb')) # TODO add more sophisticated tests to decide when the cache is no longer valid if checksum != ds.repo.get_hexsha(): # errrr, try again below meta = None # don't put in 'else', as yet to be written tests above might fail and require # regenerating meta data if meta is None: lgr.info("Loading and caching local meta-data... might take a few seconds") if not exists(cache_dir): os.makedirs(cache_dir) meta = get_metadata(ds, guess_type=False, ignore_subdatasets=False, ignore_cache=False) # merge all info on datasets into a single dict per dataset meta = flatten_metadata_graph(meta) # extract graph, if any meta = meta.get('@graph', meta) # build simple queriable representation if not isinstance(meta, list): meta = [meta] # sort entries by location (if present) sort_keys = ('location', 'description', 'id') meta = sorted(meta, key=lambda m: tuple(m.get(x, "") for x in sort_keys)) # use pickle to store the optimized graph in the cache pickle.dump( # graph plus checksum from what it was built (meta, ds.repo.get_hexsha()), open(mcache_fname, 'wb')) lgr.debug("cached meta data graph of '{}' in {}".format(ds, mcache_fname)) if report in ('', ['']): report = [] elif report and not isinstance(report, list): report = [report] match = assure_list(match) search = assure_list(search) # convert all to lower case for case insensitive matching search = {x.lower() for x in search} def get_in_matcher(m): """Function generator to provide closure for a specific value of m""" mlower = m.lower() def matcher(s): return mlower in s.lower() return matcher matchers = [ re.compile(match_).search if regex else get_in_matcher(match_) for match_ in match ] # location should be reported relative to current location # We will assume that noone chpwd while we are yielding ds_path_prefix = get_path_prefix(ds.path) # So we could provide a useful message whenever there were not a single # dataset with specified `--search` properties observed_properties = set() # for every meta data set for mds in meta: hit = False hits = [False] * len(matchers) matched_fields = set() if not mds.get('type', mds.get('schema:type', None)) == 'Dataset': # we are presently only dealing with datasets continue # TODO consider the possibility of nested and context/graph dicts # but so far we were trying to build simple lists of dicts, as much # as possible if not isinstance(mds, dict): raise NotImplementedError("nested meta data is not yet supported") # manual loop for now for k, v in iteritems(mds): if search: k_lower = k.lower() if k_lower not in search: if observed_properties is not None: # record for providing a hint later observed_properties.add(k_lower) continue # so we have a hit, no need to track observed_properties = None if isinstance(v, dict) or isinstance(v, list): v = text_type(v) for imatcher, matcher in enumerate(matchers): if matcher(v): hits[imatcher] = True matched_fields.add(k) if all(hits): hit = True # no need to do it longer than necessary if not report_matched: break if hit: location = mds.get('location', '.') report_ = matched_fields.union(report if report else {}) \ if report_matched else report if report_ == ['*']: report_dict = mds elif report_: report_dict = {k: mds[k] for k in report_ if k in mds} if report_ and not report_dict: lgr.debug( 'meta data match for %s, but no to-be-reported ' 'properties (%s) found. Present properties: %s', location, ", ".join(report_), ", ".join(sorted(mds)) ) else: report_dict = {} # it was empty but not None -- asked to # not report any specific field if isinstance(location, (list, tuple)): # could be that the same dataset installed into multiple # locations. For now report them separately for l in location: yield opj(ds_path_prefix, l), report_dict else: yield opj(ds_path_prefix, location), report_dict if search and observed_properties is not None: import difflib suggestions = { s: difflib.get_close_matches(s, observed_properties) for s in search } suggestions_str = "\n ".join( "%s for %s" % (", ".join(choices), s) for s, choices in iteritems(suggestions) if choices ) lgr.warning( "Found no properties which matched one of the one you " "specified (%s). May be you meant one among: %s.\n" "Suggestions:\n" " %s", ", ".join(search), ", ".join(observed_properties), suggestions_str if suggestions_str.strip() else "none" ) @staticmethod def result_renderer_cmdline(res, cmdlineargs): from datalad.ui import ui if res is None: res = [] format = cmdlineargs.format or 'custom' if format == 'custom': if cmdlineargs.report in ('*', ['*']) \ or cmdlineargs.report_matched \ or (cmdlineargs.report is not None and len(cmdlineargs.report) > 1): # multiline if multiple were requested and we need to disambiguate ichr = jchr = '\n' fmt = ' {k}: {v}' else: jchr = ', ' ichr = ' ' fmt = '{v}' anything = False for location, r in res: # XXX Yarik thinks that Match should be replaced with actual path to the dataset ui.message('{}{}{}{}'.format( ansi_colors.color_word(location, ansi_colors.DATASET), ':' if r else '', ichr, jchr.join( [ fmt.format( k=ansi_colors.color_word(k, ansi_colors.FIELD), v=pretty_bytes(r[k])) for k in sorted(r) ]))) anything = True if not anything: ui.message("Nothing to report") elif format == 'json': import json ui.message(json.dumps(list(map(itemgetter(1), res)), indent=2)) elif format == 'yaml': import yaml lgr.warning("yaml output support is not yet polished") ui.message(yaml.safe_dump(list(map(itemgetter(1), res)), allow_unicode=True)) _lines_regex = re.compile('[\n\r]') def pretty_bytes(s): """Helper to provide sensible rendering for lists, dicts, and unicode encoded into byte-stream (why really???) """ if isinstance(s, list): return ", ".join(map(pretty_bytes, s)) elif isinstance(s, dict): return pretty_bytes(["%s=%s" % (pretty_bytes(k), pretty_bytes(v)) for k, v in s.items()]) elif isinstance(s, text_type): s_ = (os.linesep + " ").join(_lines_regex.split(s)) try: if PY3: return s_ return s_.encode('utf-8') except UnicodeEncodeError: lgr.warning("Failed to encode value correctly. Ignoring errors in encoding") # TODO: get current encoding return s_.encode('utf-8', 'ignore') if isinstance(s_, string_types) else "ERROR" else: return str(s).encode() datalad-0.4.1/datalad/metadata/tests/000077500000000000000000000000001301115270200174125ustar00rootroot00000000000000datalad-0.4.1/datalad/metadata/tests/__init__.py000066400000000000000000000000001301115270200215110ustar00rootroot00000000000000datalad-0.4.1/datalad/metadata/tests/test_base.py000066400000000000000000000360121301115270200217370ustar00rootroot00000000000000# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # -*- coding: utf-8 -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Test meta data """ import logging import pickle import os from mock import patch from operator import itemgetter from os.path import join as opj, exists from datalad.api import Dataset, aggregate_metadata, install from datalad.metadata import get_metadata_type, get_metadata from datalad.metadata import _cached_load_document from datalad.utils import swallow_logs from datalad.utils import chpwd from datalad.utils import assure_unicode from datalad.dochelpers import exc_str from datalad.tests.utils import with_tree, with_tempfile from datalad.tests.utils import assert_not_in from datalad.tests.utils import assert_in from datalad.tests.utils import swallow_outputs from datalad.tests.utils import skip_if_no_network from datalad.support.exceptions import InsufficientArgumentsError from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from nose import SkipTest from nose.tools import assert_true, assert_equal, assert_raises, assert_false try: import pyld # do here to prevent pyld from being needed except ImportError as exc: raise SkipTest(exc_str(exc)) from datalad.api import search from datalad.api import search_ _dataset_hierarchy_template = { 'origin': { 'dataset_description.json': """ { "Name": "mother_äöü東" }""", 'datapackage.json': """ { "name": "MOTHER_äöü東", "keywords": ["example", "multitype metadata"] }""", 'sub': { 'dataset_description.json': """ { "Name": "child_äöü東" }""", 'subsub': { 'dataset_description.json': """ { "Name": "grandchild_äöü東" }"""}}}} @with_tempfile(mkdir=True) def test_get_metadata_type(path): # nothing set, nothing found assert_equal(get_metadata_type(Dataset(path)), []) os.makedirs(opj(path, '.datalad')) # got section, but no setting open(opj(path, '.datalad', 'config'), 'w').write('[metadata]\n') assert_equal(get_metadata_type(Dataset(path)), []) # minimal setting open(opj(path, '.datalad', 'config'), 'w+').write('[metadata]\nnativetype = mamboschwambo\n') assert_equal(get_metadata_type(Dataset(path)), ['mamboschwambo']) @with_tree(tree={ 'dataset_description.json': "{}", 'datapackage.json': '{"name": "some"}' }) def test_get_multiple_metadata_types(path): assert_equal( sorted(get_metadata_type(Dataset(path), guess=True)), ['bids', 'frictionless_datapackage']) @with_tree(tree={ 'origin': { 'dataset_description.json': """ { "Name": "the mother" }""", 'sub': { 'dataset_description.json': """ { "Name": "child" }"""}}}) def test_basic_metadata(path): ds = Dataset(opj(path, 'origin')) meta = get_metadata(ds) assert_equal(sorted(meta[0].keys()), ['@context', 'dcterms:conformsTo']) ds.create(force=True, save=False) # with subdataset sub = ds.create('sub', force=True, if_dirty='ignore') ds.save() meta = get_metadata(ds) assert_equal( sorted(meta[0].keys()), ['@context', '@id', 'availableFrom', 'dcterms:conformsTo', 'dcterms:modified', 'type', 'version']) assert_equal(meta[0]['type'], 'Dataset') # clone and get relationship info in metadata sibling = install(opj(path, 'sibling'), source=opj(path, 'origin')) sibling_meta = get_metadata(sibling) assert_equal(sibling_meta[0]['@id'], ds.id) # origin should learn about the clone sibling.repo.push(remote='origin', refspec='git-annex') meta = get_metadata(ds) assert_equal([m['@id'] for m in meta[0]['availableFrom']], [m['@id'] for m in sibling_meta[0]['availableFrom']]) meta = get_metadata(ds, guess_type=True) # without aggregation there is not trace of subdatasets in the metadata assert_not_in('dcterms:hasPart', meta[0]) @skip_if_no_network @with_tree(tree=_dataset_hierarchy_template) def test_aggregation(path): with chpwd(path): assert_raises(InsufficientArgumentsError, aggregate_metadata, None) # a hierarchy of three (super/sub)datasets, each with some native metadata ds = Dataset(opj(path, 'origin')).create(force=True) subds = ds.create('sub', force=True, if_dirty='ignore') subsubds = subds.create('subsub', force=True, if_dirty='ignore') # aggregate from bottom to top, guess native data, no compacting of graph # should yield 6 meta data sets, one implicit, and one native per dataset # and a second natiev set for the topmost dataset aggregate_metadata(ds, guess_native_type=True, recursive=True) # no only ask the top superdataset, no recursion, just reading from the cache meta = get_metadata( ds, guess_type=False, ignore_subdatasets=False, ignore_cache=False) assert_equal(len(meta), 10) # same schema assert_equal( 10, sum([s.get('@context', {'@vocab': None})['@vocab'] == 'http://schema.org/' for s in meta])) # three different IDs assert_equal(3, len(set([s.get('@id') for s in meta]))) # and we know about all three datasets for name in ('mother_äöü東', 'child_äöü東', 'grandchild_äöü東'): assert_true(sum([s.get('name', None) == assure_unicode(name) for s in meta])) #print(meta) assert_equal( # first implicit, then two natives, then aggregate meta[3]['dcterms:hasPart']['@id'], subds.id) success = False for m in meta: p = m.get('dcterms:hasPart', {}) if p.get('@id', None) == subsubds.id: assert_equal(opj('sub', 'subsub'), p.get('location', None)) success = True assert_true(success) # save the toplevel dataset only (see below) ds.save('with aggregated meta data', auto_add_changes=True) # now clone the beast to simulate a new user installing an empty dataset clone = install(opj(path, 'clone'), source=ds.path) # ID mechanism works assert_equal(ds.id, clone.id) # get fresh meta data, the implicit one for the top-most datasets should # differ, but the rest not clonemeta = get_metadata( clone, guess_type=False, ignore_subdatasets=False, ignore_cache=False) # make sure the implicit md for the topmost come first assert_equal(clonemeta[0]['@id'], clone.id) assert_equal(clonemeta[0]['@id'], ds.id) assert_equal(clone.repo.get_hexsha(), ds.repo.get_hexsha()) assert_equal(clonemeta[0]['version'], ds.repo.get_hexsha()) # all but the implicit is identical assert_equal(clonemeta[1:], meta[1:]) # the implicit md of the clone should list a dataset ID for its subds, # although it has not been obtained! assert_equal( clonemeta[3]['dcterms:hasPart']['@id'], subds.id) # now obtain a subdataset in the clone and the IDs should be updated clone.install('sub') partial = get_metadata(clone, guess_type=False, ignore_cache=True) # ids don't change assert_equal(partial[0]['@id'], clonemeta[0]['@id']) # datasets are properly connected assert_equal(partial[1]['dcterms:hasPart']['@id'], partial[2]['@id']) # query smoke test if os.environ.get('DATALAD_TESTS_NONETWORK'): raise SkipTest assert_equal(len(list(clone.search('mother'))), 1) assert_equal(len(list(clone.search('MoTHER'))), 1) # case insensitive child_res = list(clone.search('child')) assert_equal(len(child_res), 2) # little helper to match names def assert_names(res, names, path=clone.path): assert_equal(list(map(itemgetter(0), res)), [opj(path, n) for n in names]) # should yield (location, report) tuples assert_names(child_res, ['sub', 'sub/subsub']) # result should be identical to invoking search from api # and search_ should spit out locations out with swallow_outputs() as cmo: res = list(search_('child', dataset=clone)) assert_equal(res, child_res) assert_in(res[0][0], cmo.out) # and overarching search_ just for smoke testing of processing outputs # and not puking (e.g. under PY3) with swallow_outputs() as cmo: assert list(search_('.', regex=True, dataset=clone)) assert cmo.out # test searching among specified properties only assert_names(clone.search('i', search='name'), ['sub', 'sub/subsub']) assert_names(clone.search('i', search='keywords'), ['.']) # case shouldn't matter assert_names(clone.search('i', search='Keywords'), ['.']) assert_names(clone.search('i', search=['name', 'keywords']), ['.', 'sub', 'sub/subsub']) # without report_matched, we are getting none of the fields assert(all([not x for x in map(itemgetter(1), child_res)])) # but we would get all if asking for '*' assert(all([len(x) >= 9 for x in map(itemgetter(1), list(clone.search('child', report='*')))])) # but we would get only the matching name if we ask for report_matched assert_equal( set(map(lambda x: tuple(x[1].keys()), clone.search('child', report_matched=True))), set([('name',)]) ) # and the additional field we might have asked with report assert_equal( set(map(lambda x: tuple(sorted(x[1].keys())), clone.search('child', report_matched=True, report=['schema:type']))), set([('name', 'schema:type')]) ) # and if we ask report to be 'empty', we should get no fields child_res_empty = list(clone.search('child', report='')) assert_equal(len(child_res_empty), 2) assert_equal( set(map(lambda x: tuple(x[1].keys()), child_res_empty)), set([tuple()]) ) # more tests on returned paths: assert_names(clone.search('datalad'), ['.', 'sub', 'sub/subsub']) # if we clone subdataset and query for value present in it and its kid clone_sub = clone.install('sub') assert_names(clone_sub.search('datalad'), ['.', 'subsub'], clone_sub.path) # Test 'and' for multiple search entries assert_equal(len(list(clone.search(['child', 'bids']))), 2) assert_equal(len(list(clone.search(['child', 'subsub']))), 1) assert_equal(len(list(clone.search(['bids', 'sub']))), 2) res = list(clone.search('.*', regex=True)) # with regex assert_equal(len(res), 3) # one per dataset # we do search, not match assert_equal(len(list(clone.search('randchild', regex=True))), 1) assert_equal(len(list(clone.search(['gr.nd', 'ch.ld'], regex=True))), 1) assert_equal(len(list(clone.search('randchil.', regex=True))), 1) assert_equal(len(list(clone.search('^randchild.*', regex=True))), 0) assert_equal(len(list(clone.search('^grandchild.*', regex=True))), 1) assert_equal(len(list(clone.search('grandchild'))), 1) #TODO update the clone or reclone to check whether saved meta data comes down the pipe @skip_if_no_network @with_tree(tree=_dataset_hierarchy_template) def test_aggregate_with_missing_or_duplicate_id(path): # a hierarchy of three (super/sub)datasets, each with some native metadata ds = Dataset(opj(path, 'origin')).create(force=True) subds = ds.create('sub', force=True, if_dirty='ignore') subds.repo.remove(opj('.datalad', 'config')) subds.save() assert_false(exists(opj(subds.path, '.datalad', 'config'))) subsubds = subds.create('subsub', force=True, if_dirty='ignore') # aggregate from bottom to top, guess native data, no compacting of graph # should yield 6 meta data sets, one implicit, and one native per dataset # and a second native set for the topmost dataset aggregate_metadata(ds, guess_native_type=True, recursive=True) # no only ask the top superdataset, no recursion, just reading from the cache meta = get_metadata( ds, guess_type=False, ignore_subdatasets=False, ignore_cache=False) # and we know nothing subsub for name in ('grandchild_äöü東',): assert_true(sum([s.get('name', '') == assure_unicode(name) for s in meta])) # but search should not fail with swallow_outputs(): res1 = list(search_('.', regex=True, dataset=ds)) assert res1 # and let's see now if we wouldn't fail if dataset is duplicate if we # install the same dataset twice subds_clone = ds.install(source=subds.path, path="subds2") with swallow_outputs(): res2 = list(search_('.', regex=True, dataset=ds)) # TODO: bring back when meta data RF is complete with aggregate #assert_equal(len(res1) + 1, len(res2)) #assert_equal( # set(map(itemgetter(0), res1)).union({subds_clone.path}), # set(map(itemgetter(0), res2))) @with_tempfile(mkdir=True) def test_cached_load_document(tdir): target_schema = {'buga': 'duga'} cache_filename = opj(tdir, "crap") with open(cache_filename, 'wb') as f: f.write("CRAPNOTPICKLED".encode()) with patch('datalad.metadata._get_schema_url_cache_filename', return_value=cache_filename): with patch('pyld.jsonld.load_document', return_value=target_schema), \ swallow_logs(new_level=logging.WARNING) as cml: schema = _cached_load_document("http://schema.org/") assert_equal(schema, target_schema) cml.assert_logged("cannot load cache from", level="WARNING") # but now pickled one should have been saved assert_equal(pickle.load(open(cache_filename, 'rb')), target_schema) # and if we reload it -- it should be all fine without warnings # should come from cache so no need to overload load_document with swallow_logs(new_level=logging.WARNING) as cml: schema = _cached_load_document("http://schema.org/") assert_equal(schema, target_schema) assert_not_in("cannot load cache from", cml.out) @with_tempfile(mkdir=True) def test_ignore_nondatasets(path): # we want to ignore the version/commits for this test def _kill_time(meta): for m in meta: for k in ('version', 'dcterms:modified'): if k in m: del m[k] return meta ds = Dataset(path).create() meta = _kill_time(get_metadata(ds)) n_subm = 0 # placing another repo in the dataset has no effect on metadata for cls, subpath in ((GitRepo, 'subm'), (AnnexRepo, 'annex_subm')): subm_path = opj(ds.path, subpath) r = cls(subm_path, create=True) with open(opj(subm_path, 'test'), 'w') as f: f.write('test') r.add('test') r.commit('some') assert_true(Dataset(subm_path).is_installed()) assert_equal(meta, _kill_time(get_metadata(ds))) # making it a submodule has no effect either ds.save(auto_add_changes=True) assert_equal(len(ds.get_subdatasets()), n_subm + 1) assert_equal(meta, _kill_time(get_metadata(ds))) n_subm += 1 datalad-0.4.1/datalad/metadata/tests/test_search.py000066400000000000000000000136111301115270200222720ustar00rootroot00000000000000# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # -*- coding: utf-8 -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Some additional tests for search command (some are within test_base)""" from mock import patch from datalad import cfg from datalad.api import Dataset, install from nose.tools import assert_equal, assert_raises from datalad.utils import chpwd from datalad.tests.utils import assert_in from datalad.tests.utils import assert_is_generator from datalad.tests.utils import with_tempfile from datalad.tests.utils import with_testsui from datalad.tests.utils import SkipTest from datalad.support.exceptions import NoDatasetArgumentFound from datalad.api import search from datalad.metadata import search as search_mod from datalad.tests.utils import skip_if_no_network @with_testsui(interactive=False) @with_tempfile(mkdir=True) def test_search_outside1_noninteractive_ui(tdir): # we should raise an informative exception with chpwd(tdir): with assert_raises(NoDatasetArgumentFound) as cme: list(search("bu")) assert_in('run interactively', str(cme.exception)) @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_search_outside1(tdir, newhome): with chpwd(tdir): # should fail since directory exists, but not a dataset # should not even waste our response ;) always_render = cfg.obtain('datalad.api.alwaysrender') with patch.object(search_mod, 'LOCAL_CENTRAL_PATH', newhome): if always_render: # we do try to render results which actually causes exception # to come right away assert_raises(NoDatasetArgumentFound, search, "bu") else: gen = search("bu") assert_is_generator(gen) assert_raises(NoDatasetArgumentFound, next, gen) # and if we point to some non-existing dataset -- the same in both cases # but might come before even next if always_render with assert_raises(ValueError): next(search("bu", dataset=newhome)) @with_testsui(responses='yes') @with_tempfile(mkdir=True) @with_tempfile() def test_search_outside1_install_central_ds(tdir, central_dspath): with chpwd(tdir): # let's mock out even actual install/search calls with \ patch.object(search_mod, 'LOCAL_CENTRAL_PATH', central_dspath), \ patch('datalad.api.install', return_value=Dataset(central_dspath)) as mock_install, \ patch('datalad.distribution.dataset.Dataset.search', new_callable=_mock_search): _check_mocked_install(central_dspath, mock_install) # now on subsequent run, we want to mock as if dataset already exists # at central location and then do search again from datalad.ui import ui ui.add_responses('yes') mock_install.reset_mock() with patch( 'datalad.distribution.dataset.Dataset.is_installed', True): _check_mocked_install(central_dspath, mock_install) # and what if we say "no" to install? ui.add_responses('no') mock_install.reset_mock() with assert_raises(NoDatasetArgumentFound): list(search(".", regex=True)) # and if path exists and is a valid dataset and we say "no" Dataset(central_dspath).create() ui.add_responses('no') mock_install.reset_mock() with assert_raises(NoDatasetArgumentFound): list(search(".", regex=True)) _mocked_search_results = [ ('ds1', {'f': 'v'}), ('d2/ds2', {'f1': 'v1'}) ] class _mock_search(object): def __call__(*args, **kwargs): for loc, report in _mocked_search_results: yield loc, report def _check_mocked_install(central_dspath, mock_install): gen = search(".", regex=True) assert_is_generator(gen) # we no longer do any custom path tune up from the one returned by search # so should match what search returns assert_equal( list(gen), [(loc, report) for loc, report in _mocked_search_results]) mock_install.assert_called_once_with(central_dspath, source='///') @skip_if_no_network @with_tempfile def test_our_metadataset_search(tdir): # smoke test for basic search operations on our super-megadataset # expensive operation but ok ds = install(path=tdir, source="///") assert list(ds.search('.', report='*', regex=True)) assert list(ds.search('.', report='*')) assert list(ds.search('.', report_matched=True)) # and either we could provide output in different formats import simplejson from datalad.utils import swallow_outputs from datalad.api import search_ with swallow_outputs() as cmo: assert list(search_('.', report='*', regex=True, format='json', dataset=ds)) out = cmo.out # since this one is just absorbs all first, we can't go one by one assert simplejson.loads(out) try: import yaml except ImportError: raise SkipTest("no yaml module") with swallow_outputs() as cmo: assert list(search_('.', report='*', regex=True, format='yaml', dataset=ds)) out = cmo.out assert yaml.load(out) @with_tempfile def test_search_non_dataset(tdir): from datalad.support.gitrepo import GitRepo GitRepo(tdir, create=True) with assert_raises(NoDatasetArgumentFound) as cme: list(search('smth', dataset=tdir)) # Should instruct user how that repo could become a datalad dataset assert_in("datalad create --force", str(cme.exception)) datalad-0.4.1/datalad/resources/000077500000000000000000000000001301115270200165025ustar00rootroot00000000000000datalad-0.4.1/datalad/resources/website/000077500000000000000000000000001301115270200201445ustar00rootroot00000000000000datalad-0.4.1/datalad/resources/website/assets/000077500000000000000000000000001301115270200214465ustar00rootroot00000000000000datalad-0.4.1/datalad/resources/website/assets/css/000077500000000000000000000000001301115270200222365ustar00rootroot00000000000000datalad-0.4.1/datalad/resources/website/assets/css/jquery.dataTables-1.10.12.css000066400000000000000000000360771301115270200271450ustar00rootroot00000000000000/* * Table styles */ table.dataTable { width: 100%; margin: 0 auto; clear: both; border-collapse: separate; border-spacing: 0; /* * Header and footer styles */ /* * Body styles */ } table.dataTable thead th, table.dataTable tfoot th { font-weight: bold; } table.dataTable thead th, table.dataTable thead td { padding: 10px 18px; border-bottom: 1px solid #111; } table.dataTable thead th:active, table.dataTable thead td:active { outline: none; } table.dataTable tfoot th, table.dataTable tfoot td { padding: 10px 18px 6px 18px; border-top: 1px solid #111; } table.dataTable thead .sorting, table.dataTable thead .sorting_asc, table.dataTable thead .sorting_desc { cursor: pointer; *cursor: hand; } table.dataTable thead .sorting, table.dataTable thead .sorting_asc, table.dataTable thead .sorting_desc, table.dataTable thead .sorting_asc_disabled, table.dataTable thead .sorting_desc_disabled { background-repeat: no-repeat; background-position: center right; } table.dataTable thead .sorting { background-image: url("../images/sort_both.png"); } table.dataTable thead .sorting_asc { background-image: url("../images/sort_asc.png"); } table.dataTable thead .sorting_desc { background-image: url("../images/sort_desc.png"); } table.dataTable thead .sorting_asc_disabled { background-image: url("../images/sort_asc_disabled.png"); } table.dataTable thead .sorting_desc_disabled { background-image: url("../images/sort_desc_disabled.png"); } table.dataTable tbody tr { background-color: #ffffff; } table.dataTable tbody tr.selected { background-color: #B0BED9; } table.dataTable tbody th, table.dataTable tbody td { padding: 8px 10px; } table.dataTable.row-border tbody th, table.dataTable.row-border tbody td, table.dataTable.display tbody th, table.dataTable.display tbody td { border-top: 1px solid #ddd; } table.dataTable.row-border tbody tr:first-child th, table.dataTable.row-border tbody tr:first-child td, table.dataTable.display tbody tr:first-child th, table.dataTable.display tbody tr:first-child td { border-top: none; } table.dataTable.cell-border tbody th, table.dataTable.cell-border tbody td { border-top: 1px solid #ddd; border-right: 1px solid #ddd; } table.dataTable.cell-border tbody tr th:first-child, table.dataTable.cell-border tbody tr td:first-child { border-left: 1px solid #ddd; } table.dataTable.cell-border tbody tr:first-child th, table.dataTable.cell-border tbody tr:first-child td { border-top: none; } table.dataTable.stripe tbody tr.odd, table.dataTable.display tbody tr.odd { background-color: #f9f9f9; } table.dataTable.stripe tbody tr.odd.selected, table.dataTable.display tbody tr.odd.selected { background-color: #acbad4; } table.dataTable.hover tbody tr:hover, table.dataTable.display tbody tr:hover { background-color: #f6f6f6; } table.dataTable.hover tbody tr:hover.selected, table.dataTable.display tbody tr:hover.selected { background-color: #aab7d1; } table.dataTable.order-column tbody tr > .sorting_1, table.dataTable.order-column tbody tr > .sorting_2, table.dataTable.order-column tbody tr > .sorting_3, table.dataTable.display tbody tr > .sorting_1, table.dataTable.display tbody tr > .sorting_2, table.dataTable.display tbody tr > .sorting_3 { background-color: #fafafa; } table.dataTable.order-column tbody tr.selected > .sorting_1, table.dataTable.order-column tbody tr.selected > .sorting_2, table.dataTable.order-column tbody tr.selected > .sorting_3, table.dataTable.display tbody tr.selected > .sorting_1, table.dataTable.display tbody tr.selected > .sorting_2, table.dataTable.display tbody tr.selected > .sorting_3 { background-color: #acbad5; } table.dataTable.display tbody tr.odd > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd > .sorting_1 { background-color: #f1f1f1; } table.dataTable.display tbody tr.odd > .sorting_2, table.dataTable.order-column.stripe tbody tr.odd > .sorting_2 { background-color: #f3f3f3; } table.dataTable.display tbody tr.odd > .sorting_3, table.dataTable.order-column.stripe tbody tr.odd > .sorting_3 { background-color: whitesmoke; } table.dataTable.display tbody tr.odd.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_1 { background-color: #a6b4cd; } table.dataTable.display tbody tr.odd.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_2 { background-color: #a8b5cf; } table.dataTable.display tbody tr.odd.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_3 { background-color: #a9b7d1; } table.dataTable.display tbody tr.even > .sorting_1, table.dataTable.order-column.stripe tbody tr.even > .sorting_1 { background-color: #fafafa; } table.dataTable.display tbody tr.even > .sorting_2, table.dataTable.order-column.stripe tbody tr.even > .sorting_2 { background-color: #fcfcfc; } table.dataTable.display tbody tr.even > .sorting_3, table.dataTable.order-column.stripe tbody tr.even > .sorting_3 { background-color: #fefefe; } table.dataTable.display tbody tr.even.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_1 { background-color: #acbad5; } table.dataTable.display tbody tr.even.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_2 { background-color: #aebcd6; } table.dataTable.display tbody tr.even.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_3 { background-color: #afbdd8; } table.dataTable.display tbody tr:hover > .sorting_1, table.dataTable.order-column.hover tbody tr:hover > .sorting_1 { background-color: #eaeaea; } table.dataTable.display tbody tr:hover > .sorting_2, table.dataTable.order-column.hover tbody tr:hover > .sorting_2 { background-color: #ececec; } table.dataTable.display tbody tr:hover > .sorting_3, table.dataTable.order-column.hover tbody tr:hover > .sorting_3 { background-color: #efefef; } table.dataTable.display tbody tr:hover.selected > .sorting_1, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_1 { background-color: #a2aec7; } table.dataTable.display tbody tr:hover.selected > .sorting_2, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_2 { background-color: #a3b0c9; } table.dataTable.display tbody tr:hover.selected > .sorting_3, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_3 { background-color: #a5b2cb; } table.dataTable.no-footer { border-bottom: 1px solid #111; } table.dataTable.nowrap th, table.dataTable.nowrap td { white-space: nowrap; } table.dataTable.compact thead th, table.dataTable.compact thead td { padding: 4px 17px 4px 4px; } table.dataTable.compact tfoot th, table.dataTable.compact tfoot td { padding: 4px; } table.dataTable.compact tbody th, table.dataTable.compact tbody td { padding: 4px; } table.dataTable th.dt-left, table.dataTable td.dt-left { text-align: left; } table.dataTable th.dt-center, table.dataTable td.dt-center, table.dataTable td.dataTables_empty { text-align: center; } table.dataTable th.dt-right, table.dataTable td.dt-right { text-align: right; } table.dataTable th.dt-justify, table.dataTable td.dt-justify { text-align: justify; } table.dataTable th.dt-nowrap, table.dataTable td.dt-nowrap { white-space: nowrap; } table.dataTable thead th.dt-head-left, table.dataTable thead td.dt-head-left, table.dataTable tfoot th.dt-head-left, table.dataTable tfoot td.dt-head-left { text-align: left; } table.dataTable thead th.dt-head-center, table.dataTable thead td.dt-head-center, table.dataTable tfoot th.dt-head-center, table.dataTable tfoot td.dt-head-center { text-align: center; } table.dataTable thead th.dt-head-right, table.dataTable thead td.dt-head-right, table.dataTable tfoot th.dt-head-right, table.dataTable tfoot td.dt-head-right { text-align: right; } table.dataTable thead th.dt-head-justify, table.dataTable thead td.dt-head-justify, table.dataTable tfoot th.dt-head-justify, table.dataTable tfoot td.dt-head-justify { text-align: justify; } table.dataTable thead th.dt-head-nowrap, table.dataTable thead td.dt-head-nowrap, table.dataTable tfoot th.dt-head-nowrap, table.dataTable tfoot td.dt-head-nowrap { white-space: nowrap; } table.dataTable tbody th.dt-body-left, table.dataTable tbody td.dt-body-left { text-align: left; } table.dataTable tbody th.dt-body-center, table.dataTable tbody td.dt-body-center { text-align: center; } table.dataTable tbody th.dt-body-right, table.dataTable tbody td.dt-body-right { text-align: right; } table.dataTable tbody th.dt-body-justify, table.dataTable tbody td.dt-body-justify { text-align: justify; } table.dataTable tbody th.dt-body-nowrap, table.dataTable tbody td.dt-body-nowrap { white-space: nowrap; } table.dataTable, table.dataTable th, table.dataTable td { -webkit-box-sizing: content-box; box-sizing: content-box; } /* * Control feature layout */ .dataTables_wrapper { position: relative; clear: both; *zoom: 1; zoom: 1; } .dataTables_wrapper .dataTables_length { float: left; } .dataTables_wrapper .dataTables_filter { float: right; text-align: right; } .dataTables_wrapper .dataTables_filter input { margin-left: 0.5em; } .dataTables_wrapper .dataTables_info { clear: both; float: left; padding-top: 0.755em; } .dataTables_wrapper .dataTables_paginate { float: right; text-align: right; padding-top: 0.25em; } .dataTables_wrapper .dataTables_paginate .paginate_button { box-sizing: border-box; display: inline-block; min-width: 1.5em; padding: 0.5em 1em; margin-left: 2px; text-align: center; text-decoration: none !important; cursor: pointer; *cursor: hand; color: #333 !important; border: 1px solid transparent; border-radius: 2px; } .dataTables_wrapper .dataTables_paginate .paginate_button.current, .dataTables_wrapper .dataTables_paginate .paginate_button.current:hover { color: #333 !important; border: 1px solid #979797; background-color: white; background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, white), color-stop(100%, #dcdcdc)); /* Chrome,Safari4+ */ background: -webkit-linear-gradient(top, white 0%, #dcdcdc 100%); /* Chrome10+,Safari5.1+ */ background: -moz-linear-gradient(top, white 0%, #dcdcdc 100%); /* FF3.6+ */ background: -ms-linear-gradient(top, white 0%, #dcdcdc 100%); /* IE10+ */ background: -o-linear-gradient(top, white 0%, #dcdcdc 100%); /* Opera 11.10+ */ background: linear-gradient(to bottom, white 0%, #dcdcdc 100%); /* W3C */ } .dataTables_wrapper .dataTables_paginate .paginate_button.disabled, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active { cursor: default; color: #666 !important; border: 1px solid transparent; background: transparent; box-shadow: none; } .dataTables_wrapper .dataTables_paginate .paginate_button:hover { color: white !important; border: 1px solid #111; background-color: #585858; background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111)); /* Chrome,Safari4+ */ background: -webkit-linear-gradient(top, #585858 0%, #111 100%); /* Chrome10+,Safari5.1+ */ background: -moz-linear-gradient(top, #585858 0%, #111 100%); /* FF3.6+ */ background: -ms-linear-gradient(top, #585858 0%, #111 100%); /* IE10+ */ background: -o-linear-gradient(top, #585858 0%, #111 100%); /* Opera 11.10+ */ background: linear-gradient(to bottom, #585858 0%, #111 100%); /* W3C */ } .dataTables_wrapper .dataTables_paginate .paginate_button:active { outline: none; background-color: #2b2b2b; background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c)); /* Chrome,Safari4+ */ background: -webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); /* Chrome10+,Safari5.1+ */ background: -moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); /* FF3.6+ */ background: -ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); /* IE10+ */ background: -o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); /* Opera 11.10+ */ background: linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%); /* W3C */ box-shadow: inset 0 0 3px #111; } .dataTables_wrapper .dataTables_paginate .ellipsis { padding: 0 1em; } .dataTables_wrapper .dataTables_processing { position: absolute; top: 50%; left: 50%; width: 100%; height: 40px; margin-left: -50%; margin-top: -25px; padding-top: 20px; text-align: center; font-size: 1.2em; background-color: white; background: -webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255, 255, 255, 0)), color-stop(25%, rgba(255, 255, 255, 0.9)), color-stop(75%, rgba(255, 255, 255, 0.9)), color-stop(100%, rgba(255, 255, 255, 0))); background: -webkit-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%); background: -moz-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%); background: -ms-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%); background: -o-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%); background: linear-gradient(to right, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%); } .dataTables_wrapper .dataTables_length, .dataTables_wrapper .dataTables_filter, .dataTables_wrapper .dataTables_info, .dataTables_wrapper .dataTables_processing, .dataTables_wrapper .dataTables_paginate { color: #333; } .dataTables_wrapper .dataTables_scroll { clear: both; } .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody { *margin-top: -1px; -webkit-overflow-scrolling: touch; } .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td { vertical-align: middle; } .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th > div.dataTables_sizing, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td > div.dataTables_sizing { height: 0; overflow: hidden; margin: 0 !important; padding: 0 !important; } .dataTables_wrapper.no-footer .dataTables_scrollBody { border-bottom: 1px solid #111; } .dataTables_wrapper.no-footer div.dataTables_scrollHead table, .dataTables_wrapper.no-footer div.dataTables_scrollBody table { border-bottom: none; } .dataTables_wrapper:after { visibility: hidden; display: block; content: ""; clear: both; height: 0; } @media screen and (max-width: 767px) { .dataTables_wrapper .dataTables_info, .dataTables_wrapper .dataTables_paginate { float: none; text-align: center; } .dataTables_wrapper .dataTables_paginate { margin-top: 0.5em; } } @media screen and (max-width: 640px) { .dataTables_wrapper .dataTables_length, .dataTables_wrapper .dataTables_filter { float: none; text-align: center; } .dataTables_wrapper .dataTables_filter { margin-top: 0.5em; } } datalad-0.4.1/datalad/resources/website/assets/images/000077500000000000000000000000001301115270200227135ustar00rootroot00000000000000datalad-0.4.1/datalad/resources/website/assets/images/datalad_banner.svg000066400000000000000000000217551301115270200263650ustar00rootroot00000000000000 image/svg+xml atala datalad-0.4.1/datalad/resources/website/assets/images/favicon-16x16.png000066400000000000000000000010101301115270200256210ustar00rootroot00000000000000PNG  IHDRasBIT|d pHYsu85tEXtSoftwarewww.inkscape.org<IDAT8ՐKQ{ÉM0E)їw.hѮEۨuQ DU BaJhNEz9 tȀ sH w_;;kyKZG{cn?}0>H%gb.?:!d"PJ`NsfVfV}m1UHo-_t'߂\4Eۙf!>j~.5[+_ C l2TxYT*Oe_{ΫSJXw˾>) } ԫh m5B ؛8|Q2A N{$IENDB`datalad-0.4.1/datalad/resources/website/assets/images/favicon.svg000066400000000000000000000407731301115270200250740ustar00rootroot00000000000000 image/svg+xml datalad-0.4.1/datalad/resources/website/assets/images/sort_asc.png000066400000000000000000000002361301115270200252370ustar00rootroot00000000000000PNG  IHDRrP6eIDATxc.Ob6jV Ŀ8R1@LA@<lOc#C3ېj/w1;)%O zF(zl-mґIENDB`datalad-0.4.1/datalad/resources/website/assets/images/sort_both.png000066400000000000000000000003071301115270200254240ustar00rootroot00000000000000PNG  IHDRYGIDATx@)X!>%55{Au]wwwPGkHGD=Xn%Vu(>,33<Ō+5>,MW5== 0 && j < len ? [ this[ j ] ] : [] ); }, end: function() { return this.prevObject || this.constructor(); }, // For internal use only. // Behaves like an Array's method, not like a jQuery method. push: push, sort: arr.sort, splice: arr.splice }; jQuery.extend = jQuery.fn.extend = function() { var options, name, src, copy, copyIsArray, clone, target = arguments[ 0 ] || {}, i = 1, length = arguments.length, deep = false; // Handle a deep copy situation if ( typeof target === "boolean" ) { deep = target; // Skip the boolean and the target target = arguments[ i ] || {}; i++; } // Handle case when target is a string or something (possible in deep copy) if ( typeof target !== "object" && !jQuery.isFunction( target ) ) { target = {}; } // Extend jQuery itself if only one argument is passed if ( i === length ) { target = this; i--; } for ( ; i < length; i++ ) { // Only deal with non-null/undefined values if ( ( options = arguments[ i ] ) != null ) { // Extend the base object for ( name in options ) { src = target[ name ]; copy = options[ name ]; // Prevent never-ending loop if ( target === copy ) { continue; } // Recurse if we're merging plain objects or arrays if ( deep && copy && ( jQuery.isPlainObject( copy ) || ( copyIsArray = jQuery.isArray( copy ) ) ) ) { if ( copyIsArray ) { copyIsArray = false; clone = src && jQuery.isArray( src ) ? src : []; } else { clone = src && jQuery.isPlainObject( src ) ? src : {}; } // Never move original objects, clone them target[ name ] = jQuery.extend( deep, clone, copy ); // Don't bring in undefined values } else if ( copy !== undefined ) { target[ name ] = copy; } } } } // Return the modified object return target; }; jQuery.extend( { // Unique for each copy of jQuery on the page expando: "jQuery" + ( version + Math.random() ).replace( /\D/g, "" ), // Assume jQuery is ready without the ready module isReady: true, error: function( msg ) { throw new Error( msg ); }, noop: function() {}, isFunction: function( obj ) { return jQuery.type( obj ) === "function"; }, isArray: Array.isArray, isWindow: function( obj ) { return obj != null && obj === obj.window; }, isNumeric: function( obj ) { // parseFloat NaNs numeric-cast false positives (null|true|false|"") // ...but misinterprets leading-number strings, particularly hex literals ("0x...") // subtraction forces infinities to NaN // adding 1 corrects loss of precision from parseFloat (#15100) var realStringObj = obj && obj.toString(); return !jQuery.isArray( obj ) && ( realStringObj - parseFloat( realStringObj ) + 1 ) >= 0; }, isPlainObject: function( obj ) { var key; // Not plain objects: // - Any object or value whose internal [[Class]] property is not "[object Object]" // - DOM nodes // - window if ( jQuery.type( obj ) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) { return false; } // Not own constructor property must be Object if ( obj.constructor && !hasOwn.call( obj, "constructor" ) && !hasOwn.call( obj.constructor.prototype || {}, "isPrototypeOf" ) ) { return false; } // Own properties are enumerated firstly, so to speed up, // if last one is own, then all properties are own for ( key in obj ) {} return key === undefined || hasOwn.call( obj, key ); }, isEmptyObject: function( obj ) { var name; for ( name in obj ) { return false; } return true; }, type: function( obj ) { if ( obj == null ) { return obj + ""; } // Support: Android<4.0, iOS<6 (functionish RegExp) return typeof obj === "object" || typeof obj === "function" ? class2type[ toString.call( obj ) ] || "object" : typeof obj; }, // Evaluates a script in a global context globalEval: function( code ) { var script, indirect = eval; code = jQuery.trim( code ); if ( code ) { // If the code includes a valid, prologue position // strict mode pragma, execute code by injecting a // script tag into the document. if ( code.indexOf( "use strict" ) === 1 ) { script = document.createElement( "script" ); script.text = code; document.head.appendChild( script ).parentNode.removeChild( script ); } else { // Otherwise, avoid the DOM node creation, insertion // and removal by using an indirect global eval indirect( code ); } } }, // Convert dashed to camelCase; used by the css and data modules // Support: IE9-11+ // Microsoft forgot to hump their vendor prefix (#9572) camelCase: function( string ) { return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); }, nodeName: function( elem, name ) { return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); }, each: function( obj, callback ) { var length, i = 0; if ( isArrayLike( obj ) ) { length = obj.length; for ( ; i < length; i++ ) { if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { break; } } } else { for ( i in obj ) { if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { break; } } } return obj; }, // Support: Android<4.1 trim: function( text ) { return text == null ? "" : ( text + "" ).replace( rtrim, "" ); }, // results is for internal usage only makeArray: function( arr, results ) { var ret = results || []; if ( arr != null ) { if ( isArrayLike( Object( arr ) ) ) { jQuery.merge( ret, typeof arr === "string" ? [ arr ] : arr ); } else { push.call( ret, arr ); } } return ret; }, inArray: function( elem, arr, i ) { return arr == null ? -1 : indexOf.call( arr, elem, i ); }, merge: function( first, second ) { var len = +second.length, j = 0, i = first.length; for ( ; j < len; j++ ) { first[ i++ ] = second[ j ]; } first.length = i; return first; }, grep: function( elems, callback, invert ) { var callbackInverse, matches = [], i = 0, length = elems.length, callbackExpect = !invert; // Go through the array, only saving the items // that pass the validator function for ( ; i < length; i++ ) { callbackInverse = !callback( elems[ i ], i ); if ( callbackInverse !== callbackExpect ) { matches.push( elems[ i ] ); } } return matches; }, // arg is for internal usage only map: function( elems, callback, arg ) { var length, value, i = 0, ret = []; // Go through the array, translating each of the items to their new values if ( isArrayLike( elems ) ) { length = elems.length; for ( ; i < length; i++ ) { value = callback( elems[ i ], i, arg ); if ( value != null ) { ret.push( value ); } } // Go through every key on the object, } else { for ( i in elems ) { value = callback( elems[ i ], i, arg ); if ( value != null ) { ret.push( value ); } } } // Flatten any nested arrays return concat.apply( [], ret ); }, // A global GUID counter for objects guid: 1, // Bind a function to a context, optionally partially applying any // arguments. proxy: function( fn, context ) { var tmp, args, proxy; if ( typeof context === "string" ) { tmp = fn[ context ]; context = fn; fn = tmp; } // Quick check to determine if target is callable, in the spec // this throws a TypeError, but we will just return undefined. if ( !jQuery.isFunction( fn ) ) { return undefined; } // Simulated bind args = slice.call( arguments, 2 ); proxy = function() { return fn.apply( context || this, args.concat( slice.call( arguments ) ) ); }; // Set the guid of unique handler to the same of original handler, so it can be removed proxy.guid = fn.guid = fn.guid || jQuery.guid++; return proxy; }, now: Date.now, // jQuery.support is not used in Core but other projects attach their // properties to it so it needs to exist. support: support } ); // JSHint would error on this code due to the Symbol not being defined in ES5. // Defining this global in .jshintrc would create a danger of using the global // unguarded in another place, it seems safer to just disable JSHint for these // three lines. /* jshint ignore: start */ if ( typeof Symbol === "function" ) { jQuery.fn[ Symbol.iterator ] = arr[ Symbol.iterator ]; } /* jshint ignore: end */ // Populate the class2type map jQuery.each( "Boolean Number String Function Array Date RegExp Object Error Symbol".split( " " ), function( i, name ) { class2type[ "[object " + name + "]" ] = name.toLowerCase(); } ); function isArrayLike( obj ) { // Support: iOS 8.2 (not reproducible in simulator) // `in` check used to prevent JIT error (gh-2145) // hasOwn isn't used here due to false negatives // regarding Nodelist length in IE var length = !!obj && "length" in obj && obj.length, type = jQuery.type( obj ); if ( type === "function" || jQuery.isWindow( obj ) ) { return false; } return type === "array" || length === 0 || typeof length === "number" && length > 0 && ( length - 1 ) in obj; } var Sizzle = /*! * Sizzle CSS Selector Engine v2.2.1 * http://sizzlejs.com/ * * Copyright jQuery Foundation and other contributors * Released under the MIT license * http://jquery.org/license * * Date: 2015-10-17 */ (function( window ) { var i, support, Expr, getText, isXML, tokenize, compile, select, outermostContext, sortInput, hasDuplicate, // Local document vars setDocument, document, docElem, documentIsHTML, rbuggyQSA, rbuggyMatches, matches, contains, // Instance-specific data expando = "sizzle" + 1 * new Date(), preferredDoc = window.document, dirruns = 0, done = 0, classCache = createCache(), tokenCache = createCache(), compilerCache = createCache(), sortOrder = function( a, b ) { if ( a === b ) { hasDuplicate = true; } return 0; }, // General-purpose constants MAX_NEGATIVE = 1 << 31, // Instance methods hasOwn = ({}).hasOwnProperty, arr = [], pop = arr.pop, push_native = arr.push, push = arr.push, slice = arr.slice, // Use a stripped-down indexOf as it's faster than native // http://jsperf.com/thor-indexof-vs-for/5 indexOf = function( list, elem ) { var i = 0, len = list.length; for ( ; i < len; i++ ) { if ( list[i] === elem ) { return i; } } return -1; }, booleans = "checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped", // Regular expressions // http://www.w3.org/TR/css3-selectors/#whitespace whitespace = "[\\x20\\t\\r\\n\\f]", // http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier identifier = "(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+", // Attribute selectors: http://www.w3.org/TR/selectors/#attribute-selectors attributes = "\\[" + whitespace + "*(" + identifier + ")(?:" + whitespace + // Operator (capture 2) "*([*^$|!~]?=)" + whitespace + // "Attribute values must be CSS identifiers [capture 5] or strings [capture 3 or capture 4]" "*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|(" + identifier + "))|)" + whitespace + "*\\]", pseudos = ":(" + identifier + ")(?:\\((" + // To reduce the number of selectors needing tokenize in the preFilter, prefer arguments: // 1. quoted (capture 3; capture 4 or capture 5) "('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|" + // 2. simple (capture 6) "((?:\\\\.|[^\\\\()[\\]]|" + attributes + ")*)|" + // 3. anything else (capture 2) ".*" + ")\\)|)", // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter rwhitespace = new RegExp( whitespace + "+", "g" ), rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), rcombinators = new RegExp( "^" + whitespace + "*([>+~]|" + whitespace + ")" + whitespace + "*" ), rattributeQuotes = new RegExp( "=" + whitespace + "*([^\\]'\"]*?)" + whitespace + "*\\]", "g" ), rpseudo = new RegExp( pseudos ), ridentifier = new RegExp( "^" + identifier + "$" ), matchExpr = { "ID": new RegExp( "^#(" + identifier + ")" ), "CLASS": new RegExp( "^\\.(" + identifier + ")" ), "TAG": new RegExp( "^(" + identifier + "|[*])" ), "ATTR": new RegExp( "^" + attributes ), "PSEUDO": new RegExp( "^" + pseudos ), "CHILD": new RegExp( "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + whitespace + "*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace + "*(\\d+)|))" + whitespace + "*\\)|)", "i" ), "bool": new RegExp( "^(?:" + booleans + ")$", "i" ), // For use in libraries implementing .is() // We use this for POS matching in `select` "needsContext": new RegExp( "^" + whitespace + "*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + whitespace + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", "i" ) }, rinputs = /^(?:input|select|textarea|button)$/i, rheader = /^h\d$/i, rnative = /^[^{]+\{\s*\[native \w/, // Easily-parseable/retrievable ID or TAG or CLASS selectors rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/, rsibling = /[+~]/, rescape = /'|\\/g, // CSS escapes http://www.w3.org/TR/CSS21/syndata.html#escaped-characters runescape = new RegExp( "\\\\([\\da-f]{1,6}" + whitespace + "?|(" + whitespace + ")|.)", "ig" ), funescape = function( _, escaped, escapedWhitespace ) { var high = "0x" + escaped - 0x10000; // NaN means non-codepoint // Support: Firefox<24 // Workaround erroneous numeric interpretation of +"0x" return high !== high || escapedWhitespace ? escaped : high < 0 ? // BMP codepoint String.fromCharCode( high + 0x10000 ) : // Supplemental Plane codepoint (surrogate pair) String.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 ); }, // Used for iframes // See setDocument() // Removing the function wrapper causes a "Permission Denied" // error in IE unloadHandler = function() { setDocument(); }; // Optimize for push.apply( _, NodeList ) try { push.apply( (arr = slice.call( preferredDoc.childNodes )), preferredDoc.childNodes ); // Support: Android<4.0 // Detect silently failing push.apply arr[ preferredDoc.childNodes.length ].nodeType; } catch ( e ) { push = { apply: arr.length ? // Leverage slice if possible function( target, els ) { push_native.apply( target, slice.call(els) ); } : // Support: IE<9 // Otherwise append directly function( target, els ) { var j = target.length, i = 0; // Can't trust NodeList.length while ( (target[j++] = els[i++]) ) {} target.length = j - 1; } }; } function Sizzle( selector, context, results, seed ) { var m, i, elem, nid, nidselect, match, groups, newSelector, newContext = context && context.ownerDocument, // nodeType defaults to 9, since context defaults to document nodeType = context ? context.nodeType : 9; results = results || []; // Return early from calls with invalid selector or context if ( typeof selector !== "string" || !selector || nodeType !== 1 && nodeType !== 9 && nodeType !== 11 ) { return results; } // Try to shortcut find operations (as opposed to filters) in HTML documents if ( !seed ) { if ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) { setDocument( context ); } context = context || document; if ( documentIsHTML ) { // If the selector is sufficiently simple, try using a "get*By*" DOM method // (excepting DocumentFragment context, where the methods don't exist) if ( nodeType !== 11 && (match = rquickExpr.exec( selector )) ) { // ID selector if ( (m = match[1]) ) { // Document context if ( nodeType === 9 ) { if ( (elem = context.getElementById( m )) ) { // Support: IE, Opera, Webkit // TODO: identify versions // getElementById can match elements by name instead of ID if ( elem.id === m ) { results.push( elem ); return results; } } else { return results; } // Element context } else { // Support: IE, Opera, Webkit // TODO: identify versions // getElementById can match elements by name instead of ID if ( newContext && (elem = newContext.getElementById( m )) && contains( context, elem ) && elem.id === m ) { results.push( elem ); return results; } } // Type selector } else if ( match[2] ) { push.apply( results, context.getElementsByTagName( selector ) ); return results; // Class selector } else if ( (m = match[3]) && support.getElementsByClassName && context.getElementsByClassName ) { push.apply( results, context.getElementsByClassName( m ) ); return results; } } // Take advantage of querySelectorAll if ( support.qsa && !compilerCache[ selector + " " ] && (!rbuggyQSA || !rbuggyQSA.test( selector )) ) { if ( nodeType !== 1 ) { newContext = context; newSelector = selector; // qSA looks outside Element context, which is not what we want // Thanks to Andrew Dupont for this workaround technique // Support: IE <=8 // Exclude object elements } else if ( context.nodeName.toLowerCase() !== "object" ) { // Capture the context ID, setting it first if necessary if ( (nid = context.getAttribute( "id" )) ) { nid = nid.replace( rescape, "\\$&" ); } else { context.setAttribute( "id", (nid = expando) ); } // Prefix every selector in the list groups = tokenize( selector ); i = groups.length; nidselect = ridentifier.test( nid ) ? "#" + nid : "[id='" + nid + "']"; while ( i-- ) { groups[i] = nidselect + " " + toSelector( groups[i] ); } newSelector = groups.join( "," ); // Expand context for sibling selectors newContext = rsibling.test( selector ) && testContext( context.parentNode ) || context; } if ( newSelector ) { try { push.apply( results, newContext.querySelectorAll( newSelector ) ); return results; } catch ( qsaError ) { } finally { if ( nid === expando ) { context.removeAttribute( "id" ); } } } } } } // All others return select( selector.replace( rtrim, "$1" ), context, results, seed ); } /** * Create key-value caches of limited size * @returns {function(string, object)} Returns the Object data after storing it on itself with * property name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength) * deleting the oldest entry */ function createCache() { var keys = []; function cache( key, value ) { // Use (key + " ") to avoid collision with native prototype properties (see Issue #157) if ( keys.push( key + " " ) > Expr.cacheLength ) { // Only keep the most recent entries delete cache[ keys.shift() ]; } return (cache[ key + " " ] = value); } return cache; } /** * Mark a function for special use by Sizzle * @param {Function} fn The function to mark */ function markFunction( fn ) { fn[ expando ] = true; return fn; } /** * Support testing using an element * @param {Function} fn Passed the created div and expects a boolean result */ function assert( fn ) { var div = document.createElement("div"); try { return !!fn( div ); } catch (e) { return false; } finally { // Remove from its parent by default if ( div.parentNode ) { div.parentNode.removeChild( div ); } // release memory in IE div = null; } } /** * Adds the same handler for all of the specified attrs * @param {String} attrs Pipe-separated list of attributes * @param {Function} handler The method that will be applied */ function addHandle( attrs, handler ) { var arr = attrs.split("|"), i = arr.length; while ( i-- ) { Expr.attrHandle[ arr[i] ] = handler; } } /** * Checks document order of two siblings * @param {Element} a * @param {Element} b * @returns {Number} Returns less than 0 if a precedes b, greater than 0 if a follows b */ function siblingCheck( a, b ) { var cur = b && a, diff = cur && a.nodeType === 1 && b.nodeType === 1 && ( ~b.sourceIndex || MAX_NEGATIVE ) - ( ~a.sourceIndex || MAX_NEGATIVE ); // Use IE sourceIndex if available on both nodes if ( diff ) { return diff; } // Check if b follows a if ( cur ) { while ( (cur = cur.nextSibling) ) { if ( cur === b ) { return -1; } } } return a ? 1 : -1; } /** * Returns a function to use in pseudos for input types * @param {String} type */ function createInputPseudo( type ) { return function( elem ) { var name = elem.nodeName.toLowerCase(); return name === "input" && elem.type === type; }; } /** * Returns a function to use in pseudos for buttons * @param {String} type */ function createButtonPseudo( type ) { return function( elem ) { var name = elem.nodeName.toLowerCase(); return (name === "input" || name === "button") && elem.type === type; }; } /** * Returns a function to use in pseudos for positionals * @param {Function} fn */ function createPositionalPseudo( fn ) { return markFunction(function( argument ) { argument = +argument; return markFunction(function( seed, matches ) { var j, matchIndexes = fn( [], seed.length, argument ), i = matchIndexes.length; // Match elements found at the specified indexes while ( i-- ) { if ( seed[ (j = matchIndexes[i]) ] ) { seed[j] = !(matches[j] = seed[j]); } } }); }); } /** * Checks a node for validity as a Sizzle context * @param {Element|Object=} context * @returns {Element|Object|Boolean} The input node if acceptable, otherwise a falsy value */ function testContext( context ) { return context && typeof context.getElementsByTagName !== "undefined" && context; } // Expose support vars for convenience support = Sizzle.support = {}; /** * Detects XML nodes * @param {Element|Object} elem An element or a document * @returns {Boolean} True iff elem is a non-HTML XML node */ isXML = Sizzle.isXML = function( elem ) { // documentElement is verified for cases where it doesn't yet exist // (such as loading iframes in IE - #4833) var documentElement = elem && (elem.ownerDocument || elem).documentElement; return documentElement ? documentElement.nodeName !== "HTML" : false; }; /** * Sets document-related variables once based on the current document * @param {Element|Object} [doc] An element or document object to use to set the document * @returns {Object} Returns the current document */ setDocument = Sizzle.setDocument = function( node ) { var hasCompare, parent, doc = node ? node.ownerDocument || node : preferredDoc; // Return early if doc is invalid or already selected if ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) { return document; } // Update global variables document = doc; docElem = document.documentElement; documentIsHTML = !isXML( document ); // Support: IE 9-11, Edge // Accessing iframe documents after unload throws "permission denied" errors (jQuery #13936) if ( (parent = document.defaultView) && parent.top !== parent ) { // Support: IE 11 if ( parent.addEventListener ) { parent.addEventListener( "unload", unloadHandler, false ); // Support: IE 9 - 10 only } else if ( parent.attachEvent ) { parent.attachEvent( "onunload", unloadHandler ); } } /* Attributes ---------------------------------------------------------------------- */ // Support: IE<8 // Verify that getAttribute really returns attributes and not properties // (excepting IE8 booleans) support.attributes = assert(function( div ) { div.className = "i"; return !div.getAttribute("className"); }); /* getElement(s)By* ---------------------------------------------------------------------- */ // Check if getElementsByTagName("*") returns only elements support.getElementsByTagName = assert(function( div ) { div.appendChild( document.createComment("") ); return !div.getElementsByTagName("*").length; }); // Support: IE<9 support.getElementsByClassName = rnative.test( document.getElementsByClassName ); // Support: IE<10 // Check if getElementById returns elements by name // The broken getElementById methods don't pick up programatically-set names, // so use a roundabout getElementsByName test support.getById = assert(function( div ) { docElem.appendChild( div ).id = expando; return !document.getElementsByName || !document.getElementsByName( expando ).length; }); // ID find and filter if ( support.getById ) { Expr.find["ID"] = function( id, context ) { if ( typeof context.getElementById !== "undefined" && documentIsHTML ) { var m = context.getElementById( id ); return m ? [ m ] : []; } }; Expr.filter["ID"] = function( id ) { var attrId = id.replace( runescape, funescape ); return function( elem ) { return elem.getAttribute("id") === attrId; }; }; } else { // Support: IE6/7 // getElementById is not reliable as a find shortcut delete Expr.find["ID"]; Expr.filter["ID"] = function( id ) { var attrId = id.replace( runescape, funescape ); return function( elem ) { var node = typeof elem.getAttributeNode !== "undefined" && elem.getAttributeNode("id"); return node && node.value === attrId; }; }; } // Tag Expr.find["TAG"] = support.getElementsByTagName ? function( tag, context ) { if ( typeof context.getElementsByTagName !== "undefined" ) { return context.getElementsByTagName( tag ); // DocumentFragment nodes don't have gEBTN } else if ( support.qsa ) { return context.querySelectorAll( tag ); } } : function( tag, context ) { var elem, tmp = [], i = 0, // By happy coincidence, a (broken) gEBTN appears on DocumentFragment nodes too results = context.getElementsByTagName( tag ); // Filter out possible comments if ( tag === "*" ) { while ( (elem = results[i++]) ) { if ( elem.nodeType === 1 ) { tmp.push( elem ); } } return tmp; } return results; }; // Class Expr.find["CLASS"] = support.getElementsByClassName && function( className, context ) { if ( typeof context.getElementsByClassName !== "undefined" && documentIsHTML ) { return context.getElementsByClassName( className ); } }; /* QSA/matchesSelector ---------------------------------------------------------------------- */ // QSA and matchesSelector support // matchesSelector(:active) reports false when true (IE9/Opera 11.5) rbuggyMatches = []; // qSa(:focus) reports false when true (Chrome 21) // We allow this because of a bug in IE8/9 that throws an error // whenever `document.activeElement` is accessed on an iframe // So, we allow :focus to pass through QSA all the time to avoid the IE error // See http://bugs.jquery.com/ticket/13378 rbuggyQSA = []; if ( (support.qsa = rnative.test( document.querySelectorAll )) ) { // Build QSA regex // Regex strategy adopted from Diego Perini assert(function( div ) { // Select is set to empty string on purpose // This is to test IE's treatment of not explicitly // setting a boolean content attribute, // since its presence should be enough // http://bugs.jquery.com/ticket/12359 docElem.appendChild( div ).innerHTML = "" + ""; // Support: IE8, Opera 11-12.16 // Nothing should be selected when empty strings follow ^= or $= or *= // The test attribute must be unknown in Opera but "safe" for WinRT // http://msdn.microsoft.com/en-us/library/ie/hh465388.aspx#attribute_section if ( div.querySelectorAll("[msallowcapture^='']").length ) { rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:''|\"\")" ); } // Support: IE8 // Boolean attributes and "value" are not treated correctly if ( !div.querySelectorAll("[selected]").length ) { rbuggyQSA.push( "\\[" + whitespace + "*(?:value|" + booleans + ")" ); } // Support: Chrome<29, Android<4.4, Safari<7.0+, iOS<7.0+, PhantomJS<1.9.8+ if ( !div.querySelectorAll( "[id~=" + expando + "-]" ).length ) { rbuggyQSA.push("~="); } // Webkit/Opera - :checked should return selected option elements // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked // IE8 throws error here and will not see later tests if ( !div.querySelectorAll(":checked").length ) { rbuggyQSA.push(":checked"); } // Support: Safari 8+, iOS 8+ // https://bugs.webkit.org/show_bug.cgi?id=136851 // In-page `selector#id sibing-combinator selector` fails if ( !div.querySelectorAll( "a#" + expando + "+*" ).length ) { rbuggyQSA.push(".#.+[+~]"); } }); assert(function( div ) { // Support: Windows 8 Native Apps // The type and name attributes are restricted during .innerHTML assignment var input = document.createElement("input"); input.setAttribute( "type", "hidden" ); div.appendChild( input ).setAttribute( "name", "D" ); // Support: IE8 // Enforce case-sensitivity of name attribute if ( div.querySelectorAll("[name=d]").length ) { rbuggyQSA.push( "name" + whitespace + "*[*^$|!~]?=" ); } // FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled) // IE8 throws error here and will not see later tests if ( !div.querySelectorAll(":enabled").length ) { rbuggyQSA.push( ":enabled", ":disabled" ); } // Opera 10-11 does not throw on post-comma invalid pseudos div.querySelectorAll("*,:x"); rbuggyQSA.push(",.*:"); }); } if ( (support.matchesSelector = rnative.test( (matches = docElem.matches || docElem.webkitMatchesSelector || docElem.mozMatchesSelector || docElem.oMatchesSelector || docElem.msMatchesSelector) )) ) { assert(function( div ) { // Check to see if it's possible to do matchesSelector // on a disconnected node (IE 9) support.disconnectedMatch = matches.call( div, "div" ); // This should fail with an exception // Gecko does not error, returns false instead matches.call( div, "[s!='']:x" ); rbuggyMatches.push( "!=", pseudos ); }); } rbuggyQSA = rbuggyQSA.length && new RegExp( rbuggyQSA.join("|") ); rbuggyMatches = rbuggyMatches.length && new RegExp( rbuggyMatches.join("|") ); /* Contains ---------------------------------------------------------------------- */ hasCompare = rnative.test( docElem.compareDocumentPosition ); // Element contains another // Purposefully self-exclusive // As in, an element does not contain itself contains = hasCompare || rnative.test( docElem.contains ) ? function( a, b ) { var adown = a.nodeType === 9 ? a.documentElement : a, bup = b && b.parentNode; return a === bup || !!( bup && bup.nodeType === 1 && ( adown.contains ? adown.contains( bup ) : a.compareDocumentPosition && a.compareDocumentPosition( bup ) & 16 )); } : function( a, b ) { if ( b ) { while ( (b = b.parentNode) ) { if ( b === a ) { return true; } } } return false; }; /* Sorting ---------------------------------------------------------------------- */ // Document order sorting sortOrder = hasCompare ? function( a, b ) { // Flag for duplicate removal if ( a === b ) { hasDuplicate = true; return 0; } // Sort on method existence if only one input has compareDocumentPosition var compare = !a.compareDocumentPosition - !b.compareDocumentPosition; if ( compare ) { return compare; } // Calculate position if both inputs belong to the same document compare = ( a.ownerDocument || a ) === ( b.ownerDocument || b ) ? a.compareDocumentPosition( b ) : // Otherwise we know they are disconnected 1; // Disconnected nodes if ( compare & 1 || (!support.sortDetached && b.compareDocumentPosition( a ) === compare) ) { // Choose the first element that is related to our preferred document if ( a === document || a.ownerDocument === preferredDoc && contains(preferredDoc, a) ) { return -1; } if ( b === document || b.ownerDocument === preferredDoc && contains(preferredDoc, b) ) { return 1; } // Maintain original order return sortInput ? ( indexOf( sortInput, a ) - indexOf( sortInput, b ) ) : 0; } return compare & 4 ? -1 : 1; } : function( a, b ) { // Exit early if the nodes are identical if ( a === b ) { hasDuplicate = true; return 0; } var cur, i = 0, aup = a.parentNode, bup = b.parentNode, ap = [ a ], bp = [ b ]; // Parentless nodes are either documents or disconnected if ( !aup || !bup ) { return a === document ? -1 : b === document ? 1 : aup ? -1 : bup ? 1 : sortInput ? ( indexOf( sortInput, a ) - indexOf( sortInput, b ) ) : 0; // If the nodes are siblings, we can do a quick check } else if ( aup === bup ) { return siblingCheck( a, b ); } // Otherwise we need full lists of their ancestors for comparison cur = a; while ( (cur = cur.parentNode) ) { ap.unshift( cur ); } cur = b; while ( (cur = cur.parentNode) ) { bp.unshift( cur ); } // Walk down the tree looking for a discrepancy while ( ap[i] === bp[i] ) { i++; } return i ? // Do a sibling check if the nodes have a common ancestor siblingCheck( ap[i], bp[i] ) : // Otherwise nodes in our document sort first ap[i] === preferredDoc ? -1 : bp[i] === preferredDoc ? 1 : 0; }; return document; }; Sizzle.matches = function( expr, elements ) { return Sizzle( expr, null, null, elements ); }; Sizzle.matchesSelector = function( elem, expr ) { // Set document vars if needed if ( ( elem.ownerDocument || elem ) !== document ) { setDocument( elem ); } // Make sure that attribute selectors are quoted expr = expr.replace( rattributeQuotes, "='$1']" ); if ( support.matchesSelector && documentIsHTML && !compilerCache[ expr + " " ] && ( !rbuggyMatches || !rbuggyMatches.test( expr ) ) && ( !rbuggyQSA || !rbuggyQSA.test( expr ) ) ) { try { var ret = matches.call( elem, expr ); // IE 9's matchesSelector returns false on disconnected nodes if ( ret || support.disconnectedMatch || // As well, disconnected nodes are said to be in a document // fragment in IE 9 elem.document && elem.document.nodeType !== 11 ) { return ret; } } catch (e) {} } return Sizzle( expr, document, null, [ elem ] ).length > 0; }; Sizzle.contains = function( context, elem ) { // Set document vars if needed if ( ( context.ownerDocument || context ) !== document ) { setDocument( context ); } return contains( context, elem ); }; Sizzle.attr = function( elem, name ) { // Set document vars if needed if ( ( elem.ownerDocument || elem ) !== document ) { setDocument( elem ); } var fn = Expr.attrHandle[ name.toLowerCase() ], // Don't get fooled by Object.prototype properties (jQuery #13807) val = fn && hasOwn.call( Expr.attrHandle, name.toLowerCase() ) ? fn( elem, name, !documentIsHTML ) : undefined; return val !== undefined ? val : support.attributes || !documentIsHTML ? elem.getAttribute( name ) : (val = elem.getAttributeNode(name)) && val.specified ? val.value : null; }; Sizzle.error = function( msg ) { throw new Error( "Syntax error, unrecognized expression: " + msg ); }; /** * Document sorting and removing duplicates * @param {ArrayLike} results */ Sizzle.uniqueSort = function( results ) { var elem, duplicates = [], j = 0, i = 0; // Unless we *know* we can detect duplicates, assume their presence hasDuplicate = !support.detectDuplicates; sortInput = !support.sortStable && results.slice( 0 ); results.sort( sortOrder ); if ( hasDuplicate ) { while ( (elem = results[i++]) ) { if ( elem === results[ i ] ) { j = duplicates.push( i ); } } while ( j-- ) { results.splice( duplicates[ j ], 1 ); } } // Clear input after sorting to release objects // See https://github.com/jquery/sizzle/pull/225 sortInput = null; return results; }; /** * Utility function for retrieving the text value of an array of DOM nodes * @param {Array|Element} elem */ getText = Sizzle.getText = function( elem ) { var node, ret = "", i = 0, nodeType = elem.nodeType; if ( !nodeType ) { // If no nodeType, this is expected to be an array while ( (node = elem[i++]) ) { // Do not traverse comment nodes ret += getText( node ); } } else if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) { // Use textContent for elements // innerText usage removed for consistency of new lines (jQuery #11153) if ( typeof elem.textContent === "string" ) { return elem.textContent; } else { // Traverse its children for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { ret += getText( elem ); } } } else if ( nodeType === 3 || nodeType === 4 ) { return elem.nodeValue; } // Do not include comment or processing instruction nodes return ret; }; Expr = Sizzle.selectors = { // Can be adjusted by the user cacheLength: 50, createPseudo: markFunction, match: matchExpr, attrHandle: {}, find: {}, relative: { ">": { dir: "parentNode", first: true }, " ": { dir: "parentNode" }, "+": { dir: "previousSibling", first: true }, "~": { dir: "previousSibling" } }, preFilter: { "ATTR": function( match ) { match[1] = match[1].replace( runescape, funescape ); // Move the given value to match[3] whether quoted or unquoted match[3] = ( match[3] || match[4] || match[5] || "" ).replace( runescape, funescape ); if ( match[2] === "~=" ) { match[3] = " " + match[3] + " "; } return match.slice( 0, 4 ); }, "CHILD": function( match ) { /* matches from matchExpr["CHILD"] 1 type (only|nth|...) 2 what (child|of-type) 3 argument (even|odd|\d*|\d*n([+-]\d+)?|...) 4 xn-component of xn+y argument ([+-]?\d*n|) 5 sign of xn-component 6 x of xn-component 7 sign of y-component 8 y of y-component */ match[1] = match[1].toLowerCase(); if ( match[1].slice( 0, 3 ) === "nth" ) { // nth-* requires argument if ( !match[3] ) { Sizzle.error( match[0] ); } // numeric x and y parameters for Expr.filter.CHILD // remember that false/true cast respectively to 0/1 match[4] = +( match[4] ? match[5] + (match[6] || 1) : 2 * ( match[3] === "even" || match[3] === "odd" ) ); match[5] = +( ( match[7] + match[8] ) || match[3] === "odd" ); // other types prohibit arguments } else if ( match[3] ) { Sizzle.error( match[0] ); } return match; }, "PSEUDO": function( match ) { var excess, unquoted = !match[6] && match[2]; if ( matchExpr["CHILD"].test( match[0] ) ) { return null; } // Accept quoted arguments as-is if ( match[3] ) { match[2] = match[4] || match[5] || ""; // Strip excess characters from unquoted arguments } else if ( unquoted && rpseudo.test( unquoted ) && // Get excess from tokenize (recursively) (excess = tokenize( unquoted, true )) && // advance to the next closing parenthesis (excess = unquoted.indexOf( ")", unquoted.length - excess ) - unquoted.length) ) { // excess is a negative index match[0] = match[0].slice( 0, excess ); match[2] = unquoted.slice( 0, excess ); } // Return only captures needed by the pseudo filter method (type and argument) return match.slice( 0, 3 ); } }, filter: { "TAG": function( nodeNameSelector ) { var nodeName = nodeNameSelector.replace( runescape, funescape ).toLowerCase(); return nodeNameSelector === "*" ? function() { return true; } : function( elem ) { return elem.nodeName && elem.nodeName.toLowerCase() === nodeName; }; }, "CLASS": function( className ) { var pattern = classCache[ className + " " ]; return pattern || (pattern = new RegExp( "(^|" + whitespace + ")" + className + "(" + whitespace + "|$)" )) && classCache( className, function( elem ) { return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== "undefined" && elem.getAttribute("class") || "" ); }); }, "ATTR": function( name, operator, check ) { return function( elem ) { var result = Sizzle.attr( elem, name ); if ( result == null ) { return operator === "!="; } if ( !operator ) { return true; } result += ""; return operator === "=" ? result === check : operator === "!=" ? result !== check : operator === "^=" ? check && result.indexOf( check ) === 0 : operator === "*=" ? check && result.indexOf( check ) > -1 : operator === "$=" ? check && result.slice( -check.length ) === check : operator === "~=" ? ( " " + result.replace( rwhitespace, " " ) + " " ).indexOf( check ) > -1 : operator === "|=" ? result === check || result.slice( 0, check.length + 1 ) === check + "-" : false; }; }, "CHILD": function( type, what, argument, first, last ) { var simple = type.slice( 0, 3 ) !== "nth", forward = type.slice( -4 ) !== "last", ofType = what === "of-type"; return first === 1 && last === 0 ? // Shortcut for :nth-*(n) function( elem ) { return !!elem.parentNode; } : function( elem, context, xml ) { var cache, uniqueCache, outerCache, node, nodeIndex, start, dir = simple !== forward ? "nextSibling" : "previousSibling", parent = elem.parentNode, name = ofType && elem.nodeName.toLowerCase(), useCache = !xml && !ofType, diff = false; if ( parent ) { // :(first|last|only)-(child|of-type) if ( simple ) { while ( dir ) { node = elem; while ( (node = node[ dir ]) ) { if ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) { return false; } } // Reverse direction for :only-* (if we haven't yet done so) start = dir = type === "only" && !start && "nextSibling"; } return true; } start = [ forward ? parent.firstChild : parent.lastChild ]; // non-xml :nth-child(...) stores cache data on `parent` if ( forward && useCache ) { // Seek `elem` from a previously-cached index // ...in a gzip-friendly way node = parent; outerCache = node[ expando ] || (node[ expando ] = {}); // Support: IE <9 only // Defend against cloned attroperties (jQuery gh-1709) uniqueCache = outerCache[ node.uniqueID ] || (outerCache[ node.uniqueID ] = {}); cache = uniqueCache[ type ] || []; nodeIndex = cache[ 0 ] === dirruns && cache[ 1 ]; diff = nodeIndex && cache[ 2 ]; node = nodeIndex && parent.childNodes[ nodeIndex ]; while ( (node = ++nodeIndex && node && node[ dir ] || // Fallback to seeking `elem` from the start (diff = nodeIndex = 0) || start.pop()) ) { // When found, cache indexes on `parent` and break if ( node.nodeType === 1 && ++diff && node === elem ) { uniqueCache[ type ] = [ dirruns, nodeIndex, diff ]; break; } } } else { // Use previously-cached element index if available if ( useCache ) { // ...in a gzip-friendly way node = elem; outerCache = node[ expando ] || (node[ expando ] = {}); // Support: IE <9 only // Defend against cloned attroperties (jQuery gh-1709) uniqueCache = outerCache[ node.uniqueID ] || (outerCache[ node.uniqueID ] = {}); cache = uniqueCache[ type ] || []; nodeIndex = cache[ 0 ] === dirruns && cache[ 1 ]; diff = nodeIndex; } // xml :nth-child(...) // or :nth-last-child(...) or :nth(-last)?-of-type(...) if ( diff === false ) { // Use the same loop as above to seek `elem` from the start while ( (node = ++nodeIndex && node && node[ dir ] || (diff = nodeIndex = 0) || start.pop()) ) { if ( ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) && ++diff ) { // Cache the index of each encountered element if ( useCache ) { outerCache = node[ expando ] || (node[ expando ] = {}); // Support: IE <9 only // Defend against cloned attroperties (jQuery gh-1709) uniqueCache = outerCache[ node.uniqueID ] || (outerCache[ node.uniqueID ] = {}); uniqueCache[ type ] = [ dirruns, diff ]; } if ( node === elem ) { break; } } } } } // Incorporate the offset, then check against cycle size diff -= last; return diff === first || ( diff % first === 0 && diff / first >= 0 ); } }; }, "PSEUDO": function( pseudo, argument ) { // pseudo-class names are case-insensitive // http://www.w3.org/TR/selectors/#pseudo-classes // Prioritize by case sensitivity in case custom pseudos are added with uppercase letters // Remember that setFilters inherits from pseudos var args, fn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] || Sizzle.error( "unsupported pseudo: " + pseudo ); // The user may use createPseudo to indicate that // arguments are needed to create the filter function // just as Sizzle does if ( fn[ expando ] ) { return fn( argument ); } // But maintain support for old signatures if ( fn.length > 1 ) { args = [ pseudo, pseudo, "", argument ]; return Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ? markFunction(function( seed, matches ) { var idx, matched = fn( seed, argument ), i = matched.length; while ( i-- ) { idx = indexOf( seed, matched[i] ); seed[ idx ] = !( matches[ idx ] = matched[i] ); } }) : function( elem ) { return fn( elem, 0, args ); }; } return fn; } }, pseudos: { // Potentially complex pseudos "not": markFunction(function( selector ) { // Trim the selector passed to compile // to avoid treating leading and trailing // spaces as combinators var input = [], results = [], matcher = compile( selector.replace( rtrim, "$1" ) ); return matcher[ expando ] ? markFunction(function( seed, matches, context, xml ) { var elem, unmatched = matcher( seed, null, xml, [] ), i = seed.length; // Match elements unmatched by `matcher` while ( i-- ) { if ( (elem = unmatched[i]) ) { seed[i] = !(matches[i] = elem); } } }) : function( elem, context, xml ) { input[0] = elem; matcher( input, null, xml, results ); // Don't keep the element (issue #299) input[0] = null; return !results.pop(); }; }), "has": markFunction(function( selector ) { return function( elem ) { return Sizzle( selector, elem ).length > 0; }; }), "contains": markFunction(function( text ) { text = text.replace( runescape, funescape ); return function( elem ) { return ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1; }; }), // "Whether an element is represented by a :lang() selector // is based solely on the element's language value // being equal to the identifier C, // or beginning with the identifier C immediately followed by "-". // The matching of C against the element's language value is performed case-insensitively. // The identifier C does not have to be a valid language name." // http://www.w3.org/TR/selectors/#lang-pseudo "lang": markFunction( function( lang ) { // lang value must be a valid identifier if ( !ridentifier.test(lang || "") ) { Sizzle.error( "unsupported lang: " + lang ); } lang = lang.replace( runescape, funescape ).toLowerCase(); return function( elem ) { var elemLang; do { if ( (elemLang = documentIsHTML ? elem.lang : elem.getAttribute("xml:lang") || elem.getAttribute("lang")) ) { elemLang = elemLang.toLowerCase(); return elemLang === lang || elemLang.indexOf( lang + "-" ) === 0; } } while ( (elem = elem.parentNode) && elem.nodeType === 1 ); return false; }; }), // Miscellaneous "target": function( elem ) { var hash = window.location && window.location.hash; return hash && hash.slice( 1 ) === elem.id; }, "root": function( elem ) { return elem === docElem; }, "focus": function( elem ) { return elem === document.activeElement && (!document.hasFocus || document.hasFocus()) && !!(elem.type || elem.href || ~elem.tabIndex); }, // Boolean properties "enabled": function( elem ) { return elem.disabled === false; }, "disabled": function( elem ) { return elem.disabled === true; }, "checked": function( elem ) { // In CSS3, :checked should return both checked and selected elements // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked var nodeName = elem.nodeName.toLowerCase(); return (nodeName === "input" && !!elem.checked) || (nodeName === "option" && !!elem.selected); }, "selected": function( elem ) { // Accessing this property makes selected-by-default // options in Safari work properly if ( elem.parentNode ) { elem.parentNode.selectedIndex; } return elem.selected === true; }, // Contents "empty": function( elem ) { // http://www.w3.org/TR/selectors/#empty-pseudo // :empty is negated by element (1) or content nodes (text: 3; cdata: 4; entity ref: 5), // but not by others (comment: 8; processing instruction: 7; etc.) // nodeType < 6 works because attributes (2) do not appear as children for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { if ( elem.nodeType < 6 ) { return false; } } return true; }, "parent": function( elem ) { return !Expr.pseudos["empty"]( elem ); }, // Element/input types "header": function( elem ) { return rheader.test( elem.nodeName ); }, "input": function( elem ) { return rinputs.test( elem.nodeName ); }, "button": function( elem ) { var name = elem.nodeName.toLowerCase(); return name === "input" && elem.type === "button" || name === "button"; }, "text": function( elem ) { var attr; return elem.nodeName.toLowerCase() === "input" && elem.type === "text" && // Support: IE<8 // New HTML5 attribute values (e.g., "search") appear with elem.type === "text" ( (attr = elem.getAttribute("type")) == null || attr.toLowerCase() === "text" ); }, // Position-in-collection "first": createPositionalPseudo(function() { return [ 0 ]; }), "last": createPositionalPseudo(function( matchIndexes, length ) { return [ length - 1 ]; }), "eq": createPositionalPseudo(function( matchIndexes, length, argument ) { return [ argument < 0 ? argument + length : argument ]; }), "even": createPositionalPseudo(function( matchIndexes, length ) { var i = 0; for ( ; i < length; i += 2 ) { matchIndexes.push( i ); } return matchIndexes; }), "odd": createPositionalPseudo(function( matchIndexes, length ) { var i = 1; for ( ; i < length; i += 2 ) { matchIndexes.push( i ); } return matchIndexes; }), "lt": createPositionalPseudo(function( matchIndexes, length, argument ) { var i = argument < 0 ? argument + length : argument; for ( ; --i >= 0; ) { matchIndexes.push( i ); } return matchIndexes; }), "gt": createPositionalPseudo(function( matchIndexes, length, argument ) { var i = argument < 0 ? argument + length : argument; for ( ; ++i < length; ) { matchIndexes.push( i ); } return matchIndexes; }) } }; Expr.pseudos["nth"] = Expr.pseudos["eq"]; // Add button/input type pseudos for ( i in { radio: true, checkbox: true, file: true, password: true, image: true } ) { Expr.pseudos[ i ] = createInputPseudo( i ); } for ( i in { submit: true, reset: true } ) { Expr.pseudos[ i ] = createButtonPseudo( i ); } // Easy API for creating new setFilters function setFilters() {} setFilters.prototype = Expr.filters = Expr.pseudos; Expr.setFilters = new setFilters(); tokenize = Sizzle.tokenize = function( selector, parseOnly ) { var matched, match, tokens, type, soFar, groups, preFilters, cached = tokenCache[ selector + " " ]; if ( cached ) { return parseOnly ? 0 : cached.slice( 0 ); } soFar = selector; groups = []; preFilters = Expr.preFilter; while ( soFar ) { // Comma and first run if ( !matched || (match = rcomma.exec( soFar )) ) { if ( match ) { // Don't consume trailing commas as valid soFar = soFar.slice( match[0].length ) || soFar; } groups.push( (tokens = []) ); } matched = false; // Combinators if ( (match = rcombinators.exec( soFar )) ) { matched = match.shift(); tokens.push({ value: matched, // Cast descendant combinators to space type: match[0].replace( rtrim, " " ) }); soFar = soFar.slice( matched.length ); } // Filters for ( type in Expr.filter ) { if ( (match = matchExpr[ type ].exec( soFar )) && (!preFilters[ type ] || (match = preFilters[ type ]( match ))) ) { matched = match.shift(); tokens.push({ value: matched, type: type, matches: match }); soFar = soFar.slice( matched.length ); } } if ( !matched ) { break; } } // Return the length of the invalid excess // if we're just parsing // Otherwise, throw an error or return tokens return parseOnly ? soFar.length : soFar ? Sizzle.error( selector ) : // Cache the tokens tokenCache( selector, groups ).slice( 0 ); }; function toSelector( tokens ) { var i = 0, len = tokens.length, selector = ""; for ( ; i < len; i++ ) { selector += tokens[i].value; } return selector; } function addCombinator( matcher, combinator, base ) { var dir = combinator.dir, checkNonElements = base && dir === "parentNode", doneName = done++; return combinator.first ? // Check against closest ancestor/preceding element function( elem, context, xml ) { while ( (elem = elem[ dir ]) ) { if ( elem.nodeType === 1 || checkNonElements ) { return matcher( elem, context, xml ); } } } : // Check against all ancestor/preceding elements function( elem, context, xml ) { var oldCache, uniqueCache, outerCache, newCache = [ dirruns, doneName ]; // We can't set arbitrary data on XML nodes, so they don't benefit from combinator caching if ( xml ) { while ( (elem = elem[ dir ]) ) { if ( elem.nodeType === 1 || checkNonElements ) { if ( matcher( elem, context, xml ) ) { return true; } } } } else { while ( (elem = elem[ dir ]) ) { if ( elem.nodeType === 1 || checkNonElements ) { outerCache = elem[ expando ] || (elem[ expando ] = {}); // Support: IE <9 only // Defend against cloned attroperties (jQuery gh-1709) uniqueCache = outerCache[ elem.uniqueID ] || (outerCache[ elem.uniqueID ] = {}); if ( (oldCache = uniqueCache[ dir ]) && oldCache[ 0 ] === dirruns && oldCache[ 1 ] === doneName ) { // Assign to newCache so results back-propagate to previous elements return (newCache[ 2 ] = oldCache[ 2 ]); } else { // Reuse newcache so results back-propagate to previous elements uniqueCache[ dir ] = newCache; // A match means we're done; a fail means we have to keep checking if ( (newCache[ 2 ] = matcher( elem, context, xml )) ) { return true; } } } } } }; } function elementMatcher( matchers ) { return matchers.length > 1 ? function( elem, context, xml ) { var i = matchers.length; while ( i-- ) { if ( !matchers[i]( elem, context, xml ) ) { return false; } } return true; } : matchers[0]; } function multipleContexts( selector, contexts, results ) { var i = 0, len = contexts.length; for ( ; i < len; i++ ) { Sizzle( selector, contexts[i], results ); } return results; } function condense( unmatched, map, filter, context, xml ) { var elem, newUnmatched = [], i = 0, len = unmatched.length, mapped = map != null; for ( ; i < len; i++ ) { if ( (elem = unmatched[i]) ) { if ( !filter || filter( elem, context, xml ) ) { newUnmatched.push( elem ); if ( mapped ) { map.push( i ); } } } } return newUnmatched; } function setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) { if ( postFilter && !postFilter[ expando ] ) { postFilter = setMatcher( postFilter ); } if ( postFinder && !postFinder[ expando ] ) { postFinder = setMatcher( postFinder, postSelector ); } return markFunction(function( seed, results, context, xml ) { var temp, i, elem, preMap = [], postMap = [], preexisting = results.length, // Get initial elements from seed or context elems = seed || multipleContexts( selector || "*", context.nodeType ? [ context ] : context, [] ), // Prefilter to get matcher input, preserving a map for seed-results synchronization matcherIn = preFilter && ( seed || !selector ) ? condense( elems, preMap, preFilter, context, xml ) : elems, matcherOut = matcher ? // If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results, postFinder || ( seed ? preFilter : preexisting || postFilter ) ? // ...intermediate processing is necessary [] : // ...otherwise use results directly results : matcherIn; // Find primary matches if ( matcher ) { matcher( matcherIn, matcherOut, context, xml ); } // Apply postFilter if ( postFilter ) { temp = condense( matcherOut, postMap ); postFilter( temp, [], context, xml ); // Un-match failing elements by moving them back to matcherIn i = temp.length; while ( i-- ) { if ( (elem = temp[i]) ) { matcherOut[ postMap[i] ] = !(matcherIn[ postMap[i] ] = elem); } } } if ( seed ) { if ( postFinder || preFilter ) { if ( postFinder ) { // Get the final matcherOut by condensing this intermediate into postFinder contexts temp = []; i = matcherOut.length; while ( i-- ) { if ( (elem = matcherOut[i]) ) { // Restore matcherIn since elem is not yet a final match temp.push( (matcherIn[i] = elem) ); } } postFinder( null, (matcherOut = []), temp, xml ); } // Move matched elements from seed to results to keep them synchronized i = matcherOut.length; while ( i-- ) { if ( (elem = matcherOut[i]) && (temp = postFinder ? indexOf( seed, elem ) : preMap[i]) > -1 ) { seed[temp] = !(results[temp] = elem); } } } // Add elements to results, through postFinder if defined } else { matcherOut = condense( matcherOut === results ? matcherOut.splice( preexisting, matcherOut.length ) : matcherOut ); if ( postFinder ) { postFinder( null, results, matcherOut, xml ); } else { push.apply( results, matcherOut ); } } }); } function matcherFromTokens( tokens ) { var checkContext, matcher, j, len = tokens.length, leadingRelative = Expr.relative[ tokens[0].type ], implicitRelative = leadingRelative || Expr.relative[" "], i = leadingRelative ? 1 : 0, // The foundational matcher ensures that elements are reachable from top-level context(s) matchContext = addCombinator( function( elem ) { return elem === checkContext; }, implicitRelative, true ), matchAnyContext = addCombinator( function( elem ) { return indexOf( checkContext, elem ) > -1; }, implicitRelative, true ), matchers = [ function( elem, context, xml ) { var ret = ( !leadingRelative && ( xml || context !== outermostContext ) ) || ( (checkContext = context).nodeType ? matchContext( elem, context, xml ) : matchAnyContext( elem, context, xml ) ); // Avoid hanging onto element (issue #299) checkContext = null; return ret; } ]; for ( ; i < len; i++ ) { if ( (matcher = Expr.relative[ tokens[i].type ]) ) { matchers = [ addCombinator(elementMatcher( matchers ), matcher) ]; } else { matcher = Expr.filter[ tokens[i].type ].apply( null, tokens[i].matches ); // Return special upon seeing a positional matcher if ( matcher[ expando ] ) { // Find the next relative operator (if any) for proper handling j = ++i; for ( ; j < len; j++ ) { if ( Expr.relative[ tokens[j].type ] ) { break; } } return setMatcher( i > 1 && elementMatcher( matchers ), i > 1 && toSelector( // If the preceding token was a descendant combinator, insert an implicit any-element `*` tokens.slice( 0, i - 1 ).concat({ value: tokens[ i - 2 ].type === " " ? "*" : "" }) ).replace( rtrim, "$1" ), matcher, i < j && matcherFromTokens( tokens.slice( i, j ) ), j < len && matcherFromTokens( (tokens = tokens.slice( j )) ), j < len && toSelector( tokens ) ); } matchers.push( matcher ); } } return elementMatcher( matchers ); } function matcherFromGroupMatchers( elementMatchers, setMatchers ) { var bySet = setMatchers.length > 0, byElement = elementMatchers.length > 0, superMatcher = function( seed, context, xml, results, outermost ) { var elem, j, matcher, matchedCount = 0, i = "0", unmatched = seed && [], setMatched = [], contextBackup = outermostContext, // We must always have either seed elements or outermost context elems = seed || byElement && Expr.find["TAG"]( "*", outermost ), // Use integer dirruns iff this is the outermost matcher dirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.random() || 0.1), len = elems.length; if ( outermost ) { outermostContext = context === document || context || outermost; } // Add elements passing elementMatchers directly to results // Support: IE<9, Safari // Tolerate NodeList properties (IE: "length"; Safari: ) matching elements by id for ( ; i !== len && (elem = elems[i]) != null; i++ ) { if ( byElement && elem ) { j = 0; if ( !context && elem.ownerDocument !== document ) { setDocument( elem ); xml = !documentIsHTML; } while ( (matcher = elementMatchers[j++]) ) { if ( matcher( elem, context || document, xml) ) { results.push( elem ); break; } } if ( outermost ) { dirruns = dirrunsUnique; } } // Track unmatched elements for set filters if ( bySet ) { // They will have gone through all possible matchers if ( (elem = !matcher && elem) ) { matchedCount--; } // Lengthen the array for every element, matched or not if ( seed ) { unmatched.push( elem ); } } } // `i` is now the count of elements visited above, and adding it to `matchedCount` // makes the latter nonnegative. matchedCount += i; // Apply set filters to unmatched elements // NOTE: This can be skipped if there are no unmatched elements (i.e., `matchedCount` // equals `i`), unless we didn't visit _any_ elements in the above loop because we have // no element matchers and no seed. // Incrementing an initially-string "0" `i` allows `i` to remain a string only in that // case, which will result in a "00" `matchedCount` that differs from `i` but is also // numerically zero. if ( bySet && i !== matchedCount ) { j = 0; while ( (matcher = setMatchers[j++]) ) { matcher( unmatched, setMatched, context, xml ); } if ( seed ) { // Reintegrate element matches to eliminate the need for sorting if ( matchedCount > 0 ) { while ( i-- ) { if ( !(unmatched[i] || setMatched[i]) ) { setMatched[i] = pop.call( results ); } } } // Discard index placeholder values to get only actual matches setMatched = condense( setMatched ); } // Add matches to results push.apply( results, setMatched ); // Seedless set matches succeeding multiple successful matchers stipulate sorting if ( outermost && !seed && setMatched.length > 0 && ( matchedCount + setMatchers.length ) > 1 ) { Sizzle.uniqueSort( results ); } } // Override manipulation of globals by nested matchers if ( outermost ) { dirruns = dirrunsUnique; outermostContext = contextBackup; } return unmatched; }; return bySet ? markFunction( superMatcher ) : superMatcher; } compile = Sizzle.compile = function( selector, match /* Internal Use Only */ ) { var i, setMatchers = [], elementMatchers = [], cached = compilerCache[ selector + " " ]; if ( !cached ) { // Generate a function of recursive functions that can be used to check each element if ( !match ) { match = tokenize( selector ); } i = match.length; while ( i-- ) { cached = matcherFromTokens( match[i] ); if ( cached[ expando ] ) { setMatchers.push( cached ); } else { elementMatchers.push( cached ); } } // Cache the compiled function cached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) ); // Save selector and tokenization cached.selector = selector; } return cached; }; /** * A low-level selection function that works with Sizzle's compiled * selector functions * @param {String|Function} selector A selector or a pre-compiled * selector function built with Sizzle.compile * @param {Element} context * @param {Array} [results] * @param {Array} [seed] A set of elements to match against */ select = Sizzle.select = function( selector, context, results, seed ) { var i, tokens, token, type, find, compiled = typeof selector === "function" && selector, match = !seed && tokenize( (selector = compiled.selector || selector) ); results = results || []; // Try to minimize operations if there is only one selector in the list and no seed // (the latter of which guarantees us context) if ( match.length === 1 ) { // Reduce context if the leading compound selector is an ID tokens = match[0] = match[0].slice( 0 ); if ( tokens.length > 2 && (token = tokens[0]).type === "ID" && support.getById && context.nodeType === 9 && documentIsHTML && Expr.relative[ tokens[1].type ] ) { context = ( Expr.find["ID"]( token.matches[0].replace(runescape, funescape), context ) || [] )[0]; if ( !context ) { return results; // Precompiled matchers will still verify ancestry, so step up a level } else if ( compiled ) { context = context.parentNode; } selector = selector.slice( tokens.shift().value.length ); } // Fetch a seed set for right-to-left matching i = matchExpr["needsContext"].test( selector ) ? 0 : tokens.length; while ( i-- ) { token = tokens[i]; // Abort if we hit a combinator if ( Expr.relative[ (type = token.type) ] ) { break; } if ( (find = Expr.find[ type ]) ) { // Search, expanding context for leading sibling combinators if ( (seed = find( token.matches[0].replace( runescape, funescape ), rsibling.test( tokens[0].type ) && testContext( context.parentNode ) || context )) ) { // If seed is empty or no tokens remain, we can return early tokens.splice( i, 1 ); selector = seed.length && toSelector( tokens ); if ( !selector ) { push.apply( results, seed ); return results; } break; } } } } // Compile and execute a filtering function if one is not provided // Provide `match` to avoid retokenization if we modified the selector above ( compiled || compile( selector, match ) )( seed, context, !documentIsHTML, results, !context || rsibling.test( selector ) && testContext( context.parentNode ) || context ); return results; }; // One-time assignments // Sort stability support.sortStable = expando.split("").sort( sortOrder ).join("") === expando; // Support: Chrome 14-35+ // Always assume duplicates if they aren't passed to the comparison function support.detectDuplicates = !!hasDuplicate; // Initialize against the default document setDocument(); // Support: Webkit<537.32 - Safari 6.0.3/Chrome 25 (fixed in Chrome 27) // Detached nodes confoundingly follow *each other* support.sortDetached = assert(function( div1 ) { // Should return 1, but returns 4 (following) return div1.compareDocumentPosition( document.createElement("div") ) & 1; }); // Support: IE<8 // Prevent attribute/property "interpolation" // http://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx if ( !assert(function( div ) { div.innerHTML = ""; return div.firstChild.getAttribute("href") === "#" ; }) ) { addHandle( "type|href|height|width", function( elem, name, isXML ) { if ( !isXML ) { return elem.getAttribute( name, name.toLowerCase() === "type" ? 1 : 2 ); } }); } // Support: IE<9 // Use defaultValue in place of getAttribute("value") if ( !support.attributes || !assert(function( div ) { div.innerHTML = ""; div.firstChild.setAttribute( "value", "" ); return div.firstChild.getAttribute( "value" ) === ""; }) ) { addHandle( "value", function( elem, name, isXML ) { if ( !isXML && elem.nodeName.toLowerCase() === "input" ) { return elem.defaultValue; } }); } // Support: IE<9 // Use getAttributeNode to fetch booleans when getAttribute lies if ( !assert(function( div ) { return div.getAttribute("disabled") == null; }) ) { addHandle( booleans, function( elem, name, isXML ) { var val; if ( !isXML ) { return elem[ name ] === true ? name.toLowerCase() : (val = elem.getAttributeNode( name )) && val.specified ? val.value : null; } }); } return Sizzle; })( window ); jQuery.find = Sizzle; jQuery.expr = Sizzle.selectors; jQuery.expr[ ":" ] = jQuery.expr.pseudos; jQuery.uniqueSort = jQuery.unique = Sizzle.uniqueSort; jQuery.text = Sizzle.getText; jQuery.isXMLDoc = Sizzle.isXML; jQuery.contains = Sizzle.contains; var dir = function( elem, dir, until ) { var matched = [], truncate = until !== undefined; while ( ( elem = elem[ dir ] ) && elem.nodeType !== 9 ) { if ( elem.nodeType === 1 ) { if ( truncate && jQuery( elem ).is( until ) ) { break; } matched.push( elem ); } } return matched; }; var siblings = function( n, elem ) { var matched = []; for ( ; n; n = n.nextSibling ) { if ( n.nodeType === 1 && n !== elem ) { matched.push( n ); } } return matched; }; var rneedsContext = jQuery.expr.match.needsContext; var rsingleTag = ( /^<([\w-]+)\s*\/?>(?:<\/\1>|)$/ ); var risSimple = /^.[^:#\[\.,]*$/; // Implement the identical functionality for filter and not function winnow( elements, qualifier, not ) { if ( jQuery.isFunction( qualifier ) ) { return jQuery.grep( elements, function( elem, i ) { /* jshint -W018 */ return !!qualifier.call( elem, i, elem ) !== not; } ); } if ( qualifier.nodeType ) { return jQuery.grep( elements, function( elem ) { return ( elem === qualifier ) !== not; } ); } if ( typeof qualifier === "string" ) { if ( risSimple.test( qualifier ) ) { return jQuery.filter( qualifier, elements, not ); } qualifier = jQuery.filter( qualifier, elements ); } return jQuery.grep( elements, function( elem ) { return ( indexOf.call( qualifier, elem ) > -1 ) !== not; } ); } jQuery.filter = function( expr, elems, not ) { var elem = elems[ 0 ]; if ( not ) { expr = ":not(" + expr + ")"; } return elems.length === 1 && elem.nodeType === 1 ? jQuery.find.matchesSelector( elem, expr ) ? [ elem ] : [] : jQuery.find.matches( expr, jQuery.grep( elems, function( elem ) { return elem.nodeType === 1; } ) ); }; jQuery.fn.extend( { find: function( selector ) { var i, len = this.length, ret = [], self = this; if ( typeof selector !== "string" ) { return this.pushStack( jQuery( selector ).filter( function() { for ( i = 0; i < len; i++ ) { if ( jQuery.contains( self[ i ], this ) ) { return true; } } } ) ); } for ( i = 0; i < len; i++ ) { jQuery.find( selector, self[ i ], ret ); } // Needed because $( selector, context ) becomes $( context ).find( selector ) ret = this.pushStack( len > 1 ? jQuery.unique( ret ) : ret ); ret.selector = this.selector ? this.selector + " " + selector : selector; return ret; }, filter: function( selector ) { return this.pushStack( winnow( this, selector || [], false ) ); }, not: function( selector ) { return this.pushStack( winnow( this, selector || [], true ) ); }, is: function( selector ) { return !!winnow( this, // If this is a positional/relative selector, check membership in the returned set // so $("p:first").is("p:last") won't return true for a doc with two "p". typeof selector === "string" && rneedsContext.test( selector ) ? jQuery( selector ) : selector || [], false ).length; } } ); // Initialize a jQuery object // A central reference to the root jQuery(document) var rootjQuery, // A simple way to check for HTML strings // Prioritize #id over to avoid XSS via location.hash (#9521) // Strict HTML recognition (#11290: must start with <) rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/, init = jQuery.fn.init = function( selector, context, root ) { var match, elem; // HANDLE: $(""), $(null), $(undefined), $(false) if ( !selector ) { return this; } // Method init() accepts an alternate rootjQuery // so migrate can support jQuery.sub (gh-2101) root = root || rootjQuery; // Handle HTML strings if ( typeof selector === "string" ) { if ( selector[ 0 ] === "<" && selector[ selector.length - 1 ] === ">" && selector.length >= 3 ) { // Assume that strings that start and end with <> are HTML and skip the regex check match = [ null, selector, null ]; } else { match = rquickExpr.exec( selector ); } // Match html or make sure no context is specified for #id if ( match && ( match[ 1 ] || !context ) ) { // HANDLE: $(html) -> $(array) if ( match[ 1 ] ) { context = context instanceof jQuery ? context[ 0 ] : context; // Option to run scripts is true for back-compat // Intentionally let the error be thrown if parseHTML is not present jQuery.merge( this, jQuery.parseHTML( match[ 1 ], context && context.nodeType ? context.ownerDocument || context : document, true ) ); // HANDLE: $(html, props) if ( rsingleTag.test( match[ 1 ] ) && jQuery.isPlainObject( context ) ) { for ( match in context ) { // Properties of context are called as methods if possible if ( jQuery.isFunction( this[ match ] ) ) { this[ match ]( context[ match ] ); // ...and otherwise set as attributes } else { this.attr( match, context[ match ] ); } } } return this; // HANDLE: $(#id) } else { elem = document.getElementById( match[ 2 ] ); // Support: Blackberry 4.6 // gEBID returns nodes no longer in the document (#6963) if ( elem && elem.parentNode ) { // Inject the element directly into the jQuery object this.length = 1; this[ 0 ] = elem; } this.context = document; this.selector = selector; return this; } // HANDLE: $(expr, $(...)) } else if ( !context || context.jquery ) { return ( context || root ).find( selector ); // HANDLE: $(expr, context) // (which is just equivalent to: $(context).find(expr) } else { return this.constructor( context ).find( selector ); } // HANDLE: $(DOMElement) } else if ( selector.nodeType ) { this.context = this[ 0 ] = selector; this.length = 1; return this; // HANDLE: $(function) // Shortcut for document ready } else if ( jQuery.isFunction( selector ) ) { return root.ready !== undefined ? root.ready( selector ) : // Execute immediately if ready is not present selector( jQuery ); } if ( selector.selector !== undefined ) { this.selector = selector.selector; this.context = selector.context; } return jQuery.makeArray( selector, this ); }; // Give the init function the jQuery prototype for later instantiation init.prototype = jQuery.fn; // Initialize central reference rootjQuery = jQuery( document ); var rparentsprev = /^(?:parents|prev(?:Until|All))/, // Methods guaranteed to produce a unique set when starting from a unique set guaranteedUnique = { children: true, contents: true, next: true, prev: true }; jQuery.fn.extend( { has: function( target ) { var targets = jQuery( target, this ), l = targets.length; return this.filter( function() { var i = 0; for ( ; i < l; i++ ) { if ( jQuery.contains( this, targets[ i ] ) ) { return true; } } } ); }, closest: function( selectors, context ) { var cur, i = 0, l = this.length, matched = [], pos = rneedsContext.test( selectors ) || typeof selectors !== "string" ? jQuery( selectors, context || this.context ) : 0; for ( ; i < l; i++ ) { for ( cur = this[ i ]; cur && cur !== context; cur = cur.parentNode ) { // Always skip document fragments if ( cur.nodeType < 11 && ( pos ? pos.index( cur ) > -1 : // Don't pass non-elements to Sizzle cur.nodeType === 1 && jQuery.find.matchesSelector( cur, selectors ) ) ) { matched.push( cur ); break; } } } return this.pushStack( matched.length > 1 ? jQuery.uniqueSort( matched ) : matched ); }, // Determine the position of an element within the set index: function( elem ) { // No argument, return index in parent if ( !elem ) { return ( this[ 0 ] && this[ 0 ].parentNode ) ? this.first().prevAll().length : -1; } // Index in selector if ( typeof elem === "string" ) { return indexOf.call( jQuery( elem ), this[ 0 ] ); } // Locate the position of the desired element return indexOf.call( this, // If it receives a jQuery object, the first element is used elem.jquery ? elem[ 0 ] : elem ); }, add: function( selector, context ) { return this.pushStack( jQuery.uniqueSort( jQuery.merge( this.get(), jQuery( selector, context ) ) ) ); }, addBack: function( selector ) { return this.add( selector == null ? this.prevObject : this.prevObject.filter( selector ) ); } } ); function sibling( cur, dir ) { while ( ( cur = cur[ dir ] ) && cur.nodeType !== 1 ) {} return cur; } jQuery.each( { parent: function( elem ) { var parent = elem.parentNode; return parent && parent.nodeType !== 11 ? parent : null; }, parents: function( elem ) { return dir( elem, "parentNode" ); }, parentsUntil: function( elem, i, until ) { return dir( elem, "parentNode", until ); }, next: function( elem ) { return sibling( elem, "nextSibling" ); }, prev: function( elem ) { return sibling( elem, "previousSibling" ); }, nextAll: function( elem ) { return dir( elem, "nextSibling" ); }, prevAll: function( elem ) { return dir( elem, "previousSibling" ); }, nextUntil: function( elem, i, until ) { return dir( elem, "nextSibling", until ); }, prevUntil: function( elem, i, until ) { return dir( elem, "previousSibling", until ); }, siblings: function( elem ) { return siblings( ( elem.parentNode || {} ).firstChild, elem ); }, children: function( elem ) { return siblings( elem.firstChild ); }, contents: function( elem ) { return elem.contentDocument || jQuery.merge( [], elem.childNodes ); } }, function( name, fn ) { jQuery.fn[ name ] = function( until, selector ) { var matched = jQuery.map( this, fn, until ); if ( name.slice( -5 ) !== "Until" ) { selector = until; } if ( selector && typeof selector === "string" ) { matched = jQuery.filter( selector, matched ); } if ( this.length > 1 ) { // Remove duplicates if ( !guaranteedUnique[ name ] ) { jQuery.uniqueSort( matched ); } // Reverse order for parents* and prev-derivatives if ( rparentsprev.test( name ) ) { matched.reverse(); } } return this.pushStack( matched ); }; } ); var rnotwhite = ( /\S+/g ); // Convert String-formatted options into Object-formatted ones function createOptions( options ) { var object = {}; jQuery.each( options.match( rnotwhite ) || [], function( _, flag ) { object[ flag ] = true; } ); return object; } /* * Create a callback list using the following parameters: * * options: an optional list of space-separated options that will change how * the callback list behaves or a more traditional option object * * By default a callback list will act like an event callback list and can be * "fired" multiple times. * * Possible options: * * once: will ensure the callback list can only be fired once (like a Deferred) * * memory: will keep track of previous values and will call any callback added * after the list has been fired right away with the latest "memorized" * values (like a Deferred) * * unique: will ensure a callback can only be added once (no duplicate in the list) * * stopOnFalse: interrupt callings when a callback returns false * */ jQuery.Callbacks = function( options ) { // Convert options from String-formatted to Object-formatted if needed // (we check in cache first) options = typeof options === "string" ? createOptions( options ) : jQuery.extend( {}, options ); var // Flag to know if list is currently firing firing, // Last fire value for non-forgettable lists memory, // Flag to know if list was already fired fired, // Flag to prevent firing locked, // Actual callback list list = [], // Queue of execution data for repeatable lists queue = [], // Index of currently firing callback (modified by add/remove as needed) firingIndex = -1, // Fire callbacks fire = function() { // Enforce single-firing locked = options.once; // Execute callbacks for all pending executions, // respecting firingIndex overrides and runtime changes fired = firing = true; for ( ; queue.length; firingIndex = -1 ) { memory = queue.shift(); while ( ++firingIndex < list.length ) { // Run callback and check for early termination if ( list[ firingIndex ].apply( memory[ 0 ], memory[ 1 ] ) === false && options.stopOnFalse ) { // Jump to end and forget the data so .add doesn't re-fire firingIndex = list.length; memory = false; } } } // Forget the data if we're done with it if ( !options.memory ) { memory = false; } firing = false; // Clean up if we're done firing for good if ( locked ) { // Keep an empty list if we have data for future add calls if ( memory ) { list = []; // Otherwise, this object is spent } else { list = ""; } } }, // Actual Callbacks object self = { // Add a callback or a collection of callbacks to the list add: function() { if ( list ) { // If we have memory from a past run, we should fire after adding if ( memory && !firing ) { firingIndex = list.length - 1; queue.push( memory ); } ( function add( args ) { jQuery.each( args, function( _, arg ) { if ( jQuery.isFunction( arg ) ) { if ( !options.unique || !self.has( arg ) ) { list.push( arg ); } } else if ( arg && arg.length && jQuery.type( arg ) !== "string" ) { // Inspect recursively add( arg ); } } ); } )( arguments ); if ( memory && !firing ) { fire(); } } return this; }, // Remove a callback from the list remove: function() { jQuery.each( arguments, function( _, arg ) { var index; while ( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) { list.splice( index, 1 ); // Handle firing indexes if ( index <= firingIndex ) { firingIndex--; } } } ); return this; }, // Check if a given callback is in the list. // If no argument is given, return whether or not list has callbacks attached. has: function( fn ) { return fn ? jQuery.inArray( fn, list ) > -1 : list.length > 0; }, // Remove all callbacks from the list empty: function() { if ( list ) { list = []; } return this; }, // Disable .fire and .add // Abort any current/pending executions // Clear all callbacks and values disable: function() { locked = queue = []; list = memory = ""; return this; }, disabled: function() { return !list; }, // Disable .fire // Also disable .add unless we have memory (since it would have no effect) // Abort any pending executions lock: function() { locked = queue = []; if ( !memory ) { list = memory = ""; } return this; }, locked: function() { return !!locked; }, // Call all callbacks with the given context and arguments fireWith: function( context, args ) { if ( !locked ) { args = args || []; args = [ context, args.slice ? args.slice() : args ]; queue.push( args ); if ( !firing ) { fire(); } } return this; }, // Call all the callbacks with the given arguments fire: function() { self.fireWith( this, arguments ); return this; }, // To know if the callbacks have already been called at least once fired: function() { return !!fired; } }; return self; }; jQuery.extend( { Deferred: function( func ) { var tuples = [ // action, add listener, listener list, final state [ "resolve", "done", jQuery.Callbacks( "once memory" ), "resolved" ], [ "reject", "fail", jQuery.Callbacks( "once memory" ), "rejected" ], [ "notify", "progress", jQuery.Callbacks( "memory" ) ] ], state = "pending", promise = { state: function() { return state; }, always: function() { deferred.done( arguments ).fail( arguments ); return this; }, then: function( /* fnDone, fnFail, fnProgress */ ) { var fns = arguments; return jQuery.Deferred( function( newDefer ) { jQuery.each( tuples, function( i, tuple ) { var fn = jQuery.isFunction( fns[ i ] ) && fns[ i ]; // deferred[ done | fail | progress ] for forwarding actions to newDefer deferred[ tuple[ 1 ] ]( function() { var returned = fn && fn.apply( this, arguments ); if ( returned && jQuery.isFunction( returned.promise ) ) { returned.promise() .progress( newDefer.notify ) .done( newDefer.resolve ) .fail( newDefer.reject ); } else { newDefer[ tuple[ 0 ] + "With" ]( this === promise ? newDefer.promise() : this, fn ? [ returned ] : arguments ); } } ); } ); fns = null; } ).promise(); }, // Get a promise for this deferred // If obj is provided, the promise aspect is added to the object promise: function( obj ) { return obj != null ? jQuery.extend( obj, promise ) : promise; } }, deferred = {}; // Keep pipe for back-compat promise.pipe = promise.then; // Add list-specific methods jQuery.each( tuples, function( i, tuple ) { var list = tuple[ 2 ], stateString = tuple[ 3 ]; // promise[ done | fail | progress ] = list.add promise[ tuple[ 1 ] ] = list.add; // Handle state if ( stateString ) { list.add( function() { // state = [ resolved | rejected ] state = stateString; // [ reject_list | resolve_list ].disable; progress_list.lock }, tuples[ i ^ 1 ][ 2 ].disable, tuples[ 2 ][ 2 ].lock ); } // deferred[ resolve | reject | notify ] deferred[ tuple[ 0 ] ] = function() { deferred[ tuple[ 0 ] + "With" ]( this === deferred ? promise : this, arguments ); return this; }; deferred[ tuple[ 0 ] + "With" ] = list.fireWith; } ); // Make the deferred a promise promise.promise( deferred ); // Call given func if any if ( func ) { func.call( deferred, deferred ); } // All done! return deferred; }, // Deferred helper when: function( subordinate /* , ..., subordinateN */ ) { var i = 0, resolveValues = slice.call( arguments ), length = resolveValues.length, // the count of uncompleted subordinates remaining = length !== 1 || ( subordinate && jQuery.isFunction( subordinate.promise ) ) ? length : 0, // the master Deferred. // If resolveValues consist of only a single Deferred, just use that. deferred = remaining === 1 ? subordinate : jQuery.Deferred(), // Update function for both resolve and progress values updateFunc = function( i, contexts, values ) { return function( value ) { contexts[ i ] = this; values[ i ] = arguments.length > 1 ? slice.call( arguments ) : value; if ( values === progressValues ) { deferred.notifyWith( contexts, values ); } else if ( !( --remaining ) ) { deferred.resolveWith( contexts, values ); } }; }, progressValues, progressContexts, resolveContexts; // Add listeners to Deferred subordinates; treat others as resolved if ( length > 1 ) { progressValues = new Array( length ); progressContexts = new Array( length ); resolveContexts = new Array( length ); for ( ; i < length; i++ ) { if ( resolveValues[ i ] && jQuery.isFunction( resolveValues[ i ].promise ) ) { resolveValues[ i ].promise() .progress( updateFunc( i, progressContexts, progressValues ) ) .done( updateFunc( i, resolveContexts, resolveValues ) ) .fail( deferred.reject ); } else { --remaining; } } } // If we're not waiting on anything, resolve the master if ( !remaining ) { deferred.resolveWith( resolveContexts, resolveValues ); } return deferred.promise(); } } ); // The deferred used on DOM ready var readyList; jQuery.fn.ready = function( fn ) { // Add the callback jQuery.ready.promise().done( fn ); return this; }; jQuery.extend( { // Is the DOM ready to be used? Set to true once it occurs. isReady: false, // A counter to track how many items to wait for before // the ready event fires. See #6781 readyWait: 1, // Hold (or release) the ready event holdReady: function( hold ) { if ( hold ) { jQuery.readyWait++; } else { jQuery.ready( true ); } }, // Handle when the DOM is ready ready: function( wait ) { // Abort if there are pending holds or we're already ready if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) { return; } // Remember that the DOM is ready jQuery.isReady = true; // If a normal DOM Ready event fired, decrement, and wait if need be if ( wait !== true && --jQuery.readyWait > 0 ) { return; } // If there are functions bound, to execute readyList.resolveWith( document, [ jQuery ] ); // Trigger any bound ready events if ( jQuery.fn.triggerHandler ) { jQuery( document ).triggerHandler( "ready" ); jQuery( document ).off( "ready" ); } } } ); /** * The ready event handler and self cleanup method */ function completed() { document.removeEventListener( "DOMContentLoaded", completed ); window.removeEventListener( "load", completed ); jQuery.ready(); } jQuery.ready.promise = function( obj ) { if ( !readyList ) { readyList = jQuery.Deferred(); // Catch cases where $(document).ready() is called // after the browser event has already occurred. // Support: IE9-10 only // Older IE sometimes signals "interactive" too soon if ( document.readyState === "complete" || ( document.readyState !== "loading" && !document.documentElement.doScroll ) ) { // Handle it asynchronously to allow scripts the opportunity to delay ready window.setTimeout( jQuery.ready ); } else { // Use the handy event callback document.addEventListener( "DOMContentLoaded", completed ); // A fallback to window.onload, that will always work window.addEventListener( "load", completed ); } } return readyList.promise( obj ); }; // Kick off the DOM ready check even if the user does not jQuery.ready.promise(); // Multifunctional method to get and set values of a collection // The value/s can optionally be executed if it's a function var access = function( elems, fn, key, value, chainable, emptyGet, raw ) { var i = 0, len = elems.length, bulk = key == null; // Sets many values if ( jQuery.type( key ) === "object" ) { chainable = true; for ( i in key ) { access( elems, fn, i, key[ i ], true, emptyGet, raw ); } // Sets one value } else if ( value !== undefined ) { chainable = true; if ( !jQuery.isFunction( value ) ) { raw = true; } if ( bulk ) { // Bulk operations run against the entire set if ( raw ) { fn.call( elems, value ); fn = null; // ...except when executing function values } else { bulk = fn; fn = function( elem, key, value ) { return bulk.call( jQuery( elem ), value ); }; } } if ( fn ) { for ( ; i < len; i++ ) { fn( elems[ i ], key, raw ? value : value.call( elems[ i ], i, fn( elems[ i ], key ) ) ); } } } return chainable ? elems : // Gets bulk ? fn.call( elems ) : len ? fn( elems[ 0 ], key ) : emptyGet; }; var acceptData = function( owner ) { // Accepts only: // - Node // - Node.ELEMENT_NODE // - Node.DOCUMENT_NODE // - Object // - Any /* jshint -W018 */ return owner.nodeType === 1 || owner.nodeType === 9 || !( +owner.nodeType ); }; function Data() { this.expando = jQuery.expando + Data.uid++; } Data.uid = 1; Data.prototype = { register: function( owner, initial ) { var value = initial || {}; // If it is a node unlikely to be stringify-ed or looped over // use plain assignment if ( owner.nodeType ) { owner[ this.expando ] = value; // Otherwise secure it in a non-enumerable, non-writable property // configurability must be true to allow the property to be // deleted with the delete operator } else { Object.defineProperty( owner, this.expando, { value: value, writable: true, configurable: true } ); } return owner[ this.expando ]; }, cache: function( owner ) { // We can accept data for non-element nodes in modern browsers, // but we should not, see #8335. // Always return an empty object. if ( !acceptData( owner ) ) { return {}; } // Check if the owner object already has a cache var value = owner[ this.expando ]; // If not, create one if ( !value ) { value = {}; // We can accept data for non-element nodes in modern browsers, // but we should not, see #8335. // Always return an empty object. if ( acceptData( owner ) ) { // If it is a node unlikely to be stringify-ed or looped over // use plain assignment if ( owner.nodeType ) { owner[ this.expando ] = value; // Otherwise secure it in a non-enumerable property // configurable must be true to allow the property to be // deleted when data is removed } else { Object.defineProperty( owner, this.expando, { value: value, configurable: true } ); } } } return value; }, set: function( owner, data, value ) { var prop, cache = this.cache( owner ); // Handle: [ owner, key, value ] args if ( typeof data === "string" ) { cache[ data ] = value; // Handle: [ owner, { properties } ] args } else { // Copy the properties one-by-one to the cache object for ( prop in data ) { cache[ prop ] = data[ prop ]; } } return cache; }, get: function( owner, key ) { return key === undefined ? this.cache( owner ) : owner[ this.expando ] && owner[ this.expando ][ key ]; }, access: function( owner, key, value ) { var stored; // In cases where either: // // 1. No key was specified // 2. A string key was specified, but no value provided // // Take the "read" path and allow the get method to determine // which value to return, respectively either: // // 1. The entire cache object // 2. The data stored at the key // if ( key === undefined || ( ( key && typeof key === "string" ) && value === undefined ) ) { stored = this.get( owner, key ); return stored !== undefined ? stored : this.get( owner, jQuery.camelCase( key ) ); } // When the key is not a string, or both a key and value // are specified, set or extend (existing objects) with either: // // 1. An object of properties // 2. A key and value // this.set( owner, key, value ); // Since the "set" path can have two possible entry points // return the expected data based on which path was taken[*] return value !== undefined ? value : key; }, remove: function( owner, key ) { var i, name, camel, cache = owner[ this.expando ]; if ( cache === undefined ) { return; } if ( key === undefined ) { this.register( owner ); } else { // Support array or space separated string of keys if ( jQuery.isArray( key ) ) { // If "name" is an array of keys... // When data is initially created, via ("key", "val") signature, // keys will be converted to camelCase. // Since there is no way to tell _how_ a key was added, remove // both plain key and camelCase key. #12786 // This will only penalize the array argument path. name = key.concat( key.map( jQuery.camelCase ) ); } else { camel = jQuery.camelCase( key ); // Try the string as a key before any manipulation if ( key in cache ) { name = [ key, camel ]; } else { // If a key with the spaces exists, use it. // Otherwise, create an array by matching non-whitespace name = camel; name = name in cache ? [ name ] : ( name.match( rnotwhite ) || [] ); } } i = name.length; while ( i-- ) { delete cache[ name[ i ] ]; } } // Remove the expando if there's no more data if ( key === undefined || jQuery.isEmptyObject( cache ) ) { // Support: Chrome <= 35-45+ // Webkit & Blink performance suffers when deleting properties // from DOM nodes, so set to undefined instead // https://code.google.com/p/chromium/issues/detail?id=378607 if ( owner.nodeType ) { owner[ this.expando ] = undefined; } else { delete owner[ this.expando ]; } } }, hasData: function( owner ) { var cache = owner[ this.expando ]; return cache !== undefined && !jQuery.isEmptyObject( cache ); } }; var dataPriv = new Data(); var dataUser = new Data(); // Implementation Summary // // 1. Enforce API surface and semantic compatibility with 1.9.x branch // 2. Improve the module's maintainability by reducing the storage // paths to a single mechanism. // 3. Use the same single mechanism to support "private" and "user" data. // 4. _Never_ expose "private" data to user code (TODO: Drop _data, _removeData) // 5. Avoid exposing implementation details on user objects (eg. expando properties) // 6. Provide a clear path for implementation upgrade to WeakMap in 2014 var rbrace = /^(?:\{[\w\W]*\}|\[[\w\W]*\])$/, rmultiDash = /[A-Z]/g; function dataAttr( elem, key, data ) { var name; // If nothing was found internally, try to fetch any // data from the HTML5 data-* attribute if ( data === undefined && elem.nodeType === 1 ) { name = "data-" + key.replace( rmultiDash, "-$&" ).toLowerCase(); data = elem.getAttribute( name ); if ( typeof data === "string" ) { try { data = data === "true" ? true : data === "false" ? false : data === "null" ? null : // Only convert to a number if it doesn't change the string +data + "" === data ? +data : rbrace.test( data ) ? jQuery.parseJSON( data ) : data; } catch ( e ) {} // Make sure we set the data so it isn't changed later dataUser.set( elem, key, data ); } else { data = undefined; } } return data; } jQuery.extend( { hasData: function( elem ) { return dataUser.hasData( elem ) || dataPriv.hasData( elem ); }, data: function( elem, name, data ) { return dataUser.access( elem, name, data ); }, removeData: function( elem, name ) { dataUser.remove( elem, name ); }, // TODO: Now that all calls to _data and _removeData have been replaced // with direct calls to dataPriv methods, these can be deprecated. _data: function( elem, name, data ) { return dataPriv.access( elem, name, data ); }, _removeData: function( elem, name ) { dataPriv.remove( elem, name ); } } ); jQuery.fn.extend( { data: function( key, value ) { var i, name, data, elem = this[ 0 ], attrs = elem && elem.attributes; // Gets all values if ( key === undefined ) { if ( this.length ) { data = dataUser.get( elem ); if ( elem.nodeType === 1 && !dataPriv.get( elem, "hasDataAttrs" ) ) { i = attrs.length; while ( i-- ) { // Support: IE11+ // The attrs elements can be null (#14894) if ( attrs[ i ] ) { name = attrs[ i ].name; if ( name.indexOf( "data-" ) === 0 ) { name = jQuery.camelCase( name.slice( 5 ) ); dataAttr( elem, name, data[ name ] ); } } } dataPriv.set( elem, "hasDataAttrs", true ); } } return data; } // Sets multiple values if ( typeof key === "object" ) { return this.each( function() { dataUser.set( this, key ); } ); } return access( this, function( value ) { var data, camelKey; // The calling jQuery object (element matches) is not empty // (and therefore has an element appears at this[ 0 ]) and the // `value` parameter was not undefined. An empty jQuery object // will result in `undefined` for elem = this[ 0 ] which will // throw an exception if an attempt to read a data cache is made. if ( elem && value === undefined ) { // Attempt to get data from the cache // with the key as-is data = dataUser.get( elem, key ) || // Try to find dashed key if it exists (gh-2779) // This is for 2.2.x only dataUser.get( elem, key.replace( rmultiDash, "-$&" ).toLowerCase() ); if ( data !== undefined ) { return data; } camelKey = jQuery.camelCase( key ); // Attempt to get data from the cache // with the key camelized data = dataUser.get( elem, camelKey ); if ( data !== undefined ) { return data; } // Attempt to "discover" the data in // HTML5 custom data-* attrs data = dataAttr( elem, camelKey, undefined ); if ( data !== undefined ) { return data; } // We tried really hard, but the data doesn't exist. return; } // Set the data... camelKey = jQuery.camelCase( key ); this.each( function() { // First, attempt to store a copy or reference of any // data that might've been store with a camelCased key. var data = dataUser.get( this, camelKey ); // For HTML5 data-* attribute interop, we have to // store property names with dashes in a camelCase form. // This might not apply to all properties...* dataUser.set( this, camelKey, value ); // *... In the case of properties that might _actually_ // have dashes, we need to also store a copy of that // unchanged property. if ( key.indexOf( "-" ) > -1 && data !== undefined ) { dataUser.set( this, key, value ); } } ); }, null, value, arguments.length > 1, null, true ); }, removeData: function( key ) { return this.each( function() { dataUser.remove( this, key ); } ); } } ); jQuery.extend( { queue: function( elem, type, data ) { var queue; if ( elem ) { type = ( type || "fx" ) + "queue"; queue = dataPriv.get( elem, type ); // Speed up dequeue by getting out quickly if this is just a lookup if ( data ) { if ( !queue || jQuery.isArray( data ) ) { queue = dataPriv.access( elem, type, jQuery.makeArray( data ) ); } else { queue.push( data ); } } return queue || []; } }, dequeue: function( elem, type ) { type = type || "fx"; var queue = jQuery.queue( elem, type ), startLength = queue.length, fn = queue.shift(), hooks = jQuery._queueHooks( elem, type ), next = function() { jQuery.dequeue( elem, type ); }; // If the fx queue is dequeued, always remove the progress sentinel if ( fn === "inprogress" ) { fn = queue.shift(); startLength--; } if ( fn ) { // Add a progress sentinel to prevent the fx queue from being // automatically dequeued if ( type === "fx" ) { queue.unshift( "inprogress" ); } // Clear up the last queue stop function delete hooks.stop; fn.call( elem, next, hooks ); } if ( !startLength && hooks ) { hooks.empty.fire(); } }, // Not public - generate a queueHooks object, or return the current one _queueHooks: function( elem, type ) { var key = type + "queueHooks"; return dataPriv.get( elem, key ) || dataPriv.access( elem, key, { empty: jQuery.Callbacks( "once memory" ).add( function() { dataPriv.remove( elem, [ type + "queue", key ] ); } ) } ); } } ); jQuery.fn.extend( { queue: function( type, data ) { var setter = 2; if ( typeof type !== "string" ) { data = type; type = "fx"; setter--; } if ( arguments.length < setter ) { return jQuery.queue( this[ 0 ], type ); } return data === undefined ? this : this.each( function() { var queue = jQuery.queue( this, type, data ); // Ensure a hooks for this queue jQuery._queueHooks( this, type ); if ( type === "fx" && queue[ 0 ] !== "inprogress" ) { jQuery.dequeue( this, type ); } } ); }, dequeue: function( type ) { return this.each( function() { jQuery.dequeue( this, type ); } ); }, clearQueue: function( type ) { return this.queue( type || "fx", [] ); }, // Get a promise resolved when queues of a certain type // are emptied (fx is the type by default) promise: function( type, obj ) { var tmp, count = 1, defer = jQuery.Deferred(), elements = this, i = this.length, resolve = function() { if ( !( --count ) ) { defer.resolveWith( elements, [ elements ] ); } }; if ( typeof type !== "string" ) { obj = type; type = undefined; } type = type || "fx"; while ( i-- ) { tmp = dataPriv.get( elements[ i ], type + "queueHooks" ); if ( tmp && tmp.empty ) { count++; tmp.empty.add( resolve ); } } resolve(); return defer.promise( obj ); } } ); var pnum = ( /[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/ ).source; var rcssNum = new RegExp( "^(?:([+-])=|)(" + pnum + ")([a-z%]*)$", "i" ); var cssExpand = [ "Top", "Right", "Bottom", "Left" ]; var isHidden = function( elem, el ) { // isHidden might be called from jQuery#filter function; // in that case, element will be second argument elem = el || elem; return jQuery.css( elem, "display" ) === "none" || !jQuery.contains( elem.ownerDocument, elem ); }; function adjustCSS( elem, prop, valueParts, tween ) { var adjusted, scale = 1, maxIterations = 20, currentValue = tween ? function() { return tween.cur(); } : function() { return jQuery.css( elem, prop, "" ); }, initial = currentValue(), unit = valueParts && valueParts[ 3 ] || ( jQuery.cssNumber[ prop ] ? "" : "px" ), // Starting value computation is required for potential unit mismatches initialInUnit = ( jQuery.cssNumber[ prop ] || unit !== "px" && +initial ) && rcssNum.exec( jQuery.css( elem, prop ) ); if ( initialInUnit && initialInUnit[ 3 ] !== unit ) { // Trust units reported by jQuery.css unit = unit || initialInUnit[ 3 ]; // Make sure we update the tween properties later on valueParts = valueParts || []; // Iteratively approximate from a nonzero starting point initialInUnit = +initial || 1; do { // If previous iteration zeroed out, double until we get *something*. // Use string for doubling so we don't accidentally see scale as unchanged below scale = scale || ".5"; // Adjust and apply initialInUnit = initialInUnit / scale; jQuery.style( elem, prop, initialInUnit + unit ); // Update scale, tolerating zero or NaN from tween.cur() // Break the loop if scale is unchanged or perfect, or if we've just had enough. } while ( scale !== ( scale = currentValue() / initial ) && scale !== 1 && --maxIterations ); } if ( valueParts ) { initialInUnit = +initialInUnit || +initial || 0; // Apply relative offset (+=/-=) if specified adjusted = valueParts[ 1 ] ? initialInUnit + ( valueParts[ 1 ] + 1 ) * valueParts[ 2 ] : +valueParts[ 2 ]; if ( tween ) { tween.unit = unit; tween.start = initialInUnit; tween.end = adjusted; } } return adjusted; } var rcheckableType = ( /^(?:checkbox|radio)$/i ); var rtagName = ( /<([\w:-]+)/ ); var rscriptType = ( /^$|\/(?:java|ecma)script/i ); // We have to close these tags to support XHTML (#13200) var wrapMap = { // Support: IE9 option: [ 1, "" ], // XHTML parsers do not magically insert elements in the // same way that tag soup parsers do. So we cannot shorten // this by omitting or other required elements. thead: [ 1, "", "
" ], col: [ 2, "", "
" ], tr: [ 2, "", "
" ], td: [ 3, "", "
" ], _default: [ 0, "", "" ] }; // Support: IE9 wrapMap.optgroup = wrapMap.option; wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; wrapMap.th = wrapMap.td; function getAll( context, tag ) { // Support: IE9-11+ // Use typeof to avoid zero-argument method invocation on host objects (#15151) var ret = typeof context.getElementsByTagName !== "undefined" ? context.getElementsByTagName( tag || "*" ) : typeof context.querySelectorAll !== "undefined" ? context.querySelectorAll( tag || "*" ) : []; return tag === undefined || tag && jQuery.nodeName( context, tag ) ? jQuery.merge( [ context ], ret ) : ret; } // Mark scripts as having already been evaluated function setGlobalEval( elems, refElements ) { var i = 0, l = elems.length; for ( ; i < l; i++ ) { dataPriv.set( elems[ i ], "globalEval", !refElements || dataPriv.get( refElements[ i ], "globalEval" ) ); } } var rhtml = /<|&#?\w+;/; function buildFragment( elems, context, scripts, selection, ignored ) { var elem, tmp, tag, wrap, contains, j, fragment = context.createDocumentFragment(), nodes = [], i = 0, l = elems.length; for ( ; i < l; i++ ) { elem = elems[ i ]; if ( elem || elem === 0 ) { // Add nodes directly if ( jQuery.type( elem ) === "object" ) { // Support: Android<4.1, PhantomJS<2 // push.apply(_, arraylike) throws on ancient WebKit jQuery.merge( nodes, elem.nodeType ? [ elem ] : elem ); // Convert non-html into a text node } else if ( !rhtml.test( elem ) ) { nodes.push( context.createTextNode( elem ) ); // Convert html into DOM nodes } else { tmp = tmp || fragment.appendChild( context.createElement( "div" ) ); // Deserialize a standard representation tag = ( rtagName.exec( elem ) || [ "", "" ] )[ 1 ].toLowerCase(); wrap = wrapMap[ tag ] || wrapMap._default; tmp.innerHTML = wrap[ 1 ] + jQuery.htmlPrefilter( elem ) + wrap[ 2 ]; // Descend through wrappers to the right content j = wrap[ 0 ]; while ( j-- ) { tmp = tmp.lastChild; } // Support: Android<4.1, PhantomJS<2 // push.apply(_, arraylike) throws on ancient WebKit jQuery.merge( nodes, tmp.childNodes ); // Remember the top-level container tmp = fragment.firstChild; // Ensure the created nodes are orphaned (#12392) tmp.textContent = ""; } } } // Remove wrapper from fragment fragment.textContent = ""; i = 0; while ( ( elem = nodes[ i++ ] ) ) { // Skip elements already in the context collection (trac-4087) if ( selection && jQuery.inArray( elem, selection ) > -1 ) { if ( ignored ) { ignored.push( elem ); } continue; } contains = jQuery.contains( elem.ownerDocument, elem ); // Append to fragment tmp = getAll( fragment.appendChild( elem ), "script" ); // Preserve script evaluation history if ( contains ) { setGlobalEval( tmp ); } // Capture executables if ( scripts ) { j = 0; while ( ( elem = tmp[ j++ ] ) ) { if ( rscriptType.test( elem.type || "" ) ) { scripts.push( elem ); } } } } return fragment; } ( function() { var fragment = document.createDocumentFragment(), div = fragment.appendChild( document.createElement( "div" ) ), input = document.createElement( "input" ); // Support: Android 4.0-4.3, Safari<=5.1 // Check state lost if the name is set (#11217) // Support: Windows Web Apps (WWA) // `name` and `type` must use .setAttribute for WWA (#14901) input.setAttribute( "type", "radio" ); input.setAttribute( "checked", "checked" ); input.setAttribute( "name", "t" ); div.appendChild( input ); // Support: Safari<=5.1, Android<4.2 // Older WebKit doesn't clone checked state correctly in fragments support.checkClone = div.cloneNode( true ).cloneNode( true ).lastChild.checked; // Support: IE<=11+ // Make sure textarea (and checkbox) defaultValue is properly cloned div.innerHTML = ""; support.noCloneChecked = !!div.cloneNode( true ).lastChild.defaultValue; } )(); var rkeyEvent = /^key/, rmouseEvent = /^(?:mouse|pointer|contextmenu|drag|drop)|click/, rtypenamespace = /^([^.]*)(?:\.(.+)|)/; function returnTrue() { return true; } function returnFalse() { return false; } // Support: IE9 // See #13393 for more info function safeActiveElement() { try { return document.activeElement; } catch ( err ) { } } function on( elem, types, selector, data, fn, one ) { var origFn, type; // Types can be a map of types/handlers if ( typeof types === "object" ) { // ( types-Object, selector, data ) if ( typeof selector !== "string" ) { // ( types-Object, data ) data = data || selector; selector = undefined; } for ( type in types ) { on( elem, type, selector, data, types[ type ], one ); } return elem; } if ( data == null && fn == null ) { // ( types, fn ) fn = selector; data = selector = undefined; } else if ( fn == null ) { if ( typeof selector === "string" ) { // ( types, selector, fn ) fn = data; data = undefined; } else { // ( types, data, fn ) fn = data; data = selector; selector = undefined; } } if ( fn === false ) { fn = returnFalse; } else if ( !fn ) { return elem; } if ( one === 1 ) { origFn = fn; fn = function( event ) { // Can use an empty set, since event contains the info jQuery().off( event ); return origFn.apply( this, arguments ); }; // Use same guid so caller can remove using origFn fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); } return elem.each( function() { jQuery.event.add( this, types, fn, data, selector ); } ); } /* * Helper functions for managing events -- not part of the public interface. * Props to Dean Edwards' addEvent library for many of the ideas. */ jQuery.event = { global: {}, add: function( elem, types, handler, data, selector ) { var handleObjIn, eventHandle, tmp, events, t, handleObj, special, handlers, type, namespaces, origType, elemData = dataPriv.get( elem ); // Don't attach events to noData or text/comment nodes (but allow plain objects) if ( !elemData ) { return; } // Caller can pass in an object of custom data in lieu of the handler if ( handler.handler ) { handleObjIn = handler; handler = handleObjIn.handler; selector = handleObjIn.selector; } // Make sure that the handler has a unique ID, used to find/remove it later if ( !handler.guid ) { handler.guid = jQuery.guid++; } // Init the element's event structure and main handler, if this is the first if ( !( events = elemData.events ) ) { events = elemData.events = {}; } if ( !( eventHandle = elemData.handle ) ) { eventHandle = elemData.handle = function( e ) { // Discard the second event of a jQuery.event.trigger() and // when an event is called after a page has unloaded return typeof jQuery !== "undefined" && jQuery.event.triggered !== e.type ? jQuery.event.dispatch.apply( elem, arguments ) : undefined; }; } // Handle multiple events separated by a space types = ( types || "" ).match( rnotwhite ) || [ "" ]; t = types.length; while ( t-- ) { tmp = rtypenamespace.exec( types[ t ] ) || []; type = origType = tmp[ 1 ]; namespaces = ( tmp[ 2 ] || "" ).split( "." ).sort(); // There *must* be a type, no attaching namespace-only handlers if ( !type ) { continue; } // If event changes its type, use the special event handlers for the changed type special = jQuery.event.special[ type ] || {}; // If selector defined, determine special event api type, otherwise given type type = ( selector ? special.delegateType : special.bindType ) || type; // Update special based on newly reset type special = jQuery.event.special[ type ] || {}; // handleObj is passed to all event handlers handleObj = jQuery.extend( { type: type, origType: origType, data: data, handler: handler, guid: handler.guid, selector: selector, needsContext: selector && jQuery.expr.match.needsContext.test( selector ), namespace: namespaces.join( "." ) }, handleObjIn ); // Init the event handler queue if we're the first if ( !( handlers = events[ type ] ) ) { handlers = events[ type ] = []; handlers.delegateCount = 0; // Only use addEventListener if the special events handler returns false if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { if ( elem.addEventListener ) { elem.addEventListener( type, eventHandle ); } } } if ( special.add ) { special.add.call( elem, handleObj ); if ( !handleObj.handler.guid ) { handleObj.handler.guid = handler.guid; } } // Add to the element's handler list, delegates in front if ( selector ) { handlers.splice( handlers.delegateCount++, 0, handleObj ); } else { handlers.push( handleObj ); } // Keep track of which events have ever been used, for event optimization jQuery.event.global[ type ] = true; } }, // Detach an event or set of events from an element remove: function( elem, types, handler, selector, mappedTypes ) { var j, origCount, tmp, events, t, handleObj, special, handlers, type, namespaces, origType, elemData = dataPriv.hasData( elem ) && dataPriv.get( elem ); if ( !elemData || !( events = elemData.events ) ) { return; } // Once for each type.namespace in types; type may be omitted types = ( types || "" ).match( rnotwhite ) || [ "" ]; t = types.length; while ( t-- ) { tmp = rtypenamespace.exec( types[ t ] ) || []; type = origType = tmp[ 1 ]; namespaces = ( tmp[ 2 ] || "" ).split( "." ).sort(); // Unbind all events (on this namespace, if provided) for the element if ( !type ) { for ( type in events ) { jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); } continue; } special = jQuery.event.special[ type ] || {}; type = ( selector ? special.delegateType : special.bindType ) || type; handlers = events[ type ] || []; tmp = tmp[ 2 ] && new RegExp( "(^|\\.)" + namespaces.join( "\\.(?:.*\\.|)" ) + "(\\.|$)" ); // Remove matching events origCount = j = handlers.length; while ( j-- ) { handleObj = handlers[ j ]; if ( ( mappedTypes || origType === handleObj.origType ) && ( !handler || handler.guid === handleObj.guid ) && ( !tmp || tmp.test( handleObj.namespace ) ) && ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) { handlers.splice( j, 1 ); if ( handleObj.selector ) { handlers.delegateCount--; } if ( special.remove ) { special.remove.call( elem, handleObj ); } } } // Remove generic event handler if we removed something and no more handlers exist // (avoids potential for endless recursion during removal of special event handlers) if ( origCount && !handlers.length ) { if ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) { jQuery.removeEvent( elem, type, elemData.handle ); } delete events[ type ]; } } // Remove data and the expando if it's no longer used if ( jQuery.isEmptyObject( events ) ) { dataPriv.remove( elem, "handle events" ); } }, dispatch: function( event ) { // Make a writable jQuery.Event from the native event object event = jQuery.event.fix( event ); var i, j, ret, matched, handleObj, handlerQueue = [], args = slice.call( arguments ), handlers = ( dataPriv.get( this, "events" ) || {} )[ event.type ] || [], special = jQuery.event.special[ event.type ] || {}; // Use the fix-ed jQuery.Event rather than the (read-only) native event args[ 0 ] = event; event.delegateTarget = this; // Call the preDispatch hook for the mapped type, and let it bail if desired if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { return; } // Determine handlers handlerQueue = jQuery.event.handlers.call( this, event, handlers ); // Run delegates first; they may want to stop propagation beneath us i = 0; while ( ( matched = handlerQueue[ i++ ] ) && !event.isPropagationStopped() ) { event.currentTarget = matched.elem; j = 0; while ( ( handleObj = matched.handlers[ j++ ] ) && !event.isImmediatePropagationStopped() ) { // Triggered event must either 1) have no namespace, or 2) have namespace(s) // a subset or equal to those in the bound event (both can have no namespace). if ( !event.rnamespace || event.rnamespace.test( handleObj.namespace ) ) { event.handleObj = handleObj; event.data = handleObj.data; ret = ( ( jQuery.event.special[ handleObj.origType ] || {} ).handle || handleObj.handler ).apply( matched.elem, args ); if ( ret !== undefined ) { if ( ( event.result = ret ) === false ) { event.preventDefault(); event.stopPropagation(); } } } } } // Call the postDispatch hook for the mapped type if ( special.postDispatch ) { special.postDispatch.call( this, event ); } return event.result; }, handlers: function( event, handlers ) { var i, matches, sel, handleObj, handlerQueue = [], delegateCount = handlers.delegateCount, cur = event.target; // Support (at least): Chrome, IE9 // Find delegate handlers // Black-hole SVG instance trees (#13180) // // Support: Firefox<=42+ // Avoid non-left-click in FF but don't block IE radio events (#3861, gh-2343) if ( delegateCount && cur.nodeType && ( event.type !== "click" || isNaN( event.button ) || event.button < 1 ) ) { for ( ; cur !== this; cur = cur.parentNode || this ) { // Don't check non-elements (#13208) // Don't process clicks on disabled elements (#6911, #8165, #11382, #11764) if ( cur.nodeType === 1 && ( cur.disabled !== true || event.type !== "click" ) ) { matches = []; for ( i = 0; i < delegateCount; i++ ) { handleObj = handlers[ i ]; // Don't conflict with Object.prototype properties (#13203) sel = handleObj.selector + " "; if ( matches[ sel ] === undefined ) { matches[ sel ] = handleObj.needsContext ? jQuery( sel, this ).index( cur ) > -1 : jQuery.find( sel, this, null, [ cur ] ).length; } if ( matches[ sel ] ) { matches.push( handleObj ); } } if ( matches.length ) { handlerQueue.push( { elem: cur, handlers: matches } ); } } } } // Add the remaining (directly-bound) handlers if ( delegateCount < handlers.length ) { handlerQueue.push( { elem: this, handlers: handlers.slice( delegateCount ) } ); } return handlerQueue; }, // Includes some event props shared by KeyEvent and MouseEvent props: ( "altKey bubbles cancelable ctrlKey currentTarget detail eventPhase " + "metaKey relatedTarget shiftKey target timeStamp view which" ).split( " " ), fixHooks: {}, keyHooks: { props: "char charCode key keyCode".split( " " ), filter: function( event, original ) { // Add which for key events if ( event.which == null ) { event.which = original.charCode != null ? original.charCode : original.keyCode; } return event; } }, mouseHooks: { props: ( "button buttons clientX clientY offsetX offsetY pageX pageY " + "screenX screenY toElement" ).split( " " ), filter: function( event, original ) { var eventDoc, doc, body, button = original.button; // Calculate pageX/Y if missing and clientX/Y available if ( event.pageX == null && original.clientX != null ) { eventDoc = event.target.ownerDocument || document; doc = eventDoc.documentElement; body = eventDoc.body; event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 ); event.pageY = original.clientY + ( doc && doc.scrollTop || body && body.scrollTop || 0 ) - ( doc && doc.clientTop || body && body.clientTop || 0 ); } // Add which for click: 1 === left; 2 === middle; 3 === right // Note: button is not normalized, so don't use it if ( !event.which && button !== undefined ) { event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) ); } return event; } }, fix: function( event ) { if ( event[ jQuery.expando ] ) { return event; } // Create a writable copy of the event object and normalize some properties var i, prop, copy, type = event.type, originalEvent = event, fixHook = this.fixHooks[ type ]; if ( !fixHook ) { this.fixHooks[ type ] = fixHook = rmouseEvent.test( type ) ? this.mouseHooks : rkeyEvent.test( type ) ? this.keyHooks : {}; } copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props; event = new jQuery.Event( originalEvent ); i = copy.length; while ( i-- ) { prop = copy[ i ]; event[ prop ] = originalEvent[ prop ]; } // Support: Cordova 2.5 (WebKit) (#13255) // All events should have a target; Cordova deviceready doesn't if ( !event.target ) { event.target = document; } // Support: Safari 6.0+, Chrome<28 // Target should not be a text node (#504, #13143) if ( event.target.nodeType === 3 ) { event.target = event.target.parentNode; } return fixHook.filter ? fixHook.filter( event, originalEvent ) : event; }, special: { load: { // Prevent triggered image.load events from bubbling to window.load noBubble: true }, focus: { // Fire native event if possible so blur/focus sequence is correct trigger: function() { if ( this !== safeActiveElement() && this.focus ) { this.focus(); return false; } }, delegateType: "focusin" }, blur: { trigger: function() { if ( this === safeActiveElement() && this.blur ) { this.blur(); return false; } }, delegateType: "focusout" }, click: { // For checkbox, fire native event so checked state will be right trigger: function() { if ( this.type === "checkbox" && this.click && jQuery.nodeName( this, "input" ) ) { this.click(); return false; } }, // For cross-browser consistency, don't fire native .click() on links _default: function( event ) { return jQuery.nodeName( event.target, "a" ); } }, beforeunload: { postDispatch: function( event ) { // Support: Firefox 20+ // Firefox doesn't alert if the returnValue field is not set. if ( event.result !== undefined && event.originalEvent ) { event.originalEvent.returnValue = event.result; } } } } }; jQuery.removeEvent = function( elem, type, handle ) { // This "if" is needed for plain objects if ( elem.removeEventListener ) { elem.removeEventListener( type, handle ); } }; jQuery.Event = function( src, props ) { // Allow instantiation without the 'new' keyword if ( !( this instanceof jQuery.Event ) ) { return new jQuery.Event( src, props ); } // Event object if ( src && src.type ) { this.originalEvent = src; this.type = src.type; // Events bubbling up the document may have been marked as prevented // by a handler lower down the tree; reflect the correct value. this.isDefaultPrevented = src.defaultPrevented || src.defaultPrevented === undefined && // Support: Android<4.0 src.returnValue === false ? returnTrue : returnFalse; // Event type } else { this.type = src; } // Put explicitly provided properties onto the event object if ( props ) { jQuery.extend( this, props ); } // Create a timestamp if incoming event doesn't have one this.timeStamp = src && src.timeStamp || jQuery.now(); // Mark it as fixed this[ jQuery.expando ] = true; }; // jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding // http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html jQuery.Event.prototype = { constructor: jQuery.Event, isDefaultPrevented: returnFalse, isPropagationStopped: returnFalse, isImmediatePropagationStopped: returnFalse, isSimulated: false, preventDefault: function() { var e = this.originalEvent; this.isDefaultPrevented = returnTrue; if ( e && !this.isSimulated ) { e.preventDefault(); } }, stopPropagation: function() { var e = this.originalEvent; this.isPropagationStopped = returnTrue; if ( e && !this.isSimulated ) { e.stopPropagation(); } }, stopImmediatePropagation: function() { var e = this.originalEvent; this.isImmediatePropagationStopped = returnTrue; if ( e && !this.isSimulated ) { e.stopImmediatePropagation(); } this.stopPropagation(); } }; // Create mouseenter/leave events using mouseover/out and event-time checks // so that event delegation works in jQuery. // Do the same for pointerenter/pointerleave and pointerover/pointerout // // Support: Safari 7 only // Safari sends mouseenter too often; see: // https://code.google.com/p/chromium/issues/detail?id=470258 // for the description of the bug (it existed in older Chrome versions as well). jQuery.each( { mouseenter: "mouseover", mouseleave: "mouseout", pointerenter: "pointerover", pointerleave: "pointerout" }, function( orig, fix ) { jQuery.event.special[ orig ] = { delegateType: fix, bindType: fix, handle: function( event ) { var ret, target = this, related = event.relatedTarget, handleObj = event.handleObj; // For mouseenter/leave call the handler if related is outside the target. // NB: No relatedTarget if the mouse left/entered the browser window if ( !related || ( related !== target && !jQuery.contains( target, related ) ) ) { event.type = handleObj.origType; ret = handleObj.handler.apply( this, arguments ); event.type = fix; } return ret; } }; } ); jQuery.fn.extend( { on: function( types, selector, data, fn ) { return on( this, types, selector, data, fn ); }, one: function( types, selector, data, fn ) { return on( this, types, selector, data, fn, 1 ); }, off: function( types, selector, fn ) { var handleObj, type; if ( types && types.preventDefault && types.handleObj ) { // ( event ) dispatched jQuery.Event handleObj = types.handleObj; jQuery( types.delegateTarget ).off( handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType, handleObj.selector, handleObj.handler ); return this; } if ( typeof types === "object" ) { // ( types-object [, selector] ) for ( type in types ) { this.off( type, selector, types[ type ] ); } return this; } if ( selector === false || typeof selector === "function" ) { // ( types [, fn] ) fn = selector; selector = undefined; } if ( fn === false ) { fn = returnFalse; } return this.each( function() { jQuery.event.remove( this, types, fn, selector ); } ); } } ); var rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:-]+)[^>]*)\/>/gi, // Support: IE 10-11, Edge 10240+ // In IE/Edge using regex groups here causes severe slowdowns. // See https://connect.microsoft.com/IE/feedback/details/1736512/ rnoInnerhtml = /\s*$/g; // Manipulating tables requires a tbody function manipulationTarget( elem, content ) { return jQuery.nodeName( elem, "table" ) && jQuery.nodeName( content.nodeType !== 11 ? content : content.firstChild, "tr" ) ? elem.getElementsByTagName( "tbody" )[ 0 ] || elem.appendChild( elem.ownerDocument.createElement( "tbody" ) ) : elem; } // Replace/restore the type attribute of script elements for safe DOM manipulation function disableScript( elem ) { elem.type = ( elem.getAttribute( "type" ) !== null ) + "/" + elem.type; return elem; } function restoreScript( elem ) { var match = rscriptTypeMasked.exec( elem.type ); if ( match ) { elem.type = match[ 1 ]; } else { elem.removeAttribute( "type" ); } return elem; } function cloneCopyEvent( src, dest ) { var i, l, type, pdataOld, pdataCur, udataOld, udataCur, events; if ( dest.nodeType !== 1 ) { return; } // 1. Copy private data: events, handlers, etc. if ( dataPriv.hasData( src ) ) { pdataOld = dataPriv.access( src ); pdataCur = dataPriv.set( dest, pdataOld ); events = pdataOld.events; if ( events ) { delete pdataCur.handle; pdataCur.events = {}; for ( type in events ) { for ( i = 0, l = events[ type ].length; i < l; i++ ) { jQuery.event.add( dest, type, events[ type ][ i ] ); } } } } // 2. Copy user data if ( dataUser.hasData( src ) ) { udataOld = dataUser.access( src ); udataCur = jQuery.extend( {}, udataOld ); dataUser.set( dest, udataCur ); } } // Fix IE bugs, see support tests function fixInput( src, dest ) { var nodeName = dest.nodeName.toLowerCase(); // Fails to persist the checked state of a cloned checkbox or radio button. if ( nodeName === "input" && rcheckableType.test( src.type ) ) { dest.checked = src.checked; // Fails to return the selected option to the default selected state when cloning options } else if ( nodeName === "input" || nodeName === "textarea" ) { dest.defaultValue = src.defaultValue; } } function domManip( collection, args, callback, ignored ) { // Flatten any nested arrays args = concat.apply( [], args ); var fragment, first, scripts, hasScripts, node, doc, i = 0, l = collection.length, iNoClone = l - 1, value = args[ 0 ], isFunction = jQuery.isFunction( value ); // We can't cloneNode fragments that contain checked, in WebKit if ( isFunction || ( l > 1 && typeof value === "string" && !support.checkClone && rchecked.test( value ) ) ) { return collection.each( function( index ) { var self = collection.eq( index ); if ( isFunction ) { args[ 0 ] = value.call( this, index, self.html() ); } domManip( self, args, callback, ignored ); } ); } if ( l ) { fragment = buildFragment( args, collection[ 0 ].ownerDocument, false, collection, ignored ); first = fragment.firstChild; if ( fragment.childNodes.length === 1 ) { fragment = first; } // Require either new content or an interest in ignored elements to invoke the callback if ( first || ignored ) { scripts = jQuery.map( getAll( fragment, "script" ), disableScript ); hasScripts = scripts.length; // Use the original fragment for the last item // instead of the first because it can end up // being emptied incorrectly in certain situations (#8070). for ( ; i < l; i++ ) { node = fragment; if ( i !== iNoClone ) { node = jQuery.clone( node, true, true ); // Keep references to cloned scripts for later restoration if ( hasScripts ) { // Support: Android<4.1, PhantomJS<2 // push.apply(_, arraylike) throws on ancient WebKit jQuery.merge( scripts, getAll( node, "script" ) ); } } callback.call( collection[ i ], node, i ); } if ( hasScripts ) { doc = scripts[ scripts.length - 1 ].ownerDocument; // Reenable scripts jQuery.map( scripts, restoreScript ); // Evaluate executable scripts on first document insertion for ( i = 0; i < hasScripts; i++ ) { node = scripts[ i ]; if ( rscriptType.test( node.type || "" ) && !dataPriv.access( node, "globalEval" ) && jQuery.contains( doc, node ) ) { if ( node.src ) { // Optional AJAX dependency, but won't run scripts if not present if ( jQuery._evalUrl ) { jQuery._evalUrl( node.src ); } } else { jQuery.globalEval( node.textContent.replace( rcleanScript, "" ) ); } } } } } } return collection; } function remove( elem, selector, keepData ) { var node, nodes = selector ? jQuery.filter( selector, elem ) : elem, i = 0; for ( ; ( node = nodes[ i ] ) != null; i++ ) { if ( !keepData && node.nodeType === 1 ) { jQuery.cleanData( getAll( node ) ); } if ( node.parentNode ) { if ( keepData && jQuery.contains( node.ownerDocument, node ) ) { setGlobalEval( getAll( node, "script" ) ); } node.parentNode.removeChild( node ); } } return elem; } jQuery.extend( { htmlPrefilter: function( html ) { return html.replace( rxhtmlTag, "<$1>" ); }, clone: function( elem, dataAndEvents, deepDataAndEvents ) { var i, l, srcElements, destElements, clone = elem.cloneNode( true ), inPage = jQuery.contains( elem.ownerDocument, elem ); // Fix IE cloning issues if ( !support.noCloneChecked && ( elem.nodeType === 1 || elem.nodeType === 11 ) && !jQuery.isXMLDoc( elem ) ) { // We eschew Sizzle here for performance reasons: http://jsperf.com/getall-vs-sizzle/2 destElements = getAll( clone ); srcElements = getAll( elem ); for ( i = 0, l = srcElements.length; i < l; i++ ) { fixInput( srcElements[ i ], destElements[ i ] ); } } // Copy the events from the original to the clone if ( dataAndEvents ) { if ( deepDataAndEvents ) { srcElements = srcElements || getAll( elem ); destElements = destElements || getAll( clone ); for ( i = 0, l = srcElements.length; i < l; i++ ) { cloneCopyEvent( srcElements[ i ], destElements[ i ] ); } } else { cloneCopyEvent( elem, clone ); } } // Preserve script evaluation history destElements = getAll( clone, "script" ); if ( destElements.length > 0 ) { setGlobalEval( destElements, !inPage && getAll( elem, "script" ) ); } // Return the cloned set return clone; }, cleanData: function( elems ) { var data, elem, type, special = jQuery.event.special, i = 0; for ( ; ( elem = elems[ i ] ) !== undefined; i++ ) { if ( acceptData( elem ) ) { if ( ( data = elem[ dataPriv.expando ] ) ) { if ( data.events ) { for ( type in data.events ) { if ( special[ type ] ) { jQuery.event.remove( elem, type ); // This is a shortcut to avoid jQuery.event.remove's overhead } else { jQuery.removeEvent( elem, type, data.handle ); } } } // Support: Chrome <= 35-45+ // Assign undefined instead of using delete, see Data#remove elem[ dataPriv.expando ] = undefined; } if ( elem[ dataUser.expando ] ) { // Support: Chrome <= 35-45+ // Assign undefined instead of using delete, see Data#remove elem[ dataUser.expando ] = undefined; } } } } } ); jQuery.fn.extend( { // Keep domManip exposed until 3.0 (gh-2225) domManip: domManip, detach: function( selector ) { return remove( this, selector, true ); }, remove: function( selector ) { return remove( this, selector ); }, text: function( value ) { return access( this, function( value ) { return value === undefined ? jQuery.text( this ) : this.empty().each( function() { if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { this.textContent = value; } } ); }, null, value, arguments.length ); }, append: function() { return domManip( this, arguments, function( elem ) { if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { var target = manipulationTarget( this, elem ); target.appendChild( elem ); } } ); }, prepend: function() { return domManip( this, arguments, function( elem ) { if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { var target = manipulationTarget( this, elem ); target.insertBefore( elem, target.firstChild ); } } ); }, before: function() { return domManip( this, arguments, function( elem ) { if ( this.parentNode ) { this.parentNode.insertBefore( elem, this ); } } ); }, after: function() { return domManip( this, arguments, function( elem ) { if ( this.parentNode ) { this.parentNode.insertBefore( elem, this.nextSibling ); } } ); }, empty: function() { var elem, i = 0; for ( ; ( elem = this[ i ] ) != null; i++ ) { if ( elem.nodeType === 1 ) { // Prevent memory leaks jQuery.cleanData( getAll( elem, false ) ); // Remove any remaining nodes elem.textContent = ""; } } return this; }, clone: function( dataAndEvents, deepDataAndEvents ) { dataAndEvents = dataAndEvents == null ? false : dataAndEvents; deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; return this.map( function() { return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); } ); }, html: function( value ) { return access( this, function( value ) { var elem = this[ 0 ] || {}, i = 0, l = this.length; if ( value === undefined && elem.nodeType === 1 ) { return elem.innerHTML; } // See if we can take a shortcut and just use innerHTML if ( typeof value === "string" && !rnoInnerhtml.test( value ) && !wrapMap[ ( rtagName.exec( value ) || [ "", "" ] )[ 1 ].toLowerCase() ] ) { value = jQuery.htmlPrefilter( value ); try { for ( ; i < l; i++ ) { elem = this[ i ] || {}; // Remove element nodes and prevent memory leaks if ( elem.nodeType === 1 ) { jQuery.cleanData( getAll( elem, false ) ); elem.innerHTML = value; } } elem = 0; // If using innerHTML throws an exception, use the fallback method } catch ( e ) {} } if ( elem ) { this.empty().append( value ); } }, null, value, arguments.length ); }, replaceWith: function() { var ignored = []; // Make the changes, replacing each non-ignored context element with the new content return domManip( this, arguments, function( elem ) { var parent = this.parentNode; if ( jQuery.inArray( this, ignored ) < 0 ) { jQuery.cleanData( getAll( this ) ); if ( parent ) { parent.replaceChild( elem, this ); } } // Force callback invocation }, ignored ); } } ); jQuery.each( { appendTo: "append", prependTo: "prepend", insertBefore: "before", insertAfter: "after", replaceAll: "replaceWith" }, function( name, original ) { jQuery.fn[ name ] = function( selector ) { var elems, ret = [], insert = jQuery( selector ), last = insert.length - 1, i = 0; for ( ; i <= last; i++ ) { elems = i === last ? this : this.clone( true ); jQuery( insert[ i ] )[ original ]( elems ); // Support: QtWebKit // .get() because push.apply(_, arraylike) throws push.apply( ret, elems.get() ); } return this.pushStack( ret ); }; } ); var iframe, elemdisplay = { // Support: Firefox // We have to pre-define these values for FF (#10227) HTML: "block", BODY: "block" }; /** * Retrieve the actual display of a element * @param {String} name nodeName of the element * @param {Object} doc Document object */ // Called only from within defaultDisplay function actualDisplay( name, doc ) { var elem = jQuery( doc.createElement( name ) ).appendTo( doc.body ), display = jQuery.css( elem[ 0 ], "display" ); // We don't have any data stored on the element, // so use "detach" method as fast way to get rid of the element elem.detach(); return display; } /** * Try to determine the default display value of an element * @param {String} nodeName */ function defaultDisplay( nodeName ) { var doc = document, display = elemdisplay[ nodeName ]; if ( !display ) { display = actualDisplay( nodeName, doc ); // If the simple way fails, read from inside an iframe if ( display === "none" || !display ) { // Use the already-created iframe if possible iframe = ( iframe || jQuery( "