h5netcdf-0.3.1/0000755131111500116100000000000012762421132013313 5ustar shoyereng00000000000000h5netcdf-0.3.1/h5netcdf/0000755131111500116100000000000012762421132015013 5ustar shoyereng00000000000000h5netcdf-0.3.1/h5netcdf/__init__.py0000644131111500116100000000033312751567353017141 0ustar shoyereng00000000000000""" h5netcdf ======== A Python library for the netCDF4 file-format that directly reads and writes HDF5 files via h5py, without using the Unidata netCDF library. """ from .core import File, Group, Variable, __version__ h5netcdf-0.3.1/h5netcdf/_chainmap.py0000644131111500116100000001102012751021724017277 0ustar shoyereng00000000000000# Backported from Python 3.4 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, # 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved from collections import MutableMapping try: from thread import get_ident except ImportError: from _thread import get_ident def recursive_repr(fillvalue='...'): 'Decorator to make a repr function return fillvalue for a recursive call' def decorating_function(user_function): repr_running = set() def wrapper(self): key = id(self), get_ident() if key in repr_running: return fillvalue repr_running.add(key) try: result = user_function(self) finally: repr_running.discard(key) return result # Can't use functools.wraps() here because of bootstrap issues wrapper.__module__ = getattr(user_function, '__module__') wrapper.__doc__ = getattr(user_function, '__doc__') wrapper.__name__ = getattr(user_function, '__name__') return wrapper return decorating_function class ChainMap(MutableMapping): ''' A ChainMap groups multiple dicts (or other mappings) together to create a single, updateable view. The underlying mappings are stored in a list. That list is public and can accessed or updated using the *maps* attribute. There is no other state. Lookups search the underlying mappings successively until a key is found. In contrast, writes, updates, and deletions only operate on the first mapping. ''' def __init__(self, *maps): '''Initialize a ChainMap by setting *maps* to the given mappings. If no mappings are provided, a single empty dictionary is used. ''' self.maps = list(maps) or [{}] # always at least one map def __missing__(self, key): raise KeyError(key) def __getitem__(self, key): for mapping in self.maps: try: return mapping[key] # can't use 'key in mapping' with defaultdict except KeyError: pass return self.__missing__(key) # support subclasses that define __missing__ def get(self, key, default=None): return self[key] if key in self else default def __len__(self): return len(set().union(*self.maps)) # reuses stored hash values if possible def __iter__(self): return iter(set().union(*self.maps)) def __contains__(self, key): return any(key in m for m in self.maps) def __bool__(self): return any(self.maps) @recursive_repr() def __repr__(self): return '{0.__class__.__name__}({1})'.format( self, ', '.join(repr(m) for m in self.maps)) @classmethod def fromkeys(cls, iterable, *args): 'Create a ChainMap with a single dict created from the iterable.' return cls(dict.fromkeys(iterable, *args)) def copy(self): 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' return self.__class__(self.maps[0].copy(), *self.maps[1:]) __copy__ = copy def new_child(self, m=None): # like Django's Context.push() ''' New ChainMap with a new map followed by all previous maps. If no map is provided, an empty dict is used. ''' if m is None: m = {} return self.__class__(m, *self.maps) @property def parents(self): # like Django's Context.pop() 'New ChainMap from maps[1:].' return self.__class__(*self.maps[1:]) def __setitem__(self, key, value): self.maps[0][key] = value def __delitem__(self, key): try: del self.maps[0][key] except KeyError: raise KeyError('Key not found in the first mapping: {!r}'.format(key)) def popitem(self): 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.' try: return self.maps[0].popitem() except KeyError: raise KeyError('No keys found in the first mapping.') def pop(self, key, *args): 'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].' try: return self.maps[0].pop(key, *args) except KeyError: raise KeyError('Key not found in the first mapping: {!r}'.format(key)) def clear(self): 'Clear maps[0], leaving maps[1:] intact.' self.maps[0].clear() h5netcdf-0.3.1/h5netcdf/attrs.py0000644131111500116100000000231612751567353016542 0ustar shoyereng00000000000000from collections import MutableMapping _hidden_attrs = frozenset(['REFERENCE_LIST', 'CLASS', 'DIMENSION_LIST', 'NAME', '_Netcdf4Dimid', '_Netcdf4Coordinates', '_nc3_strict', '_NCProperties']) class Attributes(MutableMapping): def __init__(self, h5attrs): self._h5attrs = h5attrs def __getitem__(self, key): if key in _hidden_attrs: raise KeyError(key) return self._h5attrs[key] def __setitem__(self, key, value): if key in _hidden_attrs: raise AttributeError('cannot write attribute with reserved name %r' % key) self._h5attrs[key] = value def __delitem__(self, key): del self._h5attrs[key] def __iter__(self): for key in self._h5attrs: if key not in _hidden_attrs: yield key def __len__(self): hidden_count = sum(1 if attr in self._h5attrs else 0 for attr in _hidden_attrs) return len(self._h5attrs) - hidden_count def __repr__(self): return '\n'.join(['%r' % type(self)] + ['%s: %r' % (k, v) for k, v in self.items()]) h5netcdf-0.3.1/h5netcdf/compat.py0000644131111500116100000000056512751554005016662 0ustar shoyereng00000000000000import sys PY2 = sys.version_info[0] < 3 if PY2: unicode = unicode else: unicode = str try: from collections import OrderedDict except ImportError: from ordereddict import OrderedDict if sys.version_info < (3, 4): # we need the optional argument to ChainMap.new_child from ._chainmap import ChainMap else: from collections import ChainMap h5netcdf-0.3.1/h5netcdf/core.py0000644131111500116100000003500512762421030016315 0ustar shoyereng00000000000000# For details on how netCDF4 builds on HDF5: # http://www.unidata.ucar.edu/software/netcdf/docs/file_format_specifications.html#netcdf_4_spec from collections import Mapping import h5py import numpy as np from .compat import ChainMap, OrderedDict from .attrs import Attributes from .dimensions import Dimensions from .utils import Frozen __version__ = '0.3.1' def _reverse_dict(dict_): return dict(zip(dict_.values(), dict_.keys())) def _join_h5paths(parent_path, child_path): return '/'.join([parent_path.rstrip('/'), child_path.lstrip('/')]) _NC_PROPERTIES = (u'version=1|h5netcdfversion=%s|hdf5libversion=%s' % (__version__, h5py.version.hdf5_version)) class BaseVariable(object): def __init__(self, parent, name, dimensions=None): self._parent = parent self._root = parent._root self._h5path = _join_h5paths(parent.name, name) self._dimensions = dimensions self._initialized = True @property def _h5ds(self): # Always refer to the root file and store not h5py object # subclasses: return self._root._h5file[self._h5path] @property def name(self): return self._h5ds.name def _lookup_dimensions(self): attrs = self._h5ds.attrs if '_Netcdf4Coordinates' in attrs: order_dim = _reverse_dict(self._parent._dim_order) return tuple(order_dim[coord_id] for coord_id in attrs['_Netcdf4Coordinates']) child_name = self.name.split('/')[-1] if child_name in self._parent.dimensions: return (child_name,) dims = [] for axis, dim in enumerate(self._h5ds.dims): # TODO: read dimension labels even if there is no associated # scale? it's not netCDF4 spec, but it is unambiguous... # Also: the netCDF lib can read HDF5 datasets with unlabeled # dimensions. if len(dim) == 0: raise ValueError('variable %r has no dimension scale ' 'associated with axis %s' % (self.name, axis)) name = dim[0].name.split('/')[-1] dims.append(name) return tuple(dims) @property def dimensions(self): if self._dimensions is None: self._dimensions = self._lookup_dimensions() return self._dimensions @property def shape(self): return self._h5ds.shape @property def ndim(self): return len(self.shape) def __len__(self): return self.shape[0] @property def dtype(self): return self._h5ds.dtype def __array__(self, *args, **kwargs): return self._h5ds.__array__(*args, **kwargs) def __getitem__(self, key): return self._h5ds[key] def __setitem__(self, key, value): self._h5ds[key] = value @property def attrs(self): return Attributes(self._h5ds.attrs) _cls_name = 'h5netcdf.Variable' def __repr__(self): if self._parent._root._closed: return '' % self._cls_name header = ('<%s %r: dimensions %s, shape %s, dtype %s>' % (self._cls_name, self.name, self.dimensions, self.shape, self.dtype)) return '\n'.join([header] + ['Attributes:'] + [' %s: %r' % (k, v) for k, v in self.attrs.items()]) class Variable(BaseVariable): @property def chunks(self): return self._h5ds.chunks @property def compression(self): return self._h5ds.compression @property def compression_opts(self): return self._h5ds.compression_opts @property def fletcher32(self): return self._h5ds.fletcher32 @property def shuffle(self): return self._h5ds.shuffle NOT_A_VARIABLE = b'This is a netCDF dimension but not a netCDF variable.' class _LazyObjectLookup(Mapping): def __init__(self, parent, object_cls): self._parent = parent self._object_cls = object_cls self._objects = OrderedDict() def __setitem__(self, name, obj): self._objects[name] = obj def add(self, name): self._objects[name] = None def __iter__(self): for name in self._objects: yield name def __len__(self): return len(self._objects) def __getitem__(self, key): if self._objects[key] is not None: return self._objects[key] else: self._objects[key] = self._object_cls(self._parent, key) return self._objects[key] class Group(Mapping): _variable_cls = Variable @property def _group_cls(self): return Group def __init__(self, parent, name): self._parent = parent self._root = parent._root self._h5path = _join_h5paths(parent.name, name) if parent is not self: self._dim_sizes = parent._dim_sizes.new_child() self._dim_order = parent._dim_order.new_child() self._variables = _LazyObjectLookup(self, self._variable_cls) self._groups = _LazyObjectLookup(self, self._group_cls) for k, v in self._h5group.items(): if isinstance(v, h5py.Group): self._groups.add(k) else: if v.attrs.get('CLASS') == b'DIMENSION_SCALE': dim_id = v.attrs.get('_Netcdf4Dimid') if '_Netcdf4Coordinates' in v.attrs: assert dim_id is not None coord_ids = v.attrs['_Netcdf4Coordinates'] size = v.shape[list(coord_ids).index(dim_id)] else: assert len(v.shape) == 1 size = v.size self._dim_sizes[k] = size if dim_id is None: dim_id = len(self._dim_order) self._dim_order[k] = dim_id if NOT_A_VARIABLE not in v.attrs.get('NAME', b''): var_name = k if k.startswith('_nc4_non_coord_'): var_name = k[len('_nc4_non_coord_'):] self._variables.add(var_name) self._initialized = True @property def _h5group(self): # Always refer to the root file and store not h5py object # subclasses: return self._root._h5file[self._h5path] @property def name(self): return self._h5group.name def _create_dimension(self, name, size=None): if name in self._dim_sizes.maps[0]: raise ValueError('dimension %r already exists' % name) if size is None: raise NotImplementedError('h5netcdf does not yet support ' 'unlimited dimensions') self._dim_sizes[name] = size self._dim_order[name] = len(self._dim_order) @property def dimensions(self): return Dimensions(self) @dimensions.setter def dimensions(self, value): for k, v in self._dim_sizes.maps[0].items(): if k in value: if v != value[k]: raise ValueError('cannot modify existing dimension %r' % k) else: raise ValueError('new dimensions do not include existing ' 'dimension %r' % k) self.dimensions.update(value) def _create_child_group(self, name): if name in self: raise ValueError('unable to create group %r (name already exists)' % name) self._h5group.create_group(name) self._groups[name] = self._group_cls(self, name) return self._groups[name] def _require_child_group(self, name): try: return self._groups[name] except KeyError: return self._create_child_group(name) def create_group(self, name): if name.startswith('/'): return self._root.create_group(name[1:]) keys = name.split('/') group = self for k in keys[:-1]: group = group._require_child_group(k) return group._create_child_group(keys[-1]) def _create_child_variable(self, name, dimensions, dtype, data, fillvalue, **kwargs): if name in self: raise ValueError('unable to create variable %r ' '(name already exists)' % name) if data is not None: data = np.asarray(data) for d, s in zip(dimensions, data.shape): if d not in self.dimensions: self.dimensions[d] = s if (dtype if dtype is not None else data.dtype) == np.bool_: raise TypeError('netCDF4 does not implement a boolean dtype') shape = tuple(self.dimensions[d] for d in dimensions) if name in self.dimensions and name not in dimensions: h5name = '_nc4_non_coord_' + name else: h5name = name self._h5group.create_dataset(h5name, shape, dtype=dtype, data=data, fillvalue=fillvalue, **kwargs) self._variables[h5name] = self._variable_cls(self, h5name, dimensions) variable = self._variables[h5name] if fillvalue is not None: value = variable.dtype.type(fillvalue) variable.attrs._h5attrs['_FillValue'] = value return variable def create_variable(self, name, dimensions=(), dtype=None, data=None, fillvalue=None, **kwargs): if name.startswith('/'): return self._root.create_variable(name[1:], dimensions, dtype, data, fillvalue, **kwargs) keys = name.split('/') group = self for k in keys[:-1]: group = group._require_child_group(k) return group._create_child_variable(keys[-1], dimensions, dtype, data, fillvalue, **kwargs) def _get_child(self, key): try: return self.variables[key] except KeyError: return self.groups[key] def __getitem__(self, key): if key.startswith('/'): return self._root[key[1:]] keys = key.split('/') item = self for k in keys: item = item._get_child(k) return item def __iter__(self): for name in self.groups: yield name for name in self.variables: yield name def __len__(self): return len(self.variables) + len(self.groups) def _create_dim_scales(self): dim_order = self._dim_order.maps[0] for dim in sorted(dim_order, key=lambda d: dim_order[d]): if dim not in self._h5group: size = self.dimensions[dim] self._h5group.create_dataset(dim, (size,), 'S1') h5ds = self._h5group[dim] h5ds.attrs['_Netcdf4Dimid'] = dim_order[dim] if len(h5ds.shape) > 1: dims = self._variables[dim].dimensions coord_ids = np.array([dim_order[d] for d in dims], 'int32') h5ds.attrs['_Netcdf4Coordinates'] = coord_ids scale_name = dim if dim in self.variables else NOT_A_VARIABLE h5ds.dims.create_scale(h5ds, scale_name) for subgroup in self.groups.values(): subgroup._create_dim_scales() def _attach_dim_scales(self, parent_h5groups=None): if parent_h5groups is None: all_h5groups = ChainMap(self._h5group) else: all_h5groups = parent_h5groups.new_child(self._h5group) for name, var in self.variables.items(): if name not in self.dimensions: for n, dim in enumerate(var.dimensions): var._h5ds.dims[n].attach_scale(all_h5groups[dim]) for subgroup in self.groups.values(): subgroup._attach_dim_scales(all_h5groups) @property def parent(self): return self._parent def flush(self): self._root.flush() sync = flush @property def groups(self): return Frozen(self._groups) @property def variables(self): return Frozen(self._variables) @property def attrs(self): return Attributes(self._h5group.attrs) _cls_name = 'h5netcdf.Group' def _repr_body(self): return (['Dimensions:'] + [' %s: %s' % (k, v) for k, v in self.dimensions.items()] + ['Groups:'] + [' %s' % g for g in self.groups] + ['Variables:'] + [' %s: %r %s' % (k, v.dimensions, v.dtype) for k, v in self.variables.items()] + ['Attributes:'] + [' %s: %r' % (k, v) for k, v in self.attrs.items()]) def __repr__(self): if self._root._closed: return '' % self._cls_name header = ('<%s %r (%s members)>' % (self._cls_name, self.name, len(self))) return '\n'.join([header] + self._repr_body()) class File(Group): def __init__(self, path, mode='a', **kwargs): try: self._h5file = h5py.File(path, mode, **kwargs) except Exception: self._closed = True raise else: self._closed = False self._dim_sizes = ChainMap() self._dim_order = ChainMap() self._mode = mode self._root = self self._h5path = '/' super(File, self).__init__(self, self._h5path) @property def mode(self): return self._h5file.mode @property def filename(self): return self._h5file.filename @property def parent(self): return None def flush(self): if 'r' not in self._mode: self._create_dim_scales() self._attach_dim_scales() self.attrs._h5attrs['_NCProperties'] = _NC_PROPERTIES sync = flush def close(self): if not self._closed: self.flush() self._h5file.close() self._closed = True __del__ = close def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() _cls_name = 'h5netcdf.File' def __repr__(self): if self._closed: return '' % self._cls_name header = '<%s %r (mode %s)>' % (self._cls_name, self.filename.split('/')[-1], self.mode) return '\n'.join([header] + self._repr_body()) h5netcdf-0.3.1/h5netcdf/dimensions.py0000644131111500116100000000142112751021724017534 0ustar shoyereng00000000000000from collections import MutableMapping class Dimensions(MutableMapping): def __init__(self, group): self._group = group def __getitem__(self, key): return self._group._dim_sizes[key] def __setitem__(self, key, value): self._group._create_dimension(key, value) def __delitem__(self, key): raise NotImplementedError('cannot yet delete dimensions') def __iter__(self): for key in self._group._dim_sizes: yield key def __len__(self): return len(self._group._dim_sizes) def __repr__(self): if self._group._root._closed: return '' return ('' % ', '.join('%s=%r' % (k, v) for k, v in self.items())) h5netcdf-0.3.1/h5netcdf/legacyapi.py0000644131111500116100000000515312751554005017333 0ustar shoyereng00000000000000import h5py from . import core from .compat import unicode class HasAttributesMixin(object): _initialized = False def getncattr(self, name): return self.attrs[name] def setncattr(self, name, value): self.attrs[name] = value def ncattrs(self): return list(self.attrs) def __getattr__(self, name): return self.attrs[name] def __setattr__(self, name, value): if self._initialized and name not in self.__dict__: self.attrs[name] = value else: object.__setattr__(self, name, value) class Variable(core.BaseVariable, HasAttributesMixin): _cls_name = 'h5netcdf.legacyapi.Variable' def chunking(self): chunks = self._h5ds.chunks if chunks is None: return 'contiguous' else: return chunks def filters(self): complevel = self._h5ds.compression_opts return {'complevel': 0 if complevel is None else complevel, 'fletcher32': self._h5ds.fletcher32, 'shuffle': self._h5ds.shuffle, 'zlib': self._h5ds.compression == 'gzip'} @property def dtype(self): dt = self._h5ds.dtype if h5py.check_dtype(vlen=dt) is unicode: return str return dt class Group(core.Group, HasAttributesMixin): _cls_name = 'h5netcdf.legacyapi.Group' _variable_cls = Variable @property def _group_cls(self): return Group createGroup = core.Group.create_group createDimension = core.Group._create_dimension def createVariable(self, varname, datatype, dimensions=(), zlib=False, complevel=4, shuffle=True, fletcher32=False, chunksizes=None, fill_value=None): if len(dimensions) == 0: # it's a scalar # rip off chunk and filter options for consistency with netCDF4-python chunksizes = None zlib = False fletcher32 = False shuffle = False if datatype is str: datatype = h5py.special_dtype(vlen=unicode) kwds = {} if zlib: # only add compression related keyword arguments if relevant (h5py # chokes otherwise) kwds['compression'] = 'gzip' kwds['compression_opts'] = complevel kwds['shuffle'] = shuffle return super(Group, self).create_variable( varname, dimensions, dtype=datatype, fletcher32=fletcher32, chunks=chunksizes, fillvalue=fill_value, **kwds) class Dataset(core.File, Group, HasAttributesMixin): _cls_name = 'h5netcdf.legacyapi.Dataset' h5netcdf-0.3.1/h5netcdf/utils.py0000644131111500116100000000126512751021724016532 0ustar shoyereng00000000000000from collections import Mapping import numpy as np class Frozen(Mapping): """Wrapper around an object implementing the mapping interface to make it immutable. If you really want to modify the mapping, the mutable version is saved under the `_mapping` attribute. """ def __init__(self, mapping): self._mapping = mapping def __getitem__(self, key): return self._mapping[key] def __iter__(self): return iter(self._mapping) def __len__(self): return len(self._mapping) def __contains__(self, key): return key in self._mapping def __repr__(self): return '%s(%r)' % (type(self).__name__, self._mapping) h5netcdf-0.3.1/h5netcdf.egg-info/0000755131111500116100000000000012762421132016505 5ustar shoyereng00000000000000h5netcdf-0.3.1/h5netcdf.egg-info/dependency_links.txt0000644131111500116100000000000112762421132022553 0ustar shoyereng00000000000000 h5netcdf-0.3.1/h5netcdf.egg-info/PKG-INFO0000644131111500116100000001574512762421132017616 0ustar shoyereng00000000000000Metadata-Version: 1.1 Name: h5netcdf Version: 0.3.1 Summary: netCDF4 via h5py Home-page: https://github.com/shoyer/h5netcdf Author: Stephan Hoyer Author-email: shoyer@gmail.com License: BSD Description: h5netcdf ======== .. image:: https://travis-ci.org/shoyer/h5netcdf.svg?branch=master :target: https://travis-ci.org/shoyer/h5netcdf .. image:: https://badge.fury.io/py/h5netcdf.svg :target: https://pypi.python.org/pypi/h5netcdf/ A Python interface for the netCDF4_ file-format that reads and writes HDF5 files API directly via h5py_, without relying on the Unidata netCDF library. .. _netCDF4: http://www.unidata.ucar.edu/software/netcdf/docs/file_format_specifications.html#netcdf_4_spec .. _h5py: http://www.h5py.org/ Why h5netcdf? ------------- - We've seen occasional reports of better performance with h5py than netCDF4-python, though in many cases performance is identical. For `one workflow`_, h5netcdf was reported to be almost **4x faster** than `netCDF4-python`_. - It has one less massive binary dependency (netCDF C). If you already have h5py installed, reading netCDF4 with h5netcdf may be much easier than installing netCDF4-Python. - Anecdotally, HDF5 users seem to be unexcited about switching to netCDF -- hopefully this will convince them that the netCDF4 is actually quite sane! - Finally, side-stepping the netCDF C library (and Cython bindings to it) gives us an easier way to identify the source of performance issues and bugs. .. _one workflow: https://github.com/Unidata/netcdf4-python/issues/390#issuecomment-93864839 .. _xarray: http://github.com/pydata/xarray/ Install ------- Ensure you have a recent version of h5py installed (I recommend using conda_). At least version 2.1 is required (for dimension scales); versions 2.3 and newer have been verified to work, though some tests only pass on h5py 2.6. Then: ``pip install h5netcdf`` .. _conda: http://conda.io/ Usage ----- h5netcdf has two APIs, a new API and a legacy API. Both interfaces currently reproduce most of the features of the netCDF interface, with the noteable exceptions of: - support for operations the rename or delete existing objects. - suport for creating unlimited dimensions. We simply haven't gotten around to implementing these features yet. Patches would be very welcome. New API ~~~~~~~ The new API supports direct hierarchical access of variables and groups. Its design is an adaptation of h5py to the netCDF data model. For example: .. code-block:: python import h5netcdf import numpy as np with h5netcdf.File('mydata.nc', 'w') as f: # set dimensions with a dictionary f.dimensions = {'x': 5} # and update them with a dict-like interface # f.dimensions['x'] = 5 # f.dimensions.update({'x': 5}) v = f.create_variable('hello', ('x',), float) v[:] = np.ones(5) # you don't need to create groups first # you also don't need to create dimensions first if you supply data # with the new variable v = f.create_variable('/grouped/data', ('y',), data=np.arange(10)) # access and modify attributes with a dict-like interface v.attrs['foo'] = 'bar' # you can access variables and groups directly using a hierarchical # keys like h5py print(f['/grouped/data']) Legacy API ~~~~~~~~~~ The legacy API is designed for compatibility with netCDF4-python_. To use it, import ``h5netcdf.legacyapi``: .. _netCDF4-python: https://github.com/Unidata/netcdf4-python .. code-block:: python import h5netcdf.legacyapi as netCDF4 # everything here would also work with this instead: # import netCDF4 import numpy as np with netCDF4.Dataset('mydata.nc', 'w') as ds: ds.createDimension('x', 5) v = ds.createVariable('hello', float, ('x',)) v[:] = np.ones(5) g = ds.createGroup('grouped') g.createDimension('y', 10) g.createVariable('data', 'i8', ('y',)) v = g['data'] v[:] = np.arange(10) v.foo = 'bar' print(ds.groups['grouped'].variables['data']) The legacy API is designed to be easy to try-out for netCDF4-python users, but it is not an exact match. Here is an incomplete list of functionality we don't include: - Utility functions ``chartostring``, ``num2date``, etc., that are not directly necessary for writing netCDF files. - We don't support the ``endian`` argument to ``createVariable``. The h5py API does not appear to offer this feature. - h5netcdf variables do not support automatic masking or scaling (e.g., of values matching the ``_FillValue`` attribute). We prefer to leave this functionality to client libraries (e.g., xarray_), which can implement their exact desired scaling behavior. Change Log ---------- Version 0.3.1: - Fix garbage collection issue. - Add missing ``.flush()`` method for groups. - Allow creating dimensions of size 0. Version 0.3.0: - Datasets are now loaded lazily. This should increase performance when opening files with a large number of groups and/or variables. - Support for writing arrays of variable length unicode strings with `dtype=str` via the legacy API. - h5netcdf now writes the _NCProperties attribute for identifying netCDF4 files. License ------- `3-clause BSD`_ .. _3-clause BSD: https://github.com/shoyer/h5netcdf/blob/master/LICENSE Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: OS Independent Classifier: Intended Audience :: Science/Research Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Topic :: Scientific/Engineering h5netcdf-0.3.1/h5netcdf.egg-info/requires.txt0000644131111500116100000000000512762421132021100 0ustar shoyereng00000000000000h5py h5netcdf-0.3.1/h5netcdf.egg-info/SOURCES.txt0000644131111500116100000000054612762421132020376 0ustar shoyereng00000000000000LICENSE MANIFEST.in README.rst setup.py h5netcdf/__init__.py h5netcdf/_chainmap.py h5netcdf/attrs.py h5netcdf/compat.py h5netcdf/core.py h5netcdf/dimensions.py h5netcdf/legacyapi.py h5netcdf/utils.py h5netcdf.egg-info/PKG-INFO h5netcdf.egg-info/SOURCES.txt h5netcdf.egg-info/dependency_links.txt h5netcdf.egg-info/requires.txt h5netcdf.egg-info/top_level.txth5netcdf-0.3.1/h5netcdf.egg-info/top_level.txt0000644131111500116100000000001112762421132021227 0ustar shoyereng00000000000000h5netcdf h5netcdf-0.3.1/LICENSE0000644131111500116100000000273312751021724014326 0ustar shoyereng00000000000000Copyright (c) 2015, Stephan Hoyer All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. h5netcdf-0.3.1/MANIFEST.in0000644131111500116100000000002012751554270015051 0ustar shoyereng00000000000000include LICENSE h5netcdf-0.3.1/PKG-INFO0000644131111500116100000001574512762421132014424 0ustar shoyereng00000000000000Metadata-Version: 1.1 Name: h5netcdf Version: 0.3.1 Summary: netCDF4 via h5py Home-page: https://github.com/shoyer/h5netcdf Author: Stephan Hoyer Author-email: shoyer@gmail.com License: BSD Description: h5netcdf ======== .. image:: https://travis-ci.org/shoyer/h5netcdf.svg?branch=master :target: https://travis-ci.org/shoyer/h5netcdf .. image:: https://badge.fury.io/py/h5netcdf.svg :target: https://pypi.python.org/pypi/h5netcdf/ A Python interface for the netCDF4_ file-format that reads and writes HDF5 files API directly via h5py_, without relying on the Unidata netCDF library. .. _netCDF4: http://www.unidata.ucar.edu/software/netcdf/docs/file_format_specifications.html#netcdf_4_spec .. _h5py: http://www.h5py.org/ Why h5netcdf? ------------- - We've seen occasional reports of better performance with h5py than netCDF4-python, though in many cases performance is identical. For `one workflow`_, h5netcdf was reported to be almost **4x faster** than `netCDF4-python`_. - It has one less massive binary dependency (netCDF C). If you already have h5py installed, reading netCDF4 with h5netcdf may be much easier than installing netCDF4-Python. - Anecdotally, HDF5 users seem to be unexcited about switching to netCDF -- hopefully this will convince them that the netCDF4 is actually quite sane! - Finally, side-stepping the netCDF C library (and Cython bindings to it) gives us an easier way to identify the source of performance issues and bugs. .. _one workflow: https://github.com/Unidata/netcdf4-python/issues/390#issuecomment-93864839 .. _xarray: http://github.com/pydata/xarray/ Install ------- Ensure you have a recent version of h5py installed (I recommend using conda_). At least version 2.1 is required (for dimension scales); versions 2.3 and newer have been verified to work, though some tests only pass on h5py 2.6. Then: ``pip install h5netcdf`` .. _conda: http://conda.io/ Usage ----- h5netcdf has two APIs, a new API and a legacy API. Both interfaces currently reproduce most of the features of the netCDF interface, with the noteable exceptions of: - support for operations the rename or delete existing objects. - suport for creating unlimited dimensions. We simply haven't gotten around to implementing these features yet. Patches would be very welcome. New API ~~~~~~~ The new API supports direct hierarchical access of variables and groups. Its design is an adaptation of h5py to the netCDF data model. For example: .. code-block:: python import h5netcdf import numpy as np with h5netcdf.File('mydata.nc', 'w') as f: # set dimensions with a dictionary f.dimensions = {'x': 5} # and update them with a dict-like interface # f.dimensions['x'] = 5 # f.dimensions.update({'x': 5}) v = f.create_variable('hello', ('x',), float) v[:] = np.ones(5) # you don't need to create groups first # you also don't need to create dimensions first if you supply data # with the new variable v = f.create_variable('/grouped/data', ('y',), data=np.arange(10)) # access and modify attributes with a dict-like interface v.attrs['foo'] = 'bar' # you can access variables and groups directly using a hierarchical # keys like h5py print(f['/grouped/data']) Legacy API ~~~~~~~~~~ The legacy API is designed for compatibility with netCDF4-python_. To use it, import ``h5netcdf.legacyapi``: .. _netCDF4-python: https://github.com/Unidata/netcdf4-python .. code-block:: python import h5netcdf.legacyapi as netCDF4 # everything here would also work with this instead: # import netCDF4 import numpy as np with netCDF4.Dataset('mydata.nc', 'w') as ds: ds.createDimension('x', 5) v = ds.createVariable('hello', float, ('x',)) v[:] = np.ones(5) g = ds.createGroup('grouped') g.createDimension('y', 10) g.createVariable('data', 'i8', ('y',)) v = g['data'] v[:] = np.arange(10) v.foo = 'bar' print(ds.groups['grouped'].variables['data']) The legacy API is designed to be easy to try-out for netCDF4-python users, but it is not an exact match. Here is an incomplete list of functionality we don't include: - Utility functions ``chartostring``, ``num2date``, etc., that are not directly necessary for writing netCDF files. - We don't support the ``endian`` argument to ``createVariable``. The h5py API does not appear to offer this feature. - h5netcdf variables do not support automatic masking or scaling (e.g., of values matching the ``_FillValue`` attribute). We prefer to leave this functionality to client libraries (e.g., xarray_), which can implement their exact desired scaling behavior. Change Log ---------- Version 0.3.1: - Fix garbage collection issue. - Add missing ``.flush()`` method for groups. - Allow creating dimensions of size 0. Version 0.3.0: - Datasets are now loaded lazily. This should increase performance when opening files with a large number of groups and/or variables. - Support for writing arrays of variable length unicode strings with `dtype=str` via the legacy API. - h5netcdf now writes the _NCProperties attribute for identifying netCDF4 files. License ------- `3-clause BSD`_ .. _3-clause BSD: https://github.com/shoyer/h5netcdf/blob/master/LICENSE Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: OS Independent Classifier: Intended Audience :: Science/Research Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Topic :: Scientific/Engineering h5netcdf-0.3.1/README.rst0000644131111500116100000001210312762420772015010 0ustar shoyereng00000000000000h5netcdf ======== .. image:: https://travis-ci.org/shoyer/h5netcdf.svg?branch=master :target: https://travis-ci.org/shoyer/h5netcdf .. image:: https://badge.fury.io/py/h5netcdf.svg :target: https://pypi.python.org/pypi/h5netcdf/ A Python interface for the netCDF4_ file-format that reads and writes HDF5 files API directly via h5py_, without relying on the Unidata netCDF library. .. _netCDF4: http://www.unidata.ucar.edu/software/netcdf/docs/file_format_specifications.html#netcdf_4_spec .. _h5py: http://www.h5py.org/ Why h5netcdf? ------------- - We've seen occasional reports of better performance with h5py than netCDF4-python, though in many cases performance is identical. For `one workflow`_, h5netcdf was reported to be almost **4x faster** than `netCDF4-python`_. - It has one less massive binary dependency (netCDF C). If you already have h5py installed, reading netCDF4 with h5netcdf may be much easier than installing netCDF4-Python. - Anecdotally, HDF5 users seem to be unexcited about switching to netCDF -- hopefully this will convince them that the netCDF4 is actually quite sane! - Finally, side-stepping the netCDF C library (and Cython bindings to it) gives us an easier way to identify the source of performance issues and bugs. .. _one workflow: https://github.com/Unidata/netcdf4-python/issues/390#issuecomment-93864839 .. _xarray: http://github.com/pydata/xarray/ Install ------- Ensure you have a recent version of h5py installed (I recommend using conda_). At least version 2.1 is required (for dimension scales); versions 2.3 and newer have been verified to work, though some tests only pass on h5py 2.6. Then: ``pip install h5netcdf`` .. _conda: http://conda.io/ Usage ----- h5netcdf has two APIs, a new API and a legacy API. Both interfaces currently reproduce most of the features of the netCDF interface, with the noteable exceptions of: - support for operations the rename or delete existing objects. - suport for creating unlimited dimensions. We simply haven't gotten around to implementing these features yet. Patches would be very welcome. New API ~~~~~~~ The new API supports direct hierarchical access of variables and groups. Its design is an adaptation of h5py to the netCDF data model. For example: .. code-block:: python import h5netcdf import numpy as np with h5netcdf.File('mydata.nc', 'w') as f: # set dimensions with a dictionary f.dimensions = {'x': 5} # and update them with a dict-like interface # f.dimensions['x'] = 5 # f.dimensions.update({'x': 5}) v = f.create_variable('hello', ('x',), float) v[:] = np.ones(5) # you don't need to create groups first # you also don't need to create dimensions first if you supply data # with the new variable v = f.create_variable('/grouped/data', ('y',), data=np.arange(10)) # access and modify attributes with a dict-like interface v.attrs['foo'] = 'bar' # you can access variables and groups directly using a hierarchical # keys like h5py print(f['/grouped/data']) Legacy API ~~~~~~~~~~ The legacy API is designed for compatibility with netCDF4-python_. To use it, import ``h5netcdf.legacyapi``: .. _netCDF4-python: https://github.com/Unidata/netcdf4-python .. code-block:: python import h5netcdf.legacyapi as netCDF4 # everything here would also work with this instead: # import netCDF4 import numpy as np with netCDF4.Dataset('mydata.nc', 'w') as ds: ds.createDimension('x', 5) v = ds.createVariable('hello', float, ('x',)) v[:] = np.ones(5) g = ds.createGroup('grouped') g.createDimension('y', 10) g.createVariable('data', 'i8', ('y',)) v = g['data'] v[:] = np.arange(10) v.foo = 'bar' print(ds.groups['grouped'].variables['data']) The legacy API is designed to be easy to try-out for netCDF4-python users, but it is not an exact match. Here is an incomplete list of functionality we don't include: - Utility functions ``chartostring``, ``num2date``, etc., that are not directly necessary for writing netCDF files. - We don't support the ``endian`` argument to ``createVariable``. The h5py API does not appear to offer this feature. - h5netcdf variables do not support automatic masking or scaling (e.g., of values matching the ``_FillValue`` attribute). We prefer to leave this functionality to client libraries (e.g., xarray_), which can implement their exact desired scaling behavior. Change Log ---------- Version 0.3.1: - Fix garbage collection issue. - Add missing ``.flush()`` method for groups. - Allow creating dimensions of size 0. Version 0.3.0: - Datasets are now loaded lazily. This should increase performance when opening files with a large number of groups and/or variables. - Support for writing arrays of variable length unicode strings with `dtype=str` via the legacy API. - h5netcdf now writes the _NCProperties attribute for identifying netCDF4 files. License ------- `3-clause BSD`_ .. _3-clause BSD: https://github.com/shoyer/h5netcdf/blob/master/LICENSE h5netcdf-0.3.1/setup.cfg0000644131111500116100000000007312762421132015134 0ustar shoyereng00000000000000[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 h5netcdf-0.3.1/setup.py0000644131111500116100000000202612762421003015022 0ustar shoyereng00000000000000import os from setuptools import setup, find_packages CLASSIFIERS = [ 'Development Status :: 4 - Beta', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Intended Audience :: Science/Research', 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Topic :: Scientific/Engineering', ] setup(name='h5netcdf', description='netCDF4 via h5py', long_description=(open('README.rst').read() if os.path.exists('README.rst') else ''), version='0.3.1', license='BSD', classifiers=CLASSIFIERS, author='Stephan Hoyer', author_email='shoyer@gmail.com', url='https://github.com/shoyer/h5netcdf', install_requires=['h5py'], tests_require=['netCDF4', 'pytest'], packages=find_packages())