pax_global_header00006660000000000000000000000064130327724240014516gustar00rootroot0000000000000052 comment=693c79d218a76bd7a5663046b15a7ff7ab5411e6 dfdatetime-20170103/000077500000000000000000000000001303277242400140635ustar00rootroot00000000000000dfdatetime-20170103/.gitignore000066400000000000000000000004241303277242400160530ustar00rootroot00000000000000# Files to ignore by git. # Back-up files *~ *.swp # Generic auto-generated build files *.pyc *.pyo # Specific auto-generated build files /.tox /__pycache__ /build /dfdatetime.egg-info /dist # Code review files /.review # Test coverage files .coverage tests-coverage.txt dfdatetime-20170103/.style.yapf000066400000000000000000000001361303277242400161620ustar00rootroot00000000000000[style] based_on_style = chromium COALESCE_BRACKETS = True SPLIT_BEFORE_FIRST_ARGUMENT = True dfdatetime-20170103/.travis.yml000066400000000000000000000015501303277242400161750ustar00rootroot00000000000000language: python matrix: include: - os: linux dist: trusty sudo: required python: 2.7 - os: linux dist: trusty sudo: required python: 3.4 - os: osx language: generic install: - ./config/travis/install.sh script: - if test `uname -s` = 'Linux' && test ${TRAVIS_PYTHON_VERSION} = "2.7"; then coverage run --source=dfdatetime --omit="*_test*,*__init__*,*test_lib*" ./run_tests.py; else ./run_tests.py; fi - python setup.py build - python setup.py sdist - python setup.py bdist - if test `uname -s` = 'Linux'; then mkdir -p ${PWD}/tmp/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/ && PYTHONPATH=${PWD}/tmp/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/ python setup.py install --prefix=${PWD}/tmp/; fi after_success: - if test `uname -s` = 'Linux' && test ${TRAVIS_PYTHON_VERSION} = "2.7"; then coveralls --verbose; fi dfdatetime-20170103/ACKNOWLEDGEMENTS000066400000000000000000000012251303277242400163400ustar00rootroot00000000000000Acknowledgements: dfdatetime Copyright 2016 The dfDateTime Project Authors. Please see the AUTHORS file for details on individual authors. dfDateTime, or Digital Forensics date and time, provides date and time objects to preserve accuracy and precision. dfDateTime, originates from the dfVFS project [https://github.com/log2timeline/dfvfs] and the dfWinReg project [//github.com/log2timeline/dfvfs]. It was largely rewritten and made into a stand-alone project to provide more flexibility and allow other projects to make use of the Digital Forensics date and time objects. dfDateTime is currently developed and maintained by: * Daniel White * Joachim Metz dfdatetime-20170103/AUTHORS000066400000000000000000000005601303277242400151340ustar00rootroot00000000000000# Names should be added to this file with this pattern: # # For individuals: # Name (email address) # # For organizations: # Organization (fnmatch pattern) # # See python fnmatch module documentation for more information. Google Inc. (*@google.com) Kristinn Gudjonsson (kiddi@kiddaland.net) Joachim Metz (joachim.metz@gmail.com) Daniel White (onager@deerpie.com) dfdatetime-20170103/LICENSE000066400000000000000000000261361303277242400151000ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. dfdatetime-20170103/MANIFEST.in000066400000000000000000000011111303277242400156130ustar00rootroot00000000000000include ACKNOWLEDGEMENTS AUTHORS LICENSE README include run_tests.py exclude .gitignore exclude *.pyc recursive-include config * recursive-exclude dfdatetime *.pyc # The example scripts are not required in a binary distribution package they # are considered source distribution files and excluded in find_package() # in setup.py. recursive-include examples *.py recursive-include test_data * # The test scripts are not required in a binary distribution package they # are considered source distribution files and excluded in find_package() # in setup.py. recursive-include tests *.py dfdatetime-20170103/README000066400000000000000000000004631303277242400147460ustar00rootroot00000000000000dfDateTime, or Digital Forensics date and time, provides date and time objects to preserve accuracy and precision. For more information see: * Project documentation: https://github.com/log2timeline/dfdatetime/wiki/Home * How to build from source: https://github.com/log2timeline/dfdatetime/wiki/Building dfdatetime-20170103/appveyor.yml000066400000000000000000000001661303277242400164560ustar00rootroot00000000000000environment: matrix: - PYTHON: "C:\\Python27" build: off test_script: - "%PYTHON%\\python.exe run_tests.py" dfdatetime-20170103/config/000077500000000000000000000000001303277242400153305ustar00rootroot00000000000000dfdatetime-20170103/config/dpkg/000077500000000000000000000000001303277242400162555ustar00rootroot00000000000000dfdatetime-20170103/config/dpkg/changelog000066400000000000000000000002301303277242400201220ustar00rootroot00000000000000dfdatetime (20170103-1) unstable; urgency=low * Auto-generated -- Log2Timeline Tue, 03 Jan 2017 20:01:40 +0100dfdatetime-20170103/config/dpkg/compat000066400000000000000000000000021303277242400174530ustar00rootroot000000000000007 dfdatetime-20170103/config/dpkg/control000066400000000000000000000016161303277242400176640ustar00rootroot00000000000000Source: dfdatetime Section: python Priority: extra Maintainer: log2timeline development team Build-Depends: debhelper (>= 7), python-all (>= 2.7~), python-setuptools, python3-all (>= 3.4~), python3-setuptools Standards-Version: 3.9.5 X-Python-Version: >= 2.7 X-Python3-Version: >= 3.4 Homepage: https://github.com/log2timeline/dfdatetime/ Package: python-dfdatetime Architecture: all Depends: ${python:Depends}, ${misc:Depends} Description: Digital Forensics date and time (dfDateTime). dfDateTime, or Digital Forensics date and time, provides date and time objects to preserve accuracy and precision. Package: python3-dfdatetime Architecture: all Depends: ${python3:Depends}, ${misc:Depends} Description: Digital Forensics date and time (dfDateTime). dfDateTime, or Digital Forensics date and time, provides date and time objects to preserve accuracy and precision. dfdatetime-20170103/config/dpkg/copyright000066400000000000000000000017041303277242400202120ustar00rootroot00000000000000Format: http://dep.debian.net/deps/dep5 Upstream-Name: dfdatetime Source: https://github.com/log2timeline/dfdatetime/ Files: * Copyright: 2016 The dfWinReg project authors. License: Apache-2.0 Files: debian/* Copyright: 2016 The dfWinReg project authors. License: Apache-2.0 License: Apache-2.0 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at . http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. . On Debian systems, the complete text of the Apache version 2.0 license can be found in "/usr/share/common-licenses/Apache-2.0". dfdatetime-20170103/config/dpkg/python-dfdatetime.docs000066400000000000000000000000501303277242400225470ustar00rootroot00000000000000ACKNOWLEDGEMENTS AUTHORS LICENSE README dfdatetime-20170103/config/dpkg/python3-dfdatetime.docs000066400000000000000000000000501303277242400226320ustar00rootroot00000000000000ACKNOWLEDGEMENTS AUTHORS LICENSE README dfdatetime-20170103/config/dpkg/rules000077500000000000000000000035221303277242400173370ustar00rootroot00000000000000#!/usr/bin/make -f # debian/rules that uses debhelper >= 7. # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 # This has to be exported to make some magic below work. export DH_OPTIONS %: dh $@ --buildsystem=python_distutils --with=python2,python3 .PHONY: override_dh_auto_clean override_dh_auto_clean: dh_auto_clean rm -rf build dfdatetime.egg-info/SOURCES.txt dfdatetime.egg-info/PKG-INFO .PHONY: override_dh_auto_build override_dh_auto_build: dh_auto_build set -ex; for python in $(shell py3versions -r); do \ $$python setup.py build; \ done; .PHONY: override_dh_auto_install override_dh_auto_install: dh_auto_install --destdir $(CURDIR)/debian/python-dfdatetime set -ex; for python in $(shell py3versions -r); do \ $$python setup.py install --root=$(CURDIR)/debian/python3-dfdatetime --install-layout=deb; \ done; # The shared doc files are provided via python-dfdatetime.docs and python3-dfdatetime.docs rm -rf $(CURDIR)/debian/python-dfdatetime/usr/share/doc/dfdatetime rm -rf $(CURDIR)/debian/python3-dfdatetime/usr/share/doc/dfdatetime .PHONY: override_dh_auto_test override_dh_auto_test: .PHONY: override_dh_installmenu override_dh_installmenu: .PHONY: override_dh_installmime override_dh_installmime: .PHONY: override_dh_installmodules override_dh_installmodules: .PHONY: override_dh_installlogcheck override_dh_installlogcheck: .PHONY: override_dh_installlogrotate override_dh_installlogrotate: .PHONY: override_dh_installpam override_dh_installpam: .PHONY: override_dh_installppp override_dh_installppp: .PHONY: override_dh_installudev override_dh_installudev: .PHONY: override_dh_installwm override_dh_installwm: .PHONY: override_dh_installxfonts override_dh_installxfonts: .PHONY: override_dh_gconf override_dh_gconf: .PHONY: override_dh_icons override_dh_icons: .PHONY: override_dh_perl override_dh_perl: dfdatetime-20170103/config/dpkg/source/000077500000000000000000000000001303277242400175555ustar00rootroot00000000000000dfdatetime-20170103/config/dpkg/source/format000066400000000000000000000000041303277242400207620ustar00rootroot000000000000001.0 dfdatetime-20170103/config/travis/000077500000000000000000000000001303277242400166405ustar00rootroot00000000000000dfdatetime-20170103/config/travis/install.sh000077500000000000000000000004641303277242400206510ustar00rootroot00000000000000#!/bin/bash # # Script to set up Travis-CI test VM. COVERALL_DEPENDENCIES="python-coverage python-coveralls python-docopt"; # Exit on error. set -e; if test `uname -s` = "Linux"; then sudo add-apt-repository ppa:gift/dev -y; sudo apt-get update -q; sudo apt-get install -y ${COVERALL_DEPENDENCIES}; fi dfdatetime-20170103/dfdatetime/000077500000000000000000000000001303277242400161715ustar00rootroot00000000000000dfdatetime-20170103/dfdatetime/__init__.py000066400000000000000000000003331303277242400203010ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Digital Forensics Date and Time (dfDateTime). dfDateTime, or Digital Forensics date and time, provides date and time objects to preserve accuracy and precision. """ __version__ = '20170103' dfdatetime-20170103/dfdatetime/cocoa_time.py000066400000000000000000000063411303277242400206510ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Cocoa timestamp implementation.""" from dfdatetime import definitions from dfdatetime import interface class CocoaTime(interface.DateTimeValues): """Class that implements a Cocoa timestamp. The Cocoa timestamp is a floating point value that contains the number of seconds since 2001-01-01 00:00:00 (also known as the Cocoa epoch). Negative values represent date and times predating the Cocoa epoch. Also see: https://developer.apple.com/library/ios/documentation/cocoa/Conceptual/ DatesAndTimes/Articles/dtDates.html Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. timestamp (float): Cocoa timestamp. """ # The difference between Jan 1, 2001 and Jan 1, 1970 in seconds. _COCOA_TO_POSIX_BASE = -978307200 def __init__(self, timestamp=None): """Initializes a Cocoa timestamp. Args: timestamp (Optional[float]): Cocoa timestamp. """ super(CocoaTime, self).__init__() self.precision = definitions.PRECISION_1_SECOND self.timestamp = timestamp def CopyFromString(self, time_string): """Copies a Cocoa timestamp from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the time string is invalid or not supported. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) microseconds = date_time_values.get(u'microseconds', None) timestamp = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) timestamp += self._COCOA_TO_POSIX_BASE timestamp = float(timestamp) if microseconds is not None: timestamp += float(microseconds) / 1000000 self.timestamp = timestamp self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the Cocoa timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if self.timestamp is None: return None, None timestamp = self.timestamp - self._COCOA_TO_POSIX_BASE remainder = int((timestamp % 1) * 10000000) return int(timestamp), remainder def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if self.timestamp is None: return timestamp = (self.timestamp - self._COCOA_TO_POSIX_BASE) * 1000000 return int(timestamp) dfdatetime-20170103/dfdatetime/definitions.py000066400000000000000000000013201303277242400210520ustar00rootroot00000000000000# -*- coding: utf-8 -*- """The date and time definitions. Also see: https://en.wikipedia.org/wiki/Day https://en.wikipedia.org/wiki/Hour https://en.wikipedia.org/wiki/Minute """ PRECISION_1_DAY = u'1d' PRECISION_1_HOUR = u'1h' PRECISION_1_NANOSECOND = u'1ns' PRECISION_100_NANOSECONDS = u'100ns' PRECISION_1_MICROSECOND = u'1us' PRECISION_1_MILLISECOND = u'1ms' PRECISION_1_MINUTE = u'1min' PRECISION_1_SECOND = u'1s' PRECISION_2_SECONDS = u'2s' PRECISION_VALUES = frozenset([ PRECISION_1_DAY, PRECISION_1_HOUR, PRECISION_1_NANOSECOND, PRECISION_100_NANOSECONDS, PRECISION_1_MICROSECOND, PRECISION_1_MILLISECOND, PRECISION_1_MINUTE, PRECISION_1_SECOND, PRECISION_2_SECONDS]) dfdatetime-20170103/dfdatetime/delphi_date_time.py000066400000000000000000000072331303277242400220300ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Delphi TDateTime implementation.""" from dfdatetime import definitions from dfdatetime import interface class DelphiDateTime(interface.DateTimeValues): """Class that implements a Delphi TDateTime timestamp. The Delphi TDateTime timestamp is a floating point value that contains the number of days since 1899-12-30 00:00:00 (also known as the epoch). Negative values represent date and times predating the epoch. The maximal correct date supported by TDateTime values is limited to: 9999-12-31 23:59:59.999 Also see: http://docwiki.embarcadero.com/Libraries/XE3/en/System.TDateTime Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. timestamp (float): Delphi TDateTime timestamp. """ # The difference between Dec 30, 1899 and Jan 1, 1970 in days. _DELPHI_TO_POSIX_BASE = 25569 # The number of seconds per day. _SECONDS_PER_DAY = 86400 # The number of microseconds per day. _MICROSECONDS_PER_DAY = 86400000000 def __init__(self, timestamp=None): """Initializes a Delphi TDateTime timestamp. Args: timestamp (Optional[float]): Delphi TDateTime timestamp. """ super(DelphiDateTime, self).__init__() self.precision = definitions.PRECISION_1_MILLISECOND self.timestamp = timestamp def CopyFromString(self, time_string): """Copies a Delphi TDateTime timestamp from a string. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the time string is invalid or not supported. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) microseconds = date_time_values.get(u'microseconds', None) if year > 9999: raise ValueError(u'Unsupported year value: {0:d}.'.format(year)) timestamp = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) timestamp = float(timestamp) / self._SECONDS_PER_DAY timestamp += self._DELPHI_TO_POSIX_BASE if microseconds is not None: timestamp += float(microseconds) / self._MICROSECONDS_PER_DAY self.timestamp = timestamp self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the Delphi TDateTime timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if self.timestamp is None: return None, None timestamp = ( (self.timestamp - self._DELPHI_TO_POSIX_BASE) * self._SECONDS_PER_DAY) remainder = int((timestamp % 1) * 10000000) return int(timestamp), remainder def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if self.timestamp is None: return timestamp = ( (self.timestamp - self._DELPHI_TO_POSIX_BASE) * self._MICROSECONDS_PER_DAY) return int(timestamp) dfdatetime-20170103/dfdatetime/fake_time.py000066400000000000000000000055211303277242400204720ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Fake timestamp implementation.""" import time from dfdatetime import definitions from dfdatetime import interface class FakeTime(interface.DateTimeValues): """Class that implements a fake timestamp. The fake timestamp is intended for testing purposes. On initialization it contains the current time in UTC in microsecond precision. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. """ def __init__(self): """Initializes a fake timestamp.""" super(FakeTime, self).__init__() # Note that time.time() and divmod return floating point values. timestamp, fraction_of_second = divmod(time.time(), 1) self._microseconds = int(fraction_of_second * 1000000) self._number_of_seconds = int(timestamp) self.precision = definitions.PRECISION_1_MICROSECOND def CopyFromString(self, time_string): """Copies a fake timestamp from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) self._number_of_seconds = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self._microseconds = date_time_values.get(u'microseconds', None) self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the fake timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if self._number_of_seconds is None: return None, None if self._microseconds is not None: return self._number_of_seconds, self._microseconds * 10 return self._number_of_seconds, None def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if self._number_of_seconds is None: return if self._microseconds is not None: return (self._number_of_seconds * 1000000) + self._microseconds return self._number_of_seconds * 1000000 dfdatetime-20170103/dfdatetime/fat_date_time.py000066400000000000000000000117041303277242400213330ustar00rootroot00000000000000# -*- coding: utf-8 -*- """FAT date time implementation.""" from dfdatetime import definitions from dfdatetime import interface class FATDateTime(interface.DateTimeValues): """Class that implements a FAT date time. The FAT date time is mainly used in DOS/Windows file formats and FAT. The FAT date and time is a 32-bit value containing two 16-bit values: * The date (lower 16-bit). * bits 0 - 4: day of month, where 1 represents the first day * bits 5 - 8: month of year, where 1 represent January * bits 9 - 15: year since 1980 * The time of day (upper 16-bit). * bits 0 - 4: seconds (in 2 second intervals) * bits 5 - 10: minutes * bits 11 - 15: hours The FAT date time has no time zone information and is typically stored in the local time of the computer. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. """ # The difference between Jan 1, 1980 and Jan 1, 1970 in seconds. _FAT_DATE_TO_POSIX_BASE = 315532800 def __init__(self, fat_date_time=None): """Initializes a FAT date time. Args: fat_date_time (Optional[int]): FAT date time. """ number_of_seconds = None if fat_date_time is not None: number_of_seconds = self._GetNumberOfSeconds(fat_date_time) super(FATDateTime, self).__init__() self._number_of_seconds = number_of_seconds self.precision = definitions.PRECISION_2_SECONDS def _GetNumberOfSeconds(self, fat_date_time): """Retrieves the number of seconds from a FAT date time. Args: fat_date_time (int): FAT date time. Returns: int: number of seconds since January 1, 1980 00:00:00. Raises: ValueError: if the month, day of month, hours, minutes or seconds value is out of bounds. """ day_of_month = (fat_date_time & 0x1f) month = ((fat_date_time >> 5) & 0x0f) year = (fat_date_time >> 9) & 0x7f days_per_month = self._GetDaysPerMonth(year, month) if day_of_month < 1 or day_of_month > days_per_month: raise ValueError(u'Day of month value out of bounds.') number_of_days = self._GetDayOfYear(1980 + year, month, day_of_month) number_of_days -= 1 for past_year in range(0, year): number_of_days += self._GetNumberOfDaysInYear(past_year) fat_date_time >>= 16 seconds = (fat_date_time & 0x1f) * 2 minutes = (fat_date_time >> 5) & 0x3f hours = (fat_date_time >> 11) & 0x1f if hours not in range(0, 24): raise ValueError(u'Hours value out of bounds.') if minutes not in range(0, 60): raise ValueError(u'Minutes value out of bounds.') if seconds not in range(0, 60): raise ValueError(u'Seconds value out of bounds.') number_of_seconds = (((hours * 60) + minutes) * 60) + seconds number_of_seconds += number_of_days * self._SECONDS_PER_DAY return number_of_seconds def CopyFromString(self, time_string): """Copies a FAT date time from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the time string is invalid or not supported. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) if year < 1980 or year > (1980 + 0x7f): raise ValueError(u'Year value not supported: {0!s}.'.format(year)) self._number_of_seconds = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self._number_of_seconds -= self._FAT_DATE_TO_POSIX_BASE self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the FAT date time to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if self._number_of_seconds is None or self._number_of_seconds < 0: return None, None timestamp = self._number_of_seconds + self._FAT_DATE_TO_POSIX_BASE return timestamp, None def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if self._number_of_seconds is None or self._number_of_seconds < 0: return return (self._number_of_seconds + self._FAT_DATE_TO_POSIX_BASE) * 1000000 dfdatetime-20170103/dfdatetime/filetime.py000066400000000000000000000065071303277242400203510ustar00rootroot00000000000000# -*- coding: utf-8 -*- """FILETIME timestamp implementation.""" from dfdatetime import definitions from dfdatetime import interface class Filetime(interface.DateTimeValues): """Class that implements a FILETIME timestamp. The FILETIME timestamp is a 64-bit integer that contains the number of 100th nano seconds since 1601-01-01 00:00:00. Do not confuse this with the FILETIME structure that consists of 2 x 32-bit integers and is presumed to be unsigned. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. timestamp (int): FILETIME timestamp. """ # The difference between Jan 1, 1601 and Jan 1, 1970 in seconds. _FILETIME_TO_POSIX_BASE = 11644473600 _UINT64_MAX = (1 << 64) - 1 def __init__(self, timestamp=None): """Initializes a FILETIME timestamp. Args: timestamp (Optional[int]): FILETIME timestamp. """ super(Filetime, self).__init__() self.precision = definitions.PRECISION_100_NANOSECONDS self.timestamp = timestamp def CopyFromString(self, time_string): """Copies a FILETIME from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the time string is invalid or not supported. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) if year < 1601: raise ValueError(u'Year value not supported: {0!s}.'.format(year)) self.timestamp = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self.timestamp += self._FILETIME_TO_POSIX_BASE self.timestamp *= 1000000 self.timestamp += date_time_values.get(u'microseconds', 0) self.timestamp *= 10 self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the FILETIME timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if (self.timestamp is None or self.timestamp < 0 or self.timestamp > self._UINT64_MAX): return None, None timestamp, remainder = divmod(self.timestamp, 10000000) timestamp -= self._FILETIME_TO_POSIX_BASE return timestamp, remainder def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if (self.timestamp is None or self.timestamp < 0 or self.timestamp > self._UINT64_MAX): return timestamp, _ = divmod(self.timestamp, 10) return timestamp - (self._FILETIME_TO_POSIX_BASE * 1000000) dfdatetime-20170103/dfdatetime/hfs_time.py000066400000000000000000000060401303277242400203410ustar00rootroot00000000000000# -*- coding: utf-8 -*- """HFS timestamp implementation.""" from dfdatetime import definitions from dfdatetime import interface class HFSTime(interface.DateTimeValues): """Class that implements a HFS timestamp. The HFS timestamp is an unsigned 32-bit integer that contains the number of seconds since 1904-01-01 00:00:00. Where in HFS the timestamp is typically in local time and in HFS+/HFSX in UTC. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. timestamp (int): HFS timestamp. """ # The difference between Jan 1, 1904 and Jan 1, 1970 in seconds. _HFS_TO_POSIX_BASE = 2082844800 _UINT32_MAX = (1 << 32) - 1 def __init__(self, timestamp=None): """Initializes a HFS timestamp. Args: timestamp (Optional[int]): HFS timestamp. """ super(HFSTime, self).__init__() self.precision = definitions.PRECISION_1_SECOND self.timestamp = timestamp def CopyFromString(self, time_string): """Copies a HFS timestamp from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the time string is invalid or not supported. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) if year < 1904 or year > 2040: raise ValueError(u'Year value not supported.') self.timestamp = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self.timestamp += self._HFS_TO_POSIX_BASE self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the HFS timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if (self.timestamp is None or self.timestamp < 0 or self.timestamp > self._UINT32_MAX): return None, None timestamp = self.timestamp - self._HFS_TO_POSIX_BASE return timestamp, 0 def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if (self.timestamp is None or self.timestamp < 0 or self.timestamp > self._UINT32_MAX): return timestamp = self.timestamp - self._HFS_TO_POSIX_BASE return timestamp * 1000000 dfdatetime-20170103/dfdatetime/interface.py000066400000000000000000000310301303277242400205000ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Date and time values interface.""" import abc import calendar class DateTimeValues(object): """Defines the date time values interface. This is the super class of different date and time representations. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. """ _DAYS_PER_MONTH = (31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) # The number of seconds in a day _SECONDS_PER_DAY = 24 * 60 * 60 def __init__(self): """Initializes date time values.""" super(DateTimeValues, self).__init__() self.is_local_time = False self.precision = None def _CopyDateFromString(self, date_string): """Copies a date from a string. Args: date_string (str): date value formatted as: YYYY-MM-DD Returns: tuple[int, int, int]: year, month, day of month. Raises: ValueError: if the date string is invalid or not supported. """ date_string_length = len(date_string) # The date string should at least contain 'YYYY-MM-DD'. if date_string_length < 10: raise ValueError(u'Date string too short.') if date_string[4] != u'-' or date_string[7] != u'-': raise ValueError(u'Invalid date string.') try: year = int(date_string[0:4], 10) except ValueError: raise ValueError(u'Unable to parse year.') try: month = int(date_string[5:7], 10) except ValueError: raise ValueError(u'Unable to parse month.') try: day_of_month = int(date_string[8:10], 10) except ValueError: raise ValueError(u'Unable to parse day of month.') days_per_month = self._GetDaysPerMonth(year, month) if day_of_month < 1 or day_of_month > days_per_month: raise ValueError(u'Day of month value out of bounds.') return year, month, day_of_month def _CopyDateTimeFromString(self, time_string): """Copies a date and time from a string. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Returns: dict[str, int]: date and time values, such as year, month, day of month, hours, minutes, seconds, microseconds. Raises: ValueError: if the time string is invalid or not supported. """ if not time_string: raise ValueError(u'Invalid time string.') time_string_length = len(time_string) year, month, day_of_month = self._CopyDateFromString(time_string) if time_string_length <= 10: return { u'year': year, u'month': month, u'day_of_month': day_of_month} # If a time of day is specified the time string it should at least # contain 'YYYY-MM-DD hh:mm:ss'. if time_string[10] != u' ': raise ValueError( u'Invalid time string - space missing as date and time separator.') hours, minutes, seconds, microseconds, time_zone_offset = ( self._CopyTimeFromString(time_string[11:])) if time_zone_offset: time_zone_hours, time_zone_minutes = divmod(time_zone_offset, 60) minutes += time_zone_minutes # Since divmod makes sure the sign of time_zone_minutes is positive # we only need to check the upper bound here, because time_zone_hours # remains signed it is corrected accordingly. if minutes >= 60: minutes -= 60 hours += 1 hours += time_zone_hours if hours < 0: hours += 24 day_of_month -= 1 elif hours >= 24: hours -= 24 day_of_month += 1 days_per_month = self._GetDaysPerMonth(year, month) if day_of_month < 1: month -= 1 if month < 1: month = 12 year -= 1 day_of_month += self._GetDaysPerMonth(year, month) elif day_of_month > days_per_month: month += 1 if month > 12: month = 1 year += 1 day_of_month -= days_per_month date_time_values = { u'year': year, u'month': month, u'day_of_month': day_of_month, u'hours': hours, u'minutes': minutes, u'seconds': seconds} if microseconds is not None: date_time_values[u'microseconds'] = microseconds return date_time_values def _CopyTimeFromString(self, time_string): """Copies a time from a string. Args: time_string (str): time value formatted as: hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The seconds fraction and time zone offset are optional. Returns: tuple[int, int, int, int, int]: hours, minutes, seconds, microseconds, time zone offset in minutes. Raises: ValueError: if the time string is invalid or not supported. """ time_string_length = len(time_string) if time_string_length < 8: raise ValueError(u'Time string too short.') # The time string should at least contain 'hh:mm:ss'. if time_string[2] != u':' or time_string[5] != u':': raise ValueError(u'Invalid time string.') try: hours = int(time_string[0:2], 10) except ValueError: raise ValueError(u'Unable to parse hours.') if hours not in range(0, 24): raise ValueError(u'Hours value out of bounds.') try: minutes = int(time_string[3:5], 10) except ValueError: raise ValueError(u'Unable to parse minutes.') if minutes not in range(0, 60): raise ValueError(u'Minutes value out of bounds.') try: seconds = int(time_string[6:8], 10) except ValueError: raise ValueError(u'Unable to parse day of seconds.') # TODO: support a leap second? if seconds not in range(0, 60): raise ValueError(u'Seconds value out of bounds.') microseconds = None time_zone_offset = None if time_string_length > 8: if time_string[8] != u'.': time_zone_string_index = 8 else: for time_zone_string_index in range(8, time_string_length): if time_string[time_zone_string_index] in (u'+', u'-'): break # The calculations that follow rely on the time zone string index # to point beyond the string in case no time zone offset was defined. if time_zone_string_index == time_string_length - 1: time_zone_string_index += 1 if time_zone_string_index > 8: fraction_of_seconds_length = time_zone_string_index - 9 if fraction_of_seconds_length not in (3, 6): raise ValueError(u'Invalid time string.') try: microseconds = int(time_string[9:time_zone_string_index], 10) except ValueError: raise ValueError(u'Unable to parse fraction of seconds.') if fraction_of_seconds_length == 3: microseconds *= 1000 if time_zone_string_index < time_string_length: if (time_string_length - time_zone_string_index != 6 or time_string[time_zone_string_index + 3] != u':'): raise ValueError(u'Invalid time string.') try: time_zone_offset = int(time_string[ time_zone_string_index + 1:time_zone_string_index + 3]) except ValueError: raise ValueError(u'Unable to parse time zone hours offset.') if time_zone_offset not in range(0, 24): raise ValueError(u'Timezone hours offset value out of bounds.') time_zone_offset *= 60 try: time_zone_offset += int(time_string[ time_zone_string_index + 4:time_zone_string_index + 6]) except ValueError: raise ValueError(u'Unable to parse time zone minutes offset.') # Note that when the sign of the time zone offset is negative # the difference needs to be added. We do so by flipping the sign. if time_string[time_zone_string_index] != u'-': time_zone_offset *= -1 return hours, minutes, seconds, microseconds, time_zone_offset def _GetDayOfYear(self, year, month, day_of_month): """Retrieves the day of the year for a specific day of a month in a year. Args: year (int): year e.g. 1970. month (int): month where 1 represents January. day_of_month (int): day of the month where 1 represents the first day. Returns: int: day of year. Raises: ValueError: if the month or day of month value is out of bounds. """ if month not in range(1, 13): raise ValueError(u'Month value out of bounds.') days_per_month = self._GetDaysPerMonth(year, month) if day_of_month < 1 or day_of_month > days_per_month: raise ValueError(u'Day of month value out of bounds.') day_of_year = day_of_month for past_month in range(1, month): day_of_year += self._GetDaysPerMonth(year, past_month) return day_of_year def _GetDaysPerMonth(self, year, month): """Retrieves the number of days in a month of a specific year. Args: year (int): year e.g. 1970. month (int): month ranging from 1 to 12. Returns: int: number of days in the month. Raises: ValueError: if the month value is out of bounds. """ if month not in range(1, 13): raise ValueError(u'Month value out of bounds.') days_per_month = self._DAYS_PER_MONTH[month - 1] if month == 2 and self._IsLeapYear(year): days_per_month += 1 return days_per_month def _GetNumberOfDaysInYear(self, year): """Retrieves the number of days in a specific year. Args: year (int): year e.g. 1970. Returns: int: number of days in the year. """ if self._IsLeapYear(year): return 366 return 365 def _GetNumberOfSecondsFromElements( self, year, month, day, hours, minutes, seconds): """Retrieves the number of seconds from the date and time elements. Args: year (int): year e.g. 1970. month (int): month of year. day(int): day of month. hours (int): hours. minutes (int): minutes. seconds (int): seconds. Returns: int: number of seconds since January 1, 1970 00:00:00 or None. Raises: ValueError: if the time elements are invalid. """ if not year or not month or not day: return # calendar.timegm does not sanity check the time elements. if hours is None: hours = 0 elif hours not in range(0, 24): raise ValueError(u'Hours value: {0!s} out of bounds.'.format(hours)) if minutes is None: minutes = 0 elif minutes not in range(0, 60): raise ValueError(u'Minutes value: {0!s} out of bounds.'.format(minutes)) # TODO: support a leap second? if seconds is None: seconds = 0 elif seconds not in range(0, 60): raise ValueError(u'Seconds value: {0!s} out of bounds.'.format(seconds)) # calendar.timegm requires the time tuple to contain at least # 6 integer values. time_elements_tuple = (year, month, day, hours, minutes, seconds) number_of_seconds = calendar.timegm(time_elements_tuple) return int(number_of_seconds) def _IsLeapYear(self, year): """Determines if a year is a leap year. Args: year (int): year e.g. 1970. Returns: bool: True if the year is a leap year. """ return (year % 4 == 0 and year % 100 != 0) or year % 400 == 0 @abc.abstractmethod def CopyFromString(self, time_string): """Copies a date time value from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the time string is invalid or not supported. """ def CopyToStatTimeTuple(self): """Copies the date time value to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ # TODO: remove this method when there is no more need for it in plaso. @abc.abstractmethod def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ dfdatetime-20170103/dfdatetime/java_time.py000066400000000000000000000061311303277242400205030ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Java java.util.Date timestamp implementation.""" from dfdatetime import definitions from dfdatetime import interface class JavaTime(interface.DateTimeValues): """Class that implements a Java java.util.Date timestamp. The Java java.util.Date timestamp is a signed integer that contains the number of milliseconds since 1970-01-01 00:00:00 (also known as the POSIX epoch). Negative values represent date and times predating the POSIX epoch. Also see: https://docs.oracle.com/javase/8/docs/api/java/util/Date.html Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. timestamp (int): Java timestamp. """ _INT64_MIN = -(1 << 63) _INT64_MAX = (1 << 63) - 1 def __init__(self, timestamp=None): """Initializes a Java timestamp. Args: timestamp (Optional[int]): Java timestamp. """ super(JavaTime, self).__init__() self.precision = definitions.PRECISION_1_MILLISECOND self.timestamp = timestamp def CopyFromString(self, time_string): """Copies a Java timestamp from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) microseconds = date_time_values.get(u'microseconds', None) self.timestamp = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self.timestamp *= 1000 if microseconds: milliseconds, _ = divmod(microseconds, 1000) self.timestamp += milliseconds self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the Java timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if (self.timestamp is None or self.timestamp < self._INT64_MIN or self.timestamp > self._INT64_MAX): return None, None timestamp, milliseconds = divmod(self.timestamp, 1000) return timestamp, milliseconds * 10000 def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if (self.timestamp is None or self.timestamp < self._INT64_MIN or self.timestamp > self._INT64_MAX): return return self.timestamp * 1000 dfdatetime-20170103/dfdatetime/posix_time.py000066400000000000000000000121011303277242400207160ustar00rootroot00000000000000# -*- coding: utf-8 -*- """POSIX timestamp implementation.""" from dfdatetime import definitions from dfdatetime import interface class PosixTime(interface.DateTimeValues): """Class that implements a POSIX timestamp. The POSIX timestamp is a signed integer that contains the number of seconds since 1970-01-01 00:00:00 (also known as the POSIX epoch). Negative values represent date and times predating the POSIX epoch. The POSIX timestamp was initially 32-bit though 64-bit variants are known to be used. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. timestamp (int): POSIX timestamp. """ def __init__(self, timestamp=None): """Initializes a POSIX timestamp. Args: timestamp (Optional[int]): POSIX timestamp. """ super(PosixTime, self).__init__() self.precision = definitions.PRECISION_1_SECOND self.timestamp = timestamp def CopyFromString(self, time_string): """Copies a POSIX timestamp from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) self.timestamp = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the POSIX timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if self.timestamp is None: return None, None return self.timestamp, None def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if self.timestamp is None: return return self.timestamp * 1000000 class PosixTimeInMicroseconds(interface.DateTimeValues): """Class that implements a POSIX timestamp in microseconds. Variant of the POSIX timestamp in microseconds. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. timestamp (int): POSIX timestamp in microseconds. """ def __init__(self, timestamp=None): """Initializes a POSIX timestamp in microseconds. Args: timestamp (Optional[int]): POSIX timestamp in microseconds. """ super(PosixTimeInMicroseconds, self).__init__() self.precision = definitions.PRECISION_1_MICROSECOND self.timestamp = timestamp def CopyFromString(self, time_string): """Copies a POSIX timestamp from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) microseconds = date_time_values.get(u'microseconds', 0) self.timestamp = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self.timestamp *= 1000000 self.timestamp += microseconds self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the POSIX timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if self.timestamp is None: return None, None timestamp, microseconds = divmod(self.timestamp, 1000000) return timestamp, microseconds * 10 def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if self.timestamp is None: return return self.timestamp dfdatetime-20170103/dfdatetime/semantic_time.py000066400000000000000000000032401303277242400213630ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Semantic time implementation.""" from dfdatetime import interface class SemanticTime(interface.DateTimeValues): """Class that implements semantic time. Semantic time is term to describe date and time values that have specific meaning such as: "Never", "Yesterday", "Not set". Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. string (str): semantic representation of the time, such as: "Never", "Not set". """ def __init__(self, string=None): """Initializes a semantic time. Args: string (str): semantic representation of the time, such as: "Never", "Not set". """ super(SemanticTime, self).__init__() self.string = string def CopyFromString(self, time_string): """Copies semantic time from a string containing a date and time value. Args: time_string (str): semantic representation of the time, such as: "Never", "Not set". Raises: ValueError: because semantic time cannot be copied from a string. """ self.string = time_string def CopyToStatTimeTuple(self): """Copies the semantic timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds, which will always be 0, 0. """ return 0, 0 def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds, which will always be 0. """ return 0 dfdatetime-20170103/dfdatetime/systemtime.py000066400000000000000000000126741303277242400207600ustar00rootroot00000000000000# -*- coding: utf-8 -*- """SYSTEMTIME structure implementation.""" from dfdatetime import definitions from dfdatetime import interface class Systemtime(interface.DateTimeValues): """Class that implements a SYSTEMTIME structure. The SYSTEMTIME structure is 16 bytes of size and contains: struct { WORD year, WORD month, WORD day_of_week, WORD day_of_month, WORD hour, WORD minute, WORD second, WORD millisecond } Attributes: year (int): year, 1601 through 30827. month (int): month of year, 1 through 12. day_of_week (int): day of week, 0 through 6. day_of_month (int): day of month, 1 through 31. hours (int): hours, 0 through 23. minutes (int): minutes, 0 through 59. seconds (int): seconds, 0 through 59. milliseconds (int): milliseconds, 0 through 999. """ def __init__(self, system_time_tuple=None): """Initializes a SYSTEMTIME structure. Args: system_time_tuple (Optional[tuple[int, int, int, int, int, int, int, int]]): system time, contains year, month, day of week, day of month, hours, minutes, seconds and milliseconds. Raises: ValueError: if the system time is invalid. """ super(Systemtime, self).__init__() self._number_of_seconds = None self.day_of_month = None self.day_of_week = None self.hours = None self.milliseconds = None self.minutes = None self.month = None self.precision = definitions.PRECISION_1_MILLISECOND self.seconds = None self.year = None if system_time_tuple: if len(system_time_tuple) < 8: raise ValueError(u'Invalid system time tuple 8 elements required.') if system_time_tuple[0] < 1601 or system_time_tuple[0] > 30827: raise ValueError(u'Year value out of bounds.') if system_time_tuple[1] not in range(1, 13): raise ValueError(u'Month value out of bounds.') if system_time_tuple[2] not in range(0, 7): raise ValueError(u'Day of week value out of bounds.') days_per_month = self._GetDaysPerMonth( system_time_tuple[0], system_time_tuple[1]) if system_time_tuple[3] < 1 or system_time_tuple[3] > days_per_month: raise ValueError(u'Day of month value out of bounds.') if system_time_tuple[4] not in range(0, 24): raise ValueError(u'Hours value out of bounds.') if system_time_tuple[5] not in range(0, 60): raise ValueError(u'Minutes value out of bounds.') # TODO: support a leap second? if system_time_tuple[6] not in range(0, 60): raise ValueError(u'Seconds value out of bounds.') if system_time_tuple[7] < 0 or system_time_tuple[7] > 999: raise ValueError(u'Milliseconds value out of bounds.') self.day_of_month = system_time_tuple[3] self.day_of_week = system_time_tuple[2] self.hours = system_time_tuple[4] self.milliseconds = system_time_tuple[7] self.minutes = system_time_tuple[5] self.month = system_time_tuple[1] self.seconds = system_time_tuple[6] self.year = system_time_tuple[0] self._number_of_seconds = self._GetNumberOfSecondsFromElements( self.year, self.month, self.day_of_month, self.hours, self.minutes, self.seconds) def CopyFromString(self, time_string): """Copies a SYSTEMTIME from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the date string is invalid or not supported. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) microseconds = date_time_values.get(u'microseconds', 0) milliseconds, _ = divmod(microseconds, 1000) if year < 1601 or year > 30827: raise ValueError(u'Unsupported year value: {0:d}.'.format(year)) self._number_of_seconds = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self.year = year self.month = month self.day_of_month = day_of_month # TODO: calculate day of week on demand. self.day_of_week = None self.hours = hours self.minutes = minutes self.seconds = seconds self.milliseconds = milliseconds self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the SYSTEMTIME structure to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if self._number_of_seconds is None: return None, None return self._number_of_seconds, self.milliseconds * 10000 def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if self._number_of_seconds is None: return return ((self._number_of_seconds * 1000) + self.milliseconds) * 1000 dfdatetime-20170103/dfdatetime/time_elements.py000066400000000000000000000243211303277242400213770ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Time elements implementation.""" from dfdatetime import definitions from dfdatetime import interface class TimeElements(interface.DateTimeValues): """Class that implements time elements. Time elements contain separate values for year, month, day of month, hours, minutes and seconds. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. """ def __init__(self, time_elements_tuple=None): """Initializes time elements. Args: time_elements_tuple (Optional[tuple[int, int, int, int, int, int]]): time elements, contains year, month, day of month, hours, minutes and seconds. Raises: ValueError: if the time elements tuple is invalid. """ super(TimeElements, self).__init__() self._number_of_seconds = None self._time_elements_tuple = time_elements_tuple self.precision = definitions.PRECISION_1_SECOND if time_elements_tuple: if len(time_elements_tuple) < 6: raise ValueError(u'Invalid time elements tuple 6 elements required.') self._number_of_seconds = self._GetNumberOfSecondsFromElements( *time_elements_tuple) def CopyFromString(self, time_string): """Copies time elements from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) self._number_of_seconds = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self._time_elements_tuple = ( year, month, day_of_month, hours, minutes, seconds) self.is_local_time = False def CopyFromStringISO8601(self, time_string): """Copies time elements from a ISO 8601 date and time string. Currently not supported: * Duration notation: "P..." * Week notation "2016-W33" * Date with week number notation "2016-W33-3" * Date without year notation "--08-17" * Ordinal date notation "2016-230" * Seconds fraction of a size other than 3 or 6 Args: time_string (str): date and time value formatted as: YYYY-MM-DDThh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the time string is invalid or not supported. """ if not time_string: raise ValueError(u'Invalid time string.') time_string_length = len(time_string) if time_string_length >= 11: if time_string[10] != u'T': raise ValueError(u'Invalid time string.') # Replace "T" by " ". time_string = u'{0:s} {1:s}'.format(time_string[:10], time_string[11:]) if time_string_length >= 20 and time_string[19] == u',': # Replace "," by ".". time_string = u'{0:s}.{1:s}'.format(time_string[:19], time_string[20:]) if time_string.endswith(u'Z'): time_string = time_string[:-1] self.CopyFromString(time_string) def CopyFromStringTuple(self, time_elements_tuple): """Copies time elements from string-based time elements tuple. Args: time_elements_tuple (Optional[tuple[str, str, str, str, str, str]]): time elements, contains year, month, day of month, hours, minutes and seconds. Raises: ValueError: if the time elements tuple is invalid. """ if len(time_elements_tuple) < 6: raise ValueError(u'Invalid time elements tuple 6 elements required.') try: year = int(time_elements_tuple[0], 10) except (TypeError, ValueError): raise ValueError(u'Invalid year value: {0!s}'.format( time_elements_tuple[0])) try: month = int(time_elements_tuple[1], 10) except (TypeError, ValueError): raise ValueError(u'Invalid month value: {0!s}'.format( time_elements_tuple[1])) try: day_of_month = int(time_elements_tuple[2], 10) except (TypeError, ValueError): raise ValueError(u'Invalid day of month value: {0!s}'.format( time_elements_tuple[2])) try: hours = int(time_elements_tuple[3], 10) except (TypeError, ValueError): raise ValueError(u'Invalid hours value: {0!s}'.format( time_elements_tuple[3])) try: minutes = int(time_elements_tuple[4], 10) except (TypeError, ValueError): raise ValueError(u'Invalid minutes value: {0!s}'.format( time_elements_tuple[4])) try: seconds = int(time_elements_tuple[5], 10) except (TypeError, ValueError): raise ValueError(u'Invalid seconds value: {0!s}'.format( time_elements_tuple[5])) self._number_of_seconds = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self._time_elements_tuple = ( year, month, day_of_month, hours, minutes, seconds) def CopyToStatTimeTuple(self): """Copies the time elements to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if self._number_of_seconds is None: return None, None return self._number_of_seconds, None def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if self._number_of_seconds is None: return return self._number_of_seconds * 1000000 class TimeElementsInMilliseconds(TimeElements): """Class that implements time elements in milliseconds. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. """ def __init__(self, time_elements_tuple=None): """Initializes time elements. Args: time_elements_tuple (Optional[tuple[int, int, int, int, int, int, int]]): time elements, contains year, month, day of month, hours, minutes, seconds and milliseconds. Raises: ValueError: if the time elements tuple is invalid. """ milliseconds = None if time_elements_tuple: if len(time_elements_tuple) < 7: raise ValueError(u'Invalid time elements tuple 7 elements required.') milliseconds = time_elements_tuple[6] time_elements_tuple = time_elements_tuple[:6] if milliseconds < 0 or milliseconds > 999: raise ValueError(u'Invalid number of milliseconds.') super(TimeElementsInMilliseconds, self).__init__( time_elements_tuple=time_elements_tuple) self._milliseconds = milliseconds self.precision = definitions.PRECISION_1_MILLISECOND def CopyFromString(self, time_string): """Copies time elements from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) microseconds = date_time_values.get(u'microseconds', 0) milliseconds, _ = divmod(microseconds, 1000) self._number_of_seconds = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self._milliseconds = milliseconds self._time_elements_tuple = ( year, month, day_of_month, hours, minutes, seconds, milliseconds) self.is_local_time = False def CopyFromStringTuple(self, time_elements_tuple): """Copies time elements from string-based time elements tuple. Args: time_elements_tuple (Optional[tuple[str, str, str, str, str, str, str]]): time elements, contains year, month, day of month, hours, minutes, seconds and milliseconds. Raises: ValueError: if the time elements tuple is invalid. """ if len(time_elements_tuple) < 7: raise ValueError(u'Invalid time elements tuple 7 elements required.') super(TimeElementsInMilliseconds, self).CopyFromStringTuple( time_elements_tuple) try: self._milliseconds = int(time_elements_tuple[6], 10) except (TypeError, ValueError): raise ValueError(u'Invalid milliseconds value: {0!s}'.format( time_elements_tuple[6])) def CopyToStatTimeTuple(self): """Copies the time elements to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if self._number_of_seconds is None or self._milliseconds is None: return None, None return self._number_of_seconds, self._milliseconds * 1000 def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if self._number_of_seconds is None or self._milliseconds is None: return return ((self._number_of_seconds * 1000) + self._milliseconds) * 1000 dfdatetime-20170103/dfdatetime/uuid_time.py000066400000000000000000000067661303277242400205460ustar00rootroot00000000000000# -*- coding: utf-8 -*- """UUID version 1 timestamp implementation.""" from dfdatetime import definitions from dfdatetime import interface class UUIDTime(interface.DateTimeValues): """Class that implements an UUID version 1 timestamp. The UUID version 1 timestamp is an unsigned 60-bit value that contains the number of 100th nano seconds intervals since 1582-10-15 00:00:00. Also see: https://en.wikipedia.org/wiki/Universally_unique_identifier Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. timestamp (int): UUID timestamp. """ # The difference between Oct 15, 1582 and Jan 1, 1970 in seconds. _UUID_TO_POSIX_BASE = 12219292800 _UINT60_MAX = (1 << 60) - 1 def __init__(self, timestamp=None): """Initializes an UUID version 1 timestamp. Args: timestamp (Optional[int]): UUID version 1 timestamp. Raises: ValueError: if the UUID version 1 timestamp is invalid. """ if timestamp and (timestamp < 0 or timestamp > self._UINT60_MAX): raise ValueError(u'Invalid UUID version 1 timestamp.') super(UUIDTime, self).__init__() self.precision = definitions.PRECISION_100_NANOSECONDS self.timestamp = timestamp def CopyFromString(self, time_string): """Copies an UUID timestamp from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the time string is invalid or not supported. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) if year < 1582: raise ValueError(u'Year value not supported.') self.timestamp = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self.timestamp += self._UUID_TO_POSIX_BASE self.timestamp *= 1000000 self.timestamp += date_time_values.get(u'microseconds', 0) self.timestamp *= 10 self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the UUID timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if (self.timestamp is None or self.timestamp < 0 or self.timestamp > self._UINT60_MAX): return None, None timestamp, remainder = divmod(self.timestamp, 10000000) timestamp -= self._UUID_TO_POSIX_BASE return timestamp, remainder def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if (self.timestamp is None or self.timestamp < 0 or self.timestamp > self._UINT60_MAX): return timestamp, _ = divmod(self.timestamp, 10) return timestamp - (self._UUID_TO_POSIX_BASE * 1000000) dfdatetime-20170103/dfdatetime/webkit_time.py000066400000000000000000000060711303277242400210520ustar00rootroot00000000000000# -*- coding: utf-8 -*- """WebKit timestamp implementation.""" from dfdatetime import definitions from dfdatetime import interface class WebKitTime(interface.DateTimeValues): """Class that implements a WebKit timestamp. The WebKit timestamp is a signed 64-bit integer that contains the number of micro seconds since 1601-01-01 00:00:00. Attributes: is_local_time (bool): True if the date and time value is in local time. precision (str): precision of the date and time value, which should be one of the PRECISION_VALUES in definitions. """ # The difference between Jan 1, 1601 and Jan 1, 1970 in seconds. _WEBKIT_TO_POSIX_BASE = 11644473600 _INT64_MIN = -(1 << 63) _INT64_MAX = (1 << 63) - 1 def __init__(self, timestamp=None): """Initializes a WebKit timestamp. Args: timestamp (Optional[int]): WebKit timestamp. """ super(WebKitTime, self).__init__() self.precision = definitions.PRECISION_1_MICROSECOND self.timestamp = timestamp def CopyFromString(self, time_string): """Copies a WebKit timestamp from a string containing a date and time value. Args: time_string (str): date and time value formatted as: YYYY-MM-DD hh:mm:ss.######[+-]##:## Where # are numeric digits ranging from 0 to 9 and the seconds fraction can be either 3 or 6 digits. The time of day, seconds fraction and time zone offset are optional. The default time zone is UTC. Raises: ValueError: if the time string is invalid or not supported. """ date_time_values = self._CopyDateTimeFromString(time_string) year = date_time_values.get(u'year', 0) month = date_time_values.get(u'month', 0) day_of_month = date_time_values.get(u'day_of_month', 0) hours = date_time_values.get(u'hours', 0) minutes = date_time_values.get(u'minutes', 0) seconds = date_time_values.get(u'seconds', 0) self.timestamp = self._GetNumberOfSecondsFromElements( year, month, day_of_month, hours, minutes, seconds) self.timestamp += self._WEBKIT_TO_POSIX_BASE self.timestamp *= 1000000 self.timestamp += date_time_values.get(u'microseconds', 0) self.is_local_time = False def CopyToStatTimeTuple(self): """Copies the WebKit timestamp to a stat timestamp tuple. Returns: tuple[int, int]: a POSIX timestamp in seconds and the remainder in 100 nano seconds or (None, None) on error. """ if (self.timestamp is None or self.timestamp < self._INT64_MIN or self.timestamp > self._INT64_MAX): return None, None timestamp, microseconds = divmod(self.timestamp, 1000000) timestamp -= self._WEBKIT_TO_POSIX_BASE return timestamp, microseconds * 10 def GetPlasoTimestamp(self): """Retrieves a timestamp that is compatible with plaso. Returns: int: a POSIX timestamp in microseconds or None on error. """ if (self.timestamp is None or self.timestamp < self._INT64_MIN or self.timestamp > self._INT64_MAX): return return self.timestamp - (self._WEBKIT_TO_POSIX_BASE * 1000000) dfdatetime-20170103/requirements.txt000066400000000000000000000000241303277242400173430ustar00rootroot00000000000000pip >= 7.0.0 pytest dfdatetime-20170103/run_tests.py000077500000000000000000000013261303277242400164700ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Script to run the tests.""" import sys import unittest # Change PYTHONPATH to include dfDateTime. sys.path.insert(0, u'.') if __name__ == '__main__': fail_unless_has_test_file = '--fail-unless-has-test-file' in sys.argv setattr(unittest, 'fail_unless_has_test_file', fail_unless_has_test_file) if fail_unless_has_test_file: # Remove --fail-unless-has-test-file otherwise it will conflict with # the argparse tests. sys.argv.remove('--fail-unless-has-test-file') test_suite = unittest.TestLoader().discover('tests', pattern='*.py') test_results = unittest.TextTestRunner(verbosity=2).run(test_suite) if not test_results.wasSuccessful(): sys.exit(1) dfdatetime-20170103/setup.cfg000066400000000000000000000003401303277242400157010ustar00rootroot00000000000000[bdist_rpm] release = 1 packager = log2timeline development team doc_files = ACKNOWLEDGEMENTS AUTHORS LICENSE README build_requires = python-setuptools dfdatetime-20170103/setup.py000077500000000000000000000103031303277242400155750ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Installation and deployment script.""" from __future__ import print_function import os import sys import run_tests try: from setuptools import find_packages, setup, Command except ImportError: from distutils.core import find_packages, setup, Command try: from setuptools.commands.bdist_rpm import bdist_rpm except ImportError: from distutils.command.bdist_rpm import bdist_rpm if sys.version < '2.7': print('Unsupported Python version: {0:s}.'.format(sys.version)) print('Supported Python versions are 2.7 or a later 2.x version.') sys.exit(1) # Change PYTHONPATH to include dfdatetime so that we can get the version. sys.path.insert(0, '.') import dfdatetime class BdistRPMCommand(bdist_rpm): """Custom handler for the bdist_rpm command.""" def _make_spec_file(self): """Generates the text of an RPM spec file. Returns: A list of strings containing the lines of text. """ # Note that bdist_rpm can be an old style class. if issubclass(BdistRPMCommand, object): spec_file = super(BdistRPMCommand, self)._make_spec_file() else: spec_file = bdist_rpm._make_spec_file(self) if sys.version_info[0] < 3: python_package = 'python' else: python_package = 'python3' description = [] summary = '' in_description = False python_spec_file = [] for index, line in enumerate(spec_file): if line.startswith('Summary: '): summary = line elif line.startswith('BuildRequires: '): line = 'BuildRequires: {0:s}-setuptools'.format(python_package) elif line.startswith('Requires: '): if python_package == 'python3': line = line.replace('python', 'python3') elif line.startswith('%description'): in_description = True elif line.startswith('%files'): line = '%files -f INSTALLED_FILES -n {0:s}-%{{name}}'.format( python_package) elif line.startswith('%prep'): in_description = False python_spec_file.append( '%package -n {0:s}-%{{name}}'.format(python_package)) python_spec_file.append('{0:s}'.format(summary)) python_spec_file.append('') python_spec_file.append( '%description -n {0:s}-%{{name}}'.format(python_package)) python_spec_file.extend(description) elif in_description: # Ignore leading white lines in the description. if not description and not line: continue description.append(line) python_spec_file.append(line) return python_spec_file class TestCommand(Command): """Run tests, implementing an interface.""" user_options = [] def initialize_options(self): self._dir = os.getcwd() def finalize_options(self): pass def run(self): test_results = run_tests.RunTests(os.path.join('.', 'dfdatetime')) dfdatetime_version = dfdatetime.__version__ # Command bdist_msi does not support the library version, neither a date # as a version but if we suffix it with .1 everything is fine. if 'bdist_msi' in sys.argv: dfdatetime_version += '.1' dfdatetime_description = ( 'Digital Forensics Date and Time (dfDateTime).') dfdatetime_long_description = ( 'dfDateTime, or Digital Forensics date and time, provides date and time ' 'objects to preserve accuracy and precision.') setup( name='dfdatetime', version=dfdatetime_version, description=dfdatetime_description, long_description=dfdatetime_long_description, license='Apache License, Version 2.0', url='https://github.com/log2timeline/dfdatetime', maintainer='dfDateTime development team', maintainer_email='log2timeline-dev@googlegroups.com', cmdclass={ 'bdist_rpm': BdistRPMCommand, 'test': TestCommand}, classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Console', 'Operating System :: OS Independent', 'Programming Language :: Python', ], packages=find_packages('.', exclude=[ 'examples', 'tests', 'tests.*', 'utils']), package_dir={ 'dfdatetime': 'dfdatetime' }, data_files=[ ('share/doc/dfdatetime', [ 'AUTHORS', 'ACKNOWLEDGEMENTS', 'LICENSE', 'README']), ], ) dfdatetime-20170103/tests/000077500000000000000000000000001303277242400152255ustar00rootroot00000000000000dfdatetime-20170103/tests/__init__.py000066400000000000000000000000301303277242400173270ustar00rootroot00000000000000# -*- coding: utf-8 -*- dfdatetime-20170103/tests/cocoa_time.py000066400000000000000000000047121303277242400177050ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the Cocoa timestamp implementation.""" import unittest from dfdatetime import cocoa_time class CocoaTimeTest(unittest.TestCase): """Tests for the Cocoa timestamp.""" def testCopyFromString(self): """Tests the CopyFromString function.""" cocoa_time_object = cocoa_time.CocoaTime() expected_timestamp = 394934400.0 cocoa_time_object.CopyFromString(u'2013-07-08') self.assertEqual(cocoa_time_object.timestamp, expected_timestamp) expected_timestamp = 395011845.0 cocoa_time_object.CopyFromString(u'2013-07-08 21:30:45') self.assertEqual(cocoa_time_object.timestamp, expected_timestamp) expected_timestamp = 395011845.546875 cocoa_time_object.CopyFromString(u'2013-07-08 21:30:45.546875') self.assertEqual(cocoa_time_object.timestamp, expected_timestamp) expected_timestamp = 395015445.546875 cocoa_time_object.CopyFromString(u'2013-07-08 21:30:45.546875-01:00') self.assertEqual(cocoa_time_object.timestamp, expected_timestamp) expected_timestamp = 395008245.546875 cocoa_time_object.CopyFromString(u'2013-07-08 21:30:45.546875+01:00') self.assertEqual(cocoa_time_object.timestamp, expected_timestamp) expected_timestamp = 86400.0 cocoa_time_object.CopyFromString(u'2001-01-02 00:00:00') self.assertEqual(cocoa_time_object.timestamp, expected_timestamp) def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" cocoa_time_object = cocoa_time.CocoaTime(timestamp=395011845.0) expected_stat_time_tuple = (1373319045, 0) stat_time_tuple = cocoa_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) cocoa_time_object = cocoa_time.CocoaTime() expected_stat_time_tuple = (None, None) stat_time_tuple = cocoa_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" cocoa_time_object = cocoa_time.CocoaTime(timestamp=395011845.0) expected_micro_posix_timestamp = 1373319045000000 micro_posix_timestamp = cocoa_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) cocoa_time_object = cocoa_time.CocoaTime() micro_posix_timestamp = cocoa_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/delphi_date_time.py000066400000000000000000000054551303277242400210700ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the Delphi TDateTime timestamp implementation.""" import unittest from dfdatetime import delphi_date_time class DelphiDateTimeTest(unittest.TestCase): """Tests for the Delphi TDateTime timestamp.""" def testCopyFromString(self): """Tests the CopyFromString function.""" delphi_date_time_object = delphi_date_time.DelphiDateTime() expected_timestamp = 41443.0 delphi_date_time_object.CopyFromString(u'2013-06-18') self.assertEqual(delphi_date_time_object.timestamp, expected_timestamp) expected_timestamp = 41443.82638888889 delphi_date_time_object.CopyFromString(u'2013-06-18 19:50:00') self.assertEqual(delphi_date_time_object.timestamp, expected_timestamp) expected_timestamp = 41443.826395218464 delphi_date_time_object.CopyFromString(u'2013-06-18 19:50:00.546875') self.assertEqual(delphi_date_time_object.timestamp, expected_timestamp) expected_timestamp = 41443.86806188513 delphi_date_time_object.CopyFromString(u'2013-06-18 19:50:00.546875-01:00') self.assertEqual(delphi_date_time_object.timestamp, expected_timestamp) expected_timestamp = 41443.78472855179 delphi_date_time_object.CopyFromString(u'2013-06-18 19:50:00.546875+01:00') self.assertEqual(delphi_date_time_object.timestamp, expected_timestamp) expected_timestamp = 1.0 delphi_date_time_object.CopyFromString(u'1899-12-31 00:00:00') self.assertEqual(delphi_date_time_object.timestamp, expected_timestamp) with self.assertRaises(ValueError): delphi_date_time_object.CopyFromString(u'10000-01-01 00:00:00') def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" delphi_date_time_object = delphi_date_time.DelphiDateTime( timestamp=41443.8263953) expected_stat_time_tuple = (1371585000, 5539197) stat_time_tuple = delphi_date_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) delphi_date_time_object = delphi_date_time.DelphiDateTime() expected_stat_time_tuple = (None, None) stat_time_tuple = delphi_date_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" delphi_date_time_object = delphi_date_time.DelphiDateTime( timestamp=41443.8263953) expected_micro_posix_timestamp = 1371585000553920 micro_posix_timestamp = delphi_date_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) delphi_date_time_object = delphi_date_time.DelphiDateTime() micro_posix_timestamp = delphi_date_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/fake_time.py000066400000000000000000000075011303277242400175260ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the fake time implementation.""" import unittest from dfdatetime import fake_time class FakeTimeTest(unittest.TestCase): """Tests for the fake time.""" # pylint: disable=protected-access def testCopyFromString(self): """Tests the CopyFromString function.""" fake_time_object = fake_time.FakeTime() expected_number_of_seconds = 1281571200 fake_time_object.CopyFromString(u'2010-08-12') self.assertEqual( fake_time_object._number_of_seconds, expected_number_of_seconds) self.assertIsNone(fake_time_object._microseconds) expected_number_of_seconds = 1281647191 fake_time_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual( fake_time_object._number_of_seconds, expected_number_of_seconds) self.assertIsNone(fake_time_object._microseconds) expected_number_of_seconds = 1281647191 fake_time_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual( fake_time_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(fake_time_object._microseconds, 546875) expected_number_of_seconds = 1281650791 fake_time_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual( fake_time_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(fake_time_object._microseconds, 546875) expected_number_of_seconds = 1281643591 fake_time_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual( fake_time_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(fake_time_object._microseconds, 546875) expected_number_of_seconds = -11644387200 fake_time_object.CopyFromString(u'1601-01-02 00:00:00') self.assertEqual( fake_time_object._number_of_seconds, expected_number_of_seconds) self.assertIsNone(fake_time_object._microseconds) def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" fake_time_object = fake_time.FakeTime() fake_time_object.CopyFromString(u'2010-08-12 21:06:31.546875') expected_stat_time_tuple = (1281647191, 5468750) stat_time_tuple = fake_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) fake_time_object = fake_time.FakeTime() fake_time_object.CopyFromString(u'2010-08-12 21:06:31') expected_stat_time_tuple = (1281647191, None) stat_time_tuple = fake_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) fake_time_object = fake_time.FakeTime() fake_time_object._number_of_seconds = None expected_stat_time_tuple = (None, None) stat_time_tuple = fake_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" fake_time_object = fake_time.FakeTime() fake_time_object.CopyFromString(u'2010-08-12 21:06:31.546875') expected_micro_posix_number_of_seconds = 1281647191546875 micro_posix_number_of_seconds = fake_time_object.GetPlasoTimestamp() self.assertEqual( micro_posix_number_of_seconds, expected_micro_posix_number_of_seconds) fake_time_object = fake_time.FakeTime() fake_time_object.CopyFromString(u'2010-08-12 21:06:31') expected_micro_posix_number_of_seconds = 1281647191000000 micro_posix_number_of_seconds = fake_time_object.GetPlasoTimestamp() self.assertEqual( micro_posix_number_of_seconds, expected_micro_posix_number_of_seconds) fake_time_object = fake_time.FakeTime() fake_time_object._number_of_seconds = None micro_posix_number_of_seconds = fake_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_number_of_seconds) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/fat_date_time.py000066400000000000000000000101121303277242400203570ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the FAT date time implementation.""" import unittest from dfdatetime import fat_date_time class FATDateTime(unittest.TestCase): """Tests for the FAT date time.""" # pylint: disable=protected-access def testCopyFromString(self): """Tests the CopyFromString function.""" fat_date_time_object = fat_date_time.FATDateTime() expected_number_of_seconds = 966038400 fat_date_time_object.CopyFromString(u'2010-08-12') self.assertEqual( fat_date_time_object._number_of_seconds, expected_number_of_seconds) expected_number_of_seconds = 966114391 fat_date_time_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual( fat_date_time_object._number_of_seconds, expected_number_of_seconds) expected_number_of_seconds = 966114391 fat_date_time_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual( fat_date_time_object._number_of_seconds, expected_number_of_seconds) expected_number_of_seconds = 966117991 fat_date_time_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual( fat_date_time_object._number_of_seconds, expected_number_of_seconds) expected_number_of_seconds = 966110791 fat_date_time_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual( fat_date_time_object._number_of_seconds, expected_number_of_seconds) expected_number_of_seconds = 86400 fat_date_time_object.CopyFromString(u'1980-01-02 00:00:00') self.assertEqual( fat_date_time_object._number_of_seconds, expected_number_of_seconds) with self.assertRaises(ValueError): fat_date_time_object.CopyFromString(u'2200-01-02 00:00:00') def testGetNumberOfSeconds(self): """Tests the _GetNumberOfSeconds function.""" fat_date_time_object = fat_date_time.FATDateTime() fat_date_time_object._GetNumberOfSeconds(0xa8d03d0c) # Invalid number of seconds. test_fat_date_time = (0xa8d03d0c & ~(0x1f << 16)) | ((30 & 0x1f) << 16) with self.assertRaises(ValueError): fat_date_time_object._GetNumberOfSeconds(test_fat_date_time) # Invalid number of minutes. test_fat_date_time = (0xa8d03d0c & ~(0x3f << 21)) | ((60 & 0x3f) << 21) with self.assertRaises(ValueError): fat_date_time_object._GetNumberOfSeconds(test_fat_date_time) # Invalid number of hours. test_fat_date_time = (0xa8d03d0c & ~(0x1f << 27)) | ((24 & 0x1f) << 27) with self.assertRaises(ValueError): fat_date_time_object._GetNumberOfSeconds(test_fat_date_time) # Invalid day of month. test_fat_date_time = (0xa8d03d0c & ~0x1f) | (32 & 0x1f) with self.assertRaises(ValueError): fat_date_time_object._GetNumberOfSeconds(test_fat_date_time) # Invalid month. test_fat_date_time = (0xa8d03d0c & ~(0x0f << 5)) | ((13 & 0x0f) << 5) with self.assertRaises(ValueError): fat_date_time_object._GetNumberOfSeconds(test_fat_date_time) def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" fat_date_time_object = fat_date_time.FATDateTime(fat_date_time=0xa8d03d0c) expected_stat_time_tuple = (1281647192, None) stat_time_tuple = fat_date_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) fat_date_time_object = fat_date_time.FATDateTime() expected_stat_time_tuple = (None, None) stat_time_tuple = fat_date_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" fat_date_time_object = fat_date_time.FATDateTime(fat_date_time=0xa8d03d0c) expected_micro_posix_timestamp = 1281647192000000 micro_posix_timestamp = fat_date_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) fat_date_time_object = fat_date_time.FATDateTime() micro_posix_timestamp = fat_date_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/filetime.py000066400000000000000000000057161303277242400174060ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the FILETIME timestamp implementation.""" import unittest from dfdatetime import filetime class FiletimeTest(unittest.TestCase): """Tests for the FILETIME timestamp.""" def testCopyFromString(self): """Tests the CopyFromString function.""" filetime_object = filetime.Filetime() expected_timestamp = 0x1cb39b14e8c4000 filetime_object.CopyFromString(u'2010-08-12') self.assertEqual(filetime_object.timestamp, expected_timestamp) expected_timestamp = 0x1cb3a623cb6a580 filetime_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual(filetime_object.timestamp, expected_timestamp) expected_timestamp = 0x01cb3a623d0a17ce filetime_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual(filetime_object.timestamp, expected_timestamp) expected_timestamp = 0x1cb3a6a9ece7fce filetime_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual(filetime_object.timestamp, expected_timestamp) expected_timestamp = 0x1cb3a59db45afce filetime_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual(filetime_object.timestamp, expected_timestamp) expected_timestamp = 86400 * 10000000 filetime_object.CopyFromString(u'1601-01-02 00:00:00') self.assertEqual(filetime_object.timestamp, expected_timestamp) with self.assertRaises(ValueError): filetime_object.CopyFromString(u'1500-01-02 00:00:00') def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" filetime_object = filetime.Filetime(timestamp=0x01cb3a623d0a17ce) expected_stat_time_tuple = (1281647191, 5468750) stat_time_tuple = filetime_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) filetime_object = filetime.Filetime(timestamp=0x1ffffffffffffffff) expected_stat_time_tuple = (None, None) stat_time_tuple = filetime_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) filetime_object = filetime.Filetime() expected_stat_time_tuple = (None, None) stat_time_tuple = filetime_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" filetime_object = filetime.Filetime(timestamp=0x01cb3a623d0a17ce) expected_micro_posix_timestamp = 1281647191546875 micro_posix_timestamp = filetime_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) filetime_object = filetime.Filetime(timestamp=0x1ffffffffffffffff) micro_posix_timestamp = filetime_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) filetime_object = filetime.Filetime() micro_posix_timestamp = filetime_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/hfs_time.py000066400000000000000000000063761303277242400174110ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the HFS timestamp implementation.""" import unittest from dfdatetime import hfs_time class HFSTimeTest(unittest.TestCase): """Tests for the HFS timestamp.""" def testCopyFromString(self): """Tests the CopyFromString function.""" hfs_time_object = hfs_time.HFSTime() expected_timestamp = 3458160000 hfs_time_object.CopyFromString(u'2013-08-01') self.assertEqual(hfs_time_object.timestamp, expected_timestamp) expected_timestamp = 3458215528 hfs_time_object.CopyFromString(u'2013-08-01 15:25:28') self.assertEqual(hfs_time_object.timestamp, expected_timestamp) expected_timestamp = 3458215528 hfs_time_object.CopyFromString(u'2013-08-01 15:25:28.546875') self.assertEqual(hfs_time_object.timestamp, expected_timestamp) expected_timestamp = 3458219128 hfs_time_object.CopyFromString(u'2013-08-01 15:25:28.546875-01:00') self.assertEqual(hfs_time_object.timestamp, expected_timestamp) expected_timestamp = 3458211928 hfs_time_object.CopyFromString(u'2013-08-01 15:25:28.546875+01:00') self.assertEqual(hfs_time_object.timestamp, expected_timestamp) expected_timestamp = 86400 hfs_time_object.CopyFromString(u'1904-01-02 00:00:00') self.assertEqual(hfs_time_object.timestamp, expected_timestamp) with self.assertRaises(ValueError): hfs_time_object.CopyFromString(u'1600-01-02 00:00:00') def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" hfs_time_object = hfs_time.HFSTime(timestamp=3458215528) expected_stat_time_tuple = (1375370728, 0) stat_time_tuple = hfs_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) hfs_time_object = hfs_time.HFSTime(timestamp=0x1ffffffff) expected_stat_time_tuple = (None, None) stat_time_tuple = hfs_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) hfs_time_object = hfs_time.HFSTime(timestamp=-0x1ffffffff) expected_stat_time_tuple = (None, None) stat_time_tuple = hfs_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) hfs_time_object = hfs_time.HFSTime() expected_stat_time_tuple = (None, None) stat_time_tuple = hfs_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" hfs_time_object = hfs_time.HFSTime(timestamp=3458215528) expected_micro_posix_timestamp = 1375370728000000 micro_posix_timestamp = hfs_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) hfs_time_object = hfs_time.HFSTime(timestamp=0x1ffffffff) micro_posix_timestamp = hfs_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) hfs_time_object = hfs_time.HFSTime(timestamp=-0x1ffffffff) micro_posix_timestamp = hfs_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) hfs_time_object = hfs_time.HFSTime() micro_posix_timestamp = hfs_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/interface.py000066400000000000000000000303471303277242400175460ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the date and time values interface.""" import unittest from dfdatetime import interface class DateTimeValuesTest(unittest.TestCase): """Tests for the date and time values interface.""" # pylint: disable=protected-access def testCopyDateFromString(self): """Tests the _CopyDateFromString function.""" date_time_values = interface.DateTimeValues() expected_date_tuple = (2010, 1, 1) date_tuple = date_time_values._CopyDateFromString(u'2010-01-01') self.assertEqual(date_tuple, expected_date_tuple) expected_date_tuple = (2010, 8, 1) date_tuple = date_time_values._CopyDateFromString(u'2010-08-01') self.assertEqual(date_tuple, expected_date_tuple) expected_date_tuple = (2010, 8, 12) date_tuple = date_time_values._CopyDateFromString(u'2010-08-12') self.assertEqual(date_tuple, expected_date_tuple) expected_date_tuple = (2010, 8, 31) date_tuple = date_time_values._CopyDateFromString(u'2010-08-31') self.assertEqual(date_tuple, expected_date_tuple) expected_date_tuple = (2010, 12, 31) date_tuple = date_time_values._CopyDateFromString(u'2010-12-31') self.assertEqual(date_tuple, expected_date_tuple) expected_date_tuple = (1601, 1, 2) date_tuple = date_time_values._CopyDateFromString(u'1601-01-02') self.assertEqual(date_tuple, expected_date_tuple) with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'195a-01-02') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'10000-01-02') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'2010-09-00') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'2010-09-31') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'1601-01-32') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'1601-01-b2') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'1601-13-02') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'1601-a1-02') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'2010-02-29') with self.assertRaises(ValueError): date_time_values._CopyDateFromString(u'2010-04-31') def testCopyDateTimeFromString(self): """Tests the _CopyDateTimeFromString function.""" date_time_values = interface.DateTimeValues() expected_date_dict = { u'year': 2010, u'month': 8, u'day_of_month': 12} date_dict = date_time_values._CopyDateTimeFromString(u'2010-08-12') self.assertEqual(date_dict, expected_date_dict) expected_date_dict = { u'year': 2010, u'month': 8, u'day_of_month': 12, u'hours': 21, u'minutes': 6, u'seconds': 31} date_dict = date_time_values._CopyDateTimeFromString( u'2010-08-12 21:06:31') self.assertEqual(date_dict, expected_date_dict) expected_date_dict = { u'year': 2010, u'month': 8, u'day_of_month': 12, u'hours': 21, u'minutes': 6, u'seconds': 31, u'microseconds': 546875} date_dict = date_time_values._CopyDateTimeFromString( u'2010-08-12 21:06:31.546875') self.assertEqual(date_dict, expected_date_dict) expected_date_dict = { u'year': 2010, u'month': 8, u'day_of_month': 12, u'hours': 22, u'minutes': 6, u'seconds': 31, u'microseconds': 546875} date_dict = date_time_values._CopyDateTimeFromString( u'2010-08-12 21:06:31.546875-01:00') self.assertEqual(date_dict, expected_date_dict) expected_date_dict = { u'year': 2010, u'month': 8, u'day_of_month': 12, u'hours': 20, u'minutes': 6, u'seconds': 31, u'microseconds': 546875} date_dict = date_time_values._CopyDateTimeFromString( u'2010-08-12 21:06:31.546875+01:00') self.assertEqual(date_dict, expected_date_dict) expected_date_dict = { u'year': 2010, u'month': 8, u'day_of_month': 12, u'hours': 20, u'minutes': 6, u'seconds': 31, u'microseconds': 546875} date_dict = date_time_values._CopyDateTimeFromString( u'2010-08-12 21:06:31.546875+01:00') self.assertEqual(date_dict, expected_date_dict) # Test backwards date correction. expected_date_dict = { u'year': 2009, u'month': 12, u'day_of_month': 31, u'hours': 23, u'minutes': 45, u'seconds': 0, u'microseconds': 123456} date_dict = date_time_values._CopyDateTimeFromString( u'2010-01-01 00:15:00.123456+00:30') self.assertEqual(date_dict, expected_date_dict) # Test forward date correction. expected_date_dict = { u'year': 2010, u'month': 1, u'day_of_month': 1, u'hours': 1, u'minutes': 15, u'seconds': 0, u'microseconds': 123456} date_dict = date_time_values._CopyDateTimeFromString( u'2009-12-31 23:45:00.123456-01:30') self.assertEqual(date_dict, expected_date_dict) with self.assertRaises(ValueError): date_time_values._CopyDateTimeFromString(u'') with self.assertRaises(ValueError): date_time_values._CopyDateTimeFromString( u'2010-08-12T21:06:31.546875+01:00') def testCopyTimeFromString(self): """Tests the _CopyTimeFromString function.""" date_time_values = interface.DateTimeValues() expected_time_tuple = (8, 4, 32, None, None) time_tuple = date_time_values._CopyTimeFromString(u'08:04:32') self.assertEqual(time_tuple, expected_time_tuple) expected_time_tuple = (20, 23, 56, None, None) time_tuple = date_time_values._CopyTimeFromString(u'20:23:56') self.assertEqual(time_tuple, expected_time_tuple) expected_time_tuple = (20, 23, 56, None, -330) time_tuple = date_time_values._CopyTimeFromString(u'20:23:56+05:30') self.assertEqual(time_tuple, expected_time_tuple) expected_time_tuple = (20, 23, 56, 327000, None) time_tuple = date_time_values._CopyTimeFromString(u'20:23:56.327') self.assertEqual(time_tuple, expected_time_tuple) expected_time_tuple = (20, 23, 56, 327000, -60) time_tuple = date_time_values._CopyTimeFromString(u'20:23:56.327+01:00') self.assertEqual(time_tuple, expected_time_tuple) expected_time_tuple = (20, 23, 56, 327124, None) time_tuple = date_time_values._CopyTimeFromString(u'20:23:56.327124') self.assertEqual(time_tuple, expected_time_tuple) expected_time_tuple = (20, 23, 56, 327124, 300) time_tuple = date_time_values._CopyTimeFromString(u'20:23:56.327124-05:00') self.assertEqual(time_tuple, expected_time_tuple) with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'14') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'14:00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'24:00:00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'12b00:00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'12:00b00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'1s:00:00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'00:60:00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'00:e0:00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'00:00:60') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'00:00:w0') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'12:00:00.00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'12:00:00.0000') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'12:00:00.00w') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'12:00:00+01b00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'12:00:00+01:0w') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'12:00:00+0w:00') with self.assertRaises(ValueError): date_time_values._CopyTimeFromString(u'12:00:00+30:00') def testGetDayOfYear(self): """Tests the _GetDayOfYear function.""" date_time_values = interface.DateTimeValues() day_of_year = date_time_values._GetDayOfYear(1999, 1, 1) self.assertEqual(day_of_year, 1) day_of_year = date_time_values._GetDayOfYear(1999, 4, 21) self.assertEqual(day_of_year, 111) day_of_year = date_time_values._GetDayOfYear(1999, 12, 31) self.assertEqual(day_of_year, 365) day_of_year = date_time_values._GetDayOfYear(2000, 1, 1) self.assertEqual(day_of_year, 1) day_of_year = date_time_values._GetDayOfYear(2000, 4, 21) self.assertEqual(day_of_year, 112) day_of_year = date_time_values._GetDayOfYear(2000, 12, 31) self.assertEqual(day_of_year, 366) with self.assertRaises(ValueError): date_time_values._GetDayOfYear(1999, 0, 1) with self.assertRaises(ValueError): date_time_values._GetDayOfYear(1999, 13, 1) with self.assertRaises(ValueError): date_time_values._GetDayOfYear(1999, 1, 0) with self.assertRaises(ValueError): date_time_values._GetDayOfYear(1999, 1, 32) def testGetDaysPerMonth(self): """Tests the _GetDaysPerMonth function.""" date_time_values = interface.DateTimeValues() expected_days_per_month = list(interface.DateTimeValues._DAYS_PER_MONTH) days_per_month = [] for month in range(1, 13): days_per_month.append(date_time_values._GetDaysPerMonth(1999, month)) self.assertEqual(days_per_month, expected_days_per_month) expected_days_per_month[1] += 1 days_per_month = [] for month in range(1, 13): days_per_month.append(date_time_values._GetDaysPerMonth(2000, month)) self.assertEqual(days_per_month, expected_days_per_month) with self.assertRaises(ValueError): date_time_values._GetDaysPerMonth(1999, 0) with self.assertRaises(ValueError): date_time_values._GetDaysPerMonth(1999, 13) def testGetNumberOfDaysInYear(self): """Tests the _GetNumberOfDaysInYear function.""" date_time_values = interface.DateTimeValues() self.assertEqual(date_time_values._GetNumberOfDaysInYear(1999), 365) self.assertEqual(date_time_values._GetNumberOfDaysInYear(2000), 366) self.assertEqual(date_time_values._GetNumberOfDaysInYear(1996), 366) def testGetNumberOfSecondsFromElements(self): """Tests the _GetNumberOfSecondsFromElements function.""" date_time_values = interface.DateTimeValues() number_of_seconds = date_time_values._GetNumberOfSecondsFromElements( 2010, 8, 12, 0, 0, 0) self.assertEqual(number_of_seconds, 1281571200) number_of_seconds = date_time_values._GetNumberOfSecondsFromElements( 2010, 8, 12, None, None, None) self.assertEqual(number_of_seconds, 1281571200) number_of_seconds = date_time_values._GetNumberOfSecondsFromElements( 2010, 8, 12, 21, 6, 31) self.assertEqual(number_of_seconds, 1281647191) number_of_seconds = date_time_values._GetNumberOfSecondsFromElements( 1601, 1, 2, 0, 0, 0) self.assertEqual(number_of_seconds, -11644387200) number_of_seconds = date_time_values._GetNumberOfSecondsFromElements( 0, 1, 2, 0, 0, 0) self.assertIsNone(number_of_seconds) with self.assertRaises(ValueError): date_time_values._GetNumberOfSecondsFromElements(2010, 13, 12, 21, 6, 31) with self.assertRaises(ValueError): date_time_values._GetNumberOfSecondsFromElements(2010, 13, 12, 24, 6, 31) with self.assertRaises(ValueError): date_time_values._GetNumberOfSecondsFromElements(2010, 13, 12, 21, 99, 31) with self.assertRaises(ValueError): date_time_values._GetNumberOfSecondsFromElements(2010, 13, 12, 21, 6, 65) def testIsLeapYear(self): """Tests the _IsLeapYear function.""" date_time_values = interface.DateTimeValues() self.assertFalse(date_time_values._IsLeapYear(1999)) self.assertTrue(date_time_values._IsLeapYear(2000)) self.assertTrue(date_time_values._IsLeapYear(1996)) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/java_time.py000066400000000000000000000047221303277242400175430ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the Java java.util.Date timestamp implementation.""" import unittest from dfdatetime import java_time class JavaTimeTest(unittest.TestCase): """Tests for the Java java.util.Date timestamp.""" def testCopyFromString(self): """Tests the CopyFromString function.""" java_time_object = java_time.JavaTime() expected_timestamp = 1281571200000 java_time_object.CopyFromString(u'2010-08-12') self.assertEqual(java_time_object.timestamp, expected_timestamp) expected_timestamp = 1281647191000 java_time_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual(java_time_object.timestamp, expected_timestamp) expected_timestamp = 1281647191546 java_time_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual(java_time_object.timestamp, expected_timestamp) expected_timestamp = 1281650791546 java_time_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual(java_time_object.timestamp, expected_timestamp) expected_timestamp = 1281643591546 java_time_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual(java_time_object.timestamp, expected_timestamp) expected_timestamp = -11644387200000 java_time_object.CopyFromString(u'1601-01-02 00:00:00') self.assertEqual(java_time_object.timestamp, expected_timestamp) def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" java_time_object = java_time.JavaTime(timestamp=1281643591546) expected_stat_time_tuple = (1281643591, 5460000) stat_time_tuple = java_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) java_time_object = java_time.JavaTime() expected_stat_time_tuple = (None, None) stat_time_tuple = java_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" java_time_object = java_time.JavaTime(timestamp=1281643591546) expected_micro_posix_timestamp = 1281643591546000 micro_posix_timestamp = java_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) java_time_object = java_time.JavaTime() micro_posix_timestamp = java_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/posix_time.py000066400000000000000000000115401303277242400177600ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the POSIX timestamp implementation.""" import unittest from dfdatetime import posix_time class PosixTimeTest(unittest.TestCase): """Tests for the POSIX timestamp.""" def testCopyFromString(self): """Tests the CopyFromString function.""" posix_time_object = posix_time.PosixTime() expected_timestamp = 1281571200 posix_time_object.CopyFromString(u'2010-08-12') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = 1281647191 posix_time_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = 1281647191 posix_time_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = 1281650791 posix_time_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = 1281643591 posix_time_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = -11644387200 posix_time_object.CopyFromString(u'1601-01-02 00:00:00') self.assertEqual(posix_time_object.timestamp, expected_timestamp) def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" posix_time_object = posix_time.PosixTime(timestamp=1281643591) expected_stat_time_tuple = (1281643591, None) stat_time_tuple = posix_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) posix_time_object = posix_time.PosixTime() expected_stat_time_tuple = (None, None) stat_time_tuple = posix_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" posix_time_object = posix_time.PosixTime(timestamp=1281643591) expected_micro_posix_timestamp = 1281643591000000 micro_posix_timestamp = posix_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) posix_time_object = posix_time.PosixTime() micro_posix_timestamp = posix_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) class PosixTimeInMicrosecondsTest(unittest.TestCase): """Tests for the POSIX timestamp in microseconds.""" def testCopyFromString(self): """Tests the CopyFromString function.""" posix_time_object = posix_time.PosixTimeInMicroseconds() expected_timestamp = 1281571200000000 posix_time_object.CopyFromString(u'2010-08-12') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = 1281647191000000 posix_time_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = 1281647191546875 posix_time_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = 1281650791546875 posix_time_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = 1281643591546875 posix_time_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual(posix_time_object.timestamp, expected_timestamp) expected_timestamp = -11644387200000000 posix_time_object.CopyFromString(u'1601-01-02 00:00:00') self.assertEqual(posix_time_object.timestamp, expected_timestamp) def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" posix_time_object = posix_time.PosixTimeInMicroseconds( timestamp=1281643591546875) expected_stat_time_tuple = (1281643591, 5468750) stat_time_tuple = posix_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) posix_time_object = posix_time.PosixTimeInMicroseconds() expected_stat_time_tuple = (None, None) stat_time_tuple = posix_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" posix_time_object = posix_time.PosixTimeInMicroseconds( timestamp=1281643591546875) expected_micro_posix_timestamp = 1281643591546875 micro_posix_timestamp = posix_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) posix_time_object = posix_time.PosixTimeInMicroseconds() micro_posix_timestamp = posix_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/semantic_time.py000066400000000000000000000022501303277242400204170ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the semantic time implementation.""" import unittest from dfdatetime import semantic_time class SemanticTimeTest(unittest.TestCase): """Tests for semantic time.""" # pylint: disable=protected-access def testCopyFromString(self): """Tests the CopyFromString function.""" semantic_time_object = semantic_time.SemanticTime() semantic_time_object.CopyFromString(u'Never') self.assertEqual(semantic_time_object.string, u'Never') def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" semantic_time_object = semantic_time.SemanticTime() expected_stat_time_tuple = (0, 0) stat_time_tuple = semantic_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" semantic_time_object = semantic_time.SemanticTime() expected_micro_posix_timestamp = 0 micro_posix_timestamp = semantic_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/systemtime.py000066400000000000000000000156231303277242400200110ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the SYSTEMTIME structure implementation.""" import unittest from dfdatetime import systemtime class FiletimeTest(unittest.TestCase): """Tests for the SYSTEMTIME structure.""" # pylint: disable=protected-access def testInitialize(self): """Tests the initialization function.""" systemtime_object = systemtime.Systemtime() self.assertIsNotNone(systemtime_object) systemtime_object = systemtime.Systemtime( system_time_tuple=(2010, 8, 4, 12, 20, 6, 31, 142)) self.assertIsNotNone(systemtime_object) self.assertEqual(systemtime_object.year, 2010) self.assertEqual(systemtime_object.month, 8) self.assertEqual(systemtime_object.day_of_month, 12) self.assertEqual(systemtime_object.hours, 20) self.assertEqual(systemtime_object.minutes, 6) self.assertEqual(systemtime_object.seconds, 31) self.assertEqual(systemtime_object.milliseconds, 142) with self.assertRaises(ValueError): systemtime.Systemtime( system_time_tuple=(2010, 8, 4, 12, 20, 6, 31)) with self.assertRaises(ValueError): systemtime.Systemtime( system_time_tuple=(1500, 8, 4, 12, 20, 6, 31, 142)) with self.assertRaises(ValueError): systemtime.Systemtime( system_time_tuple=(2010, 13, 4, 12, 20, 6, 31, 142)) with self.assertRaises(ValueError): systemtime.Systemtime( system_time_tuple=(2010, 8, 7, 12, 20, 6, 31, 142)) with self.assertRaises(ValueError): systemtime.Systemtime( system_time_tuple=(2010, 8, 4, 32, 20, 6, 31, 142)) with self.assertRaises(ValueError): systemtime.Systemtime( system_time_tuple=(2010, 8, 4, 12, 24, 6, 31, 142)) with self.assertRaises(ValueError): systemtime.Systemtime( system_time_tuple=(2010, 8, 4, 12, 20, 61, 31, 142)) with self.assertRaises(ValueError): systemtime.Systemtime( system_time_tuple=(2010, 8, 4, 12, 20, 6, 61, 142)) with self.assertRaises(ValueError): systemtime.Systemtime( system_time_tuple=(2010, 8, 4, 12, 20, 6, 31, 1001)) def testCopyFromString(self): """Tests the CopyFromString function.""" systemtime_object = systemtime.Systemtime() expected_number_of_seconds = 1281571200 systemtime_object.CopyFromString(u'2010-08-12') self.assertEqual( systemtime_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(systemtime_object.year, 2010) self.assertEqual(systemtime_object.month, 8) self.assertEqual(systemtime_object.day_of_month, 12) self.assertEqual(systemtime_object.hours, 0) self.assertEqual(systemtime_object.minutes, 0) self.assertEqual(systemtime_object.seconds, 0) self.assertEqual(systemtime_object.milliseconds, 0) expected_number_of_seconds = 1281647191 systemtime_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual( systemtime_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(systemtime_object.year, 2010) self.assertEqual(systemtime_object.month, 8) self.assertEqual(systemtime_object.day_of_month, 12) self.assertEqual(systemtime_object.hours, 21) self.assertEqual(systemtime_object.minutes, 6) self.assertEqual(systemtime_object.seconds, 31) self.assertEqual(systemtime_object.milliseconds, 0) expected_number_of_seconds = 1281647191 systemtime_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual( systemtime_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(systemtime_object.year, 2010) self.assertEqual(systemtime_object.month, 8) self.assertEqual(systemtime_object.day_of_month, 12) self.assertEqual(systemtime_object.hours, 21) self.assertEqual(systemtime_object.minutes, 6) self.assertEqual(systemtime_object.seconds, 31) self.assertEqual(systemtime_object.milliseconds, 546) expected_number_of_seconds = 1281650791 systemtime_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual( systemtime_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(systemtime_object.year, 2010) self.assertEqual(systemtime_object.month, 8) self.assertEqual(systemtime_object.day_of_month, 12) self.assertEqual(systemtime_object.hours, 22) self.assertEqual(systemtime_object.minutes, 6) self.assertEqual(systemtime_object.seconds, 31) self.assertEqual(systemtime_object.milliseconds, 546) expected_number_of_seconds = 1281643591 systemtime_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual( systemtime_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(systemtime_object.year, 2010) self.assertEqual(systemtime_object.month, 8) self.assertEqual(systemtime_object.day_of_month, 12) self.assertEqual(systemtime_object.hours, 20) self.assertEqual(systemtime_object.minutes, 6) self.assertEqual(systemtime_object.seconds, 31) self.assertEqual(systemtime_object.milliseconds, 546) expected_number_of_seconds = -11644387200 systemtime_object.CopyFromString(u'1601-01-02 00:00:00') self.assertEqual( systemtime_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(systemtime_object.year, 1601) self.assertEqual(systemtime_object.month, 1) self.assertEqual(systemtime_object.day_of_month, 2) self.assertEqual(systemtime_object.hours, 0) self.assertEqual(systemtime_object.minutes, 0) self.assertEqual(systemtime_object.seconds, 0) self.assertEqual(systemtime_object.milliseconds, 0) with self.assertRaises(ValueError): systemtime_object.CopyFromString(u'1600-01-02 00:00:00') def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" systemtime_object = systemtime.Systemtime( system_time_tuple=(2010, 8, 4, 12, 20, 6, 31, 142)) expected_stat_time_tuple = (1281643591, 1420000) stat_time_tuple = systemtime_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) systemtime_object = systemtime.Systemtime() expected_stat_time_tuple = (None, None) stat_time_tuple = systemtime_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" systemtime_object = systemtime.Systemtime( system_time_tuple=(2010, 8, 4, 12, 20, 6, 31, 142)) expected_micro_posix_number_of_seconds = 1281643591142000 micro_posix_number_of_seconds = systemtime_object.GetPlasoTimestamp() self.assertEqual( micro_posix_number_of_seconds, expected_micro_posix_number_of_seconds) systemtime_object = systemtime.Systemtime() micro_posix_number_of_seconds = systemtime_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_number_of_seconds) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/time_elements.py000066400000000000000000000455261303277242400204450ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the time elements implementation.""" import unittest from dfdatetime import time_elements class TimeElementsTest(unittest.TestCase): """Tests for the time elements.""" # pylint: disable=protected-access def testInitialize(self): """Tests the initialization function.""" time_elements_object = time_elements.TimeElements() self.assertIsNotNone(time_elements_object) expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31) time_elements_object = time_elements.TimeElements( time_elements_tuple=(2010, 8, 12, 20, 6, 31)) self.assertIsNotNone(time_elements_object) self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) with self.assertRaises(ValueError): time_elements.TimeElements( time_elements_tuple=(2010, 8, 12, 20, 6)) with self.assertRaises(ValueError): time_elements.TimeElements( time_elements_tuple=(2010, 13, 12, 20, 6, 31)) def testCopyFromString(self): """Tests the CopyFromString function.""" time_elements_object = time_elements.TimeElements() expected_time_elements_tuple = (2010, 8, 12, 0, 0, 0) expected_number_of_seconds = 1281571200 time_elements_object.CopyFromString(u'2010-08-12') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 22, 6, 31) expected_number_of_seconds = 1281650791 time_elements_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31) expected_number_of_seconds = 1281643591 time_elements_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (1601, 1, 2, 0, 0, 0) expected_number_of_seconds = -11644387200 time_elements_object.CopyFromString(u'1601-01-02 00:00:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) def testCopyFromStringISO8601(self): """Tests the CopyFromStringISO8601 function.""" time_elements_object = time_elements.TimeElements() expected_time_elements_tuple = (2010, 8, 12, 0, 0, 0) expected_number_of_seconds = 1281571200 time_elements_object.CopyFromStringISO8601(u'2010-08-12') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31Z') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31.546875') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31,546875') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 22, 6, 31) expected_number_of_seconds = 1281650791 time_elements_object.CopyFromStringISO8601( u'2010-08-12T21:06:31.546875-01:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31) expected_number_of_seconds = 1281643591 time_elements_object.CopyFromStringISO8601( u'2010-08-12T21:06:31.546875+01:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(None) with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601( u'2010-08-12 21:06:31.546875+01:00') # Valid ISO 8601 notations currently not supported. with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'2016-W33') with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'2016-W33-3') with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'--08-17') with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'2016-230') with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31.5') def testCopyFromStringTuple(self): """Tests the CopyFromStringTuple function.""" time_elements_object = time_elements.TimeElements() expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31) time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'8', u'12', u'20', u'6', u'31')) self.assertIsNotNone(time_elements_object) self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) with self.assertRaises(ValueError): time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'8', u'12', u'20', u'6')) with self.assertRaises(ValueError): time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'20A0', u'B', u'12', u'20', u'6', u'31')) with self.assertRaises(ValueError): time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'B', u'12', u'20', u'6', u'31')) with self.assertRaises(ValueError): time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'8', u'1C', u'20', u'6', u'31')) with self.assertRaises(ValueError): time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'8', u'12', u'D0', u'6', u'31')) with self.assertRaises(ValueError): time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'8', u'12', u'20', u'E', u'31')) with self.assertRaises(ValueError): time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'8', u'12', u'20', u'6', u'F1')) def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" time_elements_object = time_elements.TimeElements( time_elements_tuple=(2010, 8, 12, 20, 6, 31)) expected_stat_time_tuple = (1281643591, None) stat_time_tuple = time_elements_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) time_elements_object = time_elements.TimeElements() expected_stat_time_tuple = (None, None) stat_time_tuple = time_elements_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" time_elements_object = time_elements.TimeElements( time_elements_tuple=(2010, 8, 12, 20, 6, 31)) expected_micro_posix_number_of_seconds = 1281643591000000 micro_posix_number_of_seconds = time_elements_object.GetPlasoTimestamp() self.assertEqual( micro_posix_number_of_seconds, expected_micro_posix_number_of_seconds) time_elements_object = time_elements.TimeElements() micro_posix_number_of_seconds = time_elements_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_number_of_seconds) class TimeElementsInMillisecondsTest(unittest.TestCase): """Tests for the time elements in milliseconds.""" # pylint: disable=protected-access def testInitialize(self): """Tests the initialization function.""" time_elements_object = time_elements.TimeElements() self.assertIsNotNone(time_elements_object) expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31) time_elements_object = time_elements.TimeElementsInMilliseconds( time_elements_tuple=(2010, 8, 12, 20, 6, 31, 546)) self.assertIsNotNone(time_elements_object) self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual(time_elements_object._milliseconds, 546) with self.assertRaises(ValueError): time_elements.TimeElementsInMilliseconds( time_elements_tuple=(2010, 13, 12, 20, 6, 31)) with self.assertRaises(ValueError): time_elements.TimeElementsInMilliseconds( time_elements_tuple=(2010, 13, 12, 20, 6, 31, 1001)) def testCopyFromString(self): """Tests the CopyFromString function.""" time_elements_object = time_elements.TimeElementsInMilliseconds() expected_time_elements_tuple = (2010, 8, 12, 0, 0, 0, 0) expected_number_of_seconds = 1281571200 time_elements_object.CopyFromString(u'2010-08-12') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31, 0) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31, 546) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 22, 6, 31, 546) expected_number_of_seconds = 1281650791 time_elements_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31, 546) expected_number_of_seconds = 1281643591 time_elements_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (1601, 1, 2, 0, 0, 0, 0) expected_number_of_seconds = -11644387200 time_elements_object.CopyFromString(u'1601-01-02 00:00:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) def testCopyFromStringISO8601(self): """Tests the CopyFromStringISO8601 function.""" time_elements_object = time_elements.TimeElementsInMilliseconds() expected_time_elements_tuple = (2010, 8, 12, 0, 0, 0, 0) expected_number_of_seconds = 1281571200 time_elements_object.CopyFromStringISO8601(u'2010-08-12') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31, 0) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31, 0) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31Z') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31, 546) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31.546875') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31, 546) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31,546875') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 22, 6, 31, 546) expected_number_of_seconds = 1281650791 time_elements_object.CopyFromStringISO8601( u'2010-08-12T21:06:31.546875-01:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31, 546) expected_number_of_seconds = 1281643591 time_elements_object.CopyFromStringISO8601( u'2010-08-12T21:06:31.546875+01:00') self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual( time_elements_object._number_of_seconds, expected_number_of_seconds) with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(None) with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601( u'2010-08-12 21:06:31.546875+01:00') # Valid ISO 8601 notations currently not supported. with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'2016-W33') with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'2016-W33-3') with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'--08-17') with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'2016-230') with self.assertRaises(ValueError): time_elements_object.CopyFromStringISO8601(u'2010-08-12T21:06:31.5') def testCopyFromStringTuple(self): """Tests the CopyFromStringTuple function.""" time_elements_object = time_elements.TimeElementsInMilliseconds() expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31) time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'8', u'12', u'20', u'6', u'31', u'546')) self.assertIsNotNone(time_elements_object) self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual(time_elements_object._milliseconds, 546) with self.assertRaises(ValueError): time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'8', u'12', u'20', u'6', u'31')) with self.assertRaises(ValueError): time_elements_object.CopyFromStringTuple( time_elements_tuple=(u'2010', u'8', u'12', u'20', u'6', u'31', u'9S')) def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" time_elements_object = time_elements.TimeElementsInMilliseconds( time_elements_tuple=(2010, 8, 12, 20, 6, 31, 429)) expected_stat_time_tuple = (1281643591, 429000) stat_time_tuple = time_elements_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) time_elements_object = time_elements.TimeElementsInMilliseconds() expected_stat_time_tuple = (None, None) stat_time_tuple = time_elements_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" time_elements_object = time_elements.TimeElementsInMilliseconds( time_elements_tuple=(2010, 8, 12, 20, 6, 31, 429)) expected_micro_posix_number_of_seconds = 1281643591429000 micro_posix_number_of_seconds = time_elements_object.GetPlasoTimestamp() self.assertEqual( micro_posix_number_of_seconds, expected_micro_posix_number_of_seconds) time_elements_object = time_elements.TimeElementsInMilliseconds() micro_posix_number_of_seconds = time_elements_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_number_of_seconds) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/uuid_time.py000066400000000000000000000103601303277242400175630ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the UUID version 1 timestamp implementation.""" import uuid import unittest from dfdatetime import uuid_time class UUIDTimeTest(unittest.TestCase): """Tests for the UUID version 1 timestamp.""" def testInitialize(self): """Tests the initialization function.""" uuid_time_object = uuid_time.UUIDTime() self.assertIsNotNone(uuid_time_object) uuid_object = uuid.UUID(u'00911b54-9ef4-11e1-be53-525400123456') uuid_time_object = uuid_time.UUIDTime(timestamp=uuid_object.time) self.assertIsNotNone(uuid_time_object) expected_timestamp = 135564234616544084 self.assertEqual(uuid_time_object.timestamp, expected_timestamp) with self.assertRaises(ValueError): uuid_time.UUIDTime(timestamp=0x1fffffffffffffff) with self.assertRaises(ValueError): uuid_time.UUIDTime(timestamp=-1) def testCopyFromString(self): """Tests the CopyFromString function.""" uuid_time_object = uuid_time.UUIDTime() expected_timestamp = 135946080000000000 uuid_time_object.CopyFromString(u'2013-08-01') self.assertEqual(uuid_time_object.timestamp, expected_timestamp) expected_timestamp = 135946635280000000 uuid_time_object.CopyFromString(u'2013-08-01 15:25:28') self.assertEqual(uuid_time_object.timestamp, expected_timestamp) expected_timestamp = 135946635285468750 uuid_time_object.CopyFromString(u'2013-08-01 15:25:28.546875') self.assertEqual(uuid_time_object.timestamp, expected_timestamp) expected_timestamp = 135946671285468750 uuid_time_object.CopyFromString(u'2013-08-01 15:25:28.546875-01:00') self.assertEqual(uuid_time_object.timestamp, expected_timestamp) expected_timestamp = 135946599285468750 uuid_time_object.CopyFromString(u'2013-08-01 15:25:28.546875+01:00') self.assertEqual(uuid_time_object.timestamp, expected_timestamp) expected_timestamp = 864000000000 uuid_time_object.CopyFromString(u'1582-10-16 00:00:00') self.assertEqual(uuid_time_object.timestamp, expected_timestamp) with self.assertRaises(ValueError): uuid_time_object.CopyFromString(u'1570-01-02 00:00:00') def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" uuid_object = uuid.UUID(u'00911b54-9ef4-11e1-be53-525400123456') uuid_time_object = uuid_time.UUIDTime(timestamp=uuid_object.time) expected_stat_time_tuple = (1337130661, 6544084) stat_time_tuple = uuid_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) uuid_time_object = uuid_time.UUIDTime() uuid_time_object.timestamp = 0x1fffffffffffffff expected_stat_time_tuple = (None, None) stat_time_tuple = uuid_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) uuid_time_object = uuid_time.UUIDTime() uuid_time_object.timestamp = -1 expected_stat_time_tuple = (None, None) stat_time_tuple = uuid_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) uuid_time_object = uuid_time.UUIDTime() expected_stat_time_tuple = (None, None) stat_time_tuple = uuid_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" uuid_object = uuid.UUID(u'00911b54-9ef4-11e1-be53-525400123456') uuid_time_object = uuid_time.UUIDTime(timestamp=uuid_object.time) expected_micro_posix_timestamp = 1337130661654408 micro_posix_timestamp = uuid_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) uuid_time_object = uuid_time.UUIDTime() uuid_time_object.timestamp = 0x1fffffffffffffff micro_posix_timestamp = uuid_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) uuid_time_object = uuid_time.UUIDTime() uuid_time_object.timestamp = -1 micro_posix_timestamp = uuid_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) uuid_time_object = uuid_time.UUIDTime() micro_posix_timestamp = uuid_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tests/webkit_time.py000066400000000000000000000057231303277242400201110ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for the WebKit timestamp implementation.""" import unittest from dfdatetime import webkit_time class WebKitTimeTest(unittest.TestCase): """Tests for the WebKit timestamp.""" def testCopyFromString(self): """Tests the CopyFromString function.""" webkit_time_object = webkit_time.WebKitTime() expected_timestamp = 12926044800000000 webkit_time_object.CopyFromString(u'2010-08-12') self.assertEqual(webkit_time_object.timestamp, expected_timestamp) expected_timestamp = 12926120791000000 webkit_time_object.CopyFromString(u'2010-08-12 21:06:31') self.assertEqual(webkit_time_object.timestamp, expected_timestamp) expected_timestamp = 12926120791546875 webkit_time_object.CopyFromString(u'2010-08-12 21:06:31.546875') self.assertEqual(webkit_time_object.timestamp, expected_timestamp) expected_timestamp = 12926124391546875 webkit_time_object.CopyFromString(u'2010-08-12 21:06:31.546875-01:00') self.assertEqual(webkit_time_object.timestamp, expected_timestamp) expected_timestamp = 12926117191546875 webkit_time_object.CopyFromString(u'2010-08-12 21:06:31.546875+01:00') self.assertEqual(webkit_time_object.timestamp, expected_timestamp) expected_timestamp = 86400 * 1000000 webkit_time_object.CopyFromString(u'1601-01-02 00:00:00') self.assertEqual(webkit_time_object.timestamp, expected_timestamp) def testCopyToStatTimeTuple(self): """Tests the CopyToStatTimeTuple function.""" webkit_time_object = webkit_time.WebKitTime(timestamp=12926120791546875) expected_stat_time_tuple = (1281647191, 5468750) stat_time_tuple = webkit_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) webkit_time_object = webkit_time.WebKitTime(timestamp=0x1ffffffffffffffff) expected_stat_time_tuple = (None, None) stat_time_tuple = webkit_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) webkit_time_object = webkit_time.WebKitTime() expected_stat_time_tuple = (None, None) stat_time_tuple = webkit_time_object.CopyToStatTimeTuple() self.assertEqual(stat_time_tuple, expected_stat_time_tuple) def testGetPlasoTimestamp(self): """Tests the GetPlasoTimestamp function.""" webkit_time_object = webkit_time.WebKitTime(timestamp=12926120791546875) expected_micro_posix_timestamp = 1281647191546875 micro_posix_timestamp = webkit_time_object.GetPlasoTimestamp() self.assertEqual(micro_posix_timestamp, expected_micro_posix_timestamp) webkit_time_object = webkit_time.WebKitTime(timestamp=0x1ffffffffffffffff) micro_posix_timestamp = webkit_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) webkit_time_object = webkit_time.WebKitTime() micro_posix_timestamp = webkit_time_object.GetPlasoTimestamp() self.assertIsNone(micro_posix_timestamp) if __name__ == '__main__': unittest.main() dfdatetime-20170103/tox.ini000066400000000000000000000002221303277242400153720ustar00rootroot00000000000000[tox] envlist = py2, py3 [testenv] pip_pre = True setenv = PYTHONPATH = {toxinidir} deps = -rrequirements.txt commands = python run_tests.py dfdatetime-20170103/utils/000077500000000000000000000000001303277242400152235ustar00rootroot00000000000000dfdatetime-20170103/utils/__init__.py000066400000000000000000000000301303277242400173250ustar00rootroot00000000000000# -*- coding: utf-8 -*- dfdatetime-20170103/utils/pylintrc000066400000000000000000000177111303277242400170210ustar00rootroot00000000000000# Original file copied from: # http://src.chromium.org/chrome/trunk/tools/depot_tools/pylintrc [MASTER] # Specify a configuration file. #rcfile= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Add files or directories to the blacklist. They should be base names, not # paths. ignore=CVS # Pickle collected data for later comparisons. persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins= [MESSAGES CONTROL] # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time. #enable= # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). # CHANGED: # # C0103: Invalid name "" # C0302: Too many lines in module (N) # # I0010: Unable to consider inline option '' # I0011: Locally disabling WNNNN # # R0201: Method could be a function # R0801: Similar lines in N files # R0901: Too many ancestors (N/7) # R0902: Too many instance attributes (N/7) # R0903: Too few public methods (N/2) # R0904: Too many public methods (N/20) # R0911: Too many return statements (N/6) # R0912: Too many branches (N/12) # R0913: Too many arguments (N/5) # R0914: Too many local variables (N/15) # R0915: Too many statements (N/50) # R0921: Abstract class not referenced # R0922: Abstract class is only referenced 1 times # # W0141: Used builtin function '' # W0142: Used * or ** magic # W0402: Uses of a deprecated module 'string' # W0404: 41: Reimport 'XX' (imported line NN) # W0511: TODO # W1201: Specify string format arguments as logging function parameters # # Disabled: # locally-enabled # logging-format-interpolation # redefined-variable-type # simplifiable-if-statement # too-many-boolean-expressions (N/5) # too-many-nested-blocks (N/5) # ungrouped-imports disable=C0103,C0302,I0010,I0011,R0201,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0921,R0922,W0141,W0142,W0402,W0404,W0511,W1201,locally-enabled,logging-format-interpolation,redefined-variable-type,simplifiable-if-statement,too-many-boolean-expressions,too-many-nested-blocks,ungrouped-imports [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs # (visual studio) and html output-format=text # Put messages in a separate file for each module / package specified on the # command line instead of printing them on stdout. Reports (if any) will be # written in a file name "pylint_global.[txt|html]". files-output=no # Tells whether to display a full report or only the messages # CHANGED: reports=no # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which # respectively contain the number of errors / warnings messages and the total # number of statements analyzed. This is used by the global evaluation report # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) [VARIABLES] # Tells whether we should check for unused import in __init__ files. init-import=no # A regular expression matching the beginning of the name of unused variables. # By default this is _ and dummy but we prefer _ and unused. dummy-variables-rgx=_|unused # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. additional-builtins= [TYPECHECK] # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). ignored-classes=SQLObject,twisted.internet.reactor,hashlib,google.appengine.api.memcache # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E0201 when accessed. Python regular # expressions are accepted. generated-members=REQUEST,acl_users,aq_parent,multiprocessing.managers.SyncManager [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME,XXX,TODO [SIMILARITIES] # Minimum lines number of a similarity. min-similarity-lines=4 # Ignore comments when computing similarities. ignore-comments=yes # Ignore docstrings when computing similarities. ignore-docstrings=yes [FORMAT] # Maximum number of characters on a single line. max-line-length=80 # Maximum number of lines in a module max-module-lines=1000 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). # CHANGED: indent-string=' ' [BASIC] # List of builtins function names that should not be used, separated by a comma bad-functions=map,filter,apply,input # Regular expression which should only match correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # Regular expression which should only match correct module level names const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ # Regular expression which should only match correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ # Regular expression which should only match correct function names function-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct method names method-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct instance attribute names attr-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct argument names argument-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct variable names variable-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct list comprehension / # generator expression variable names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Good variable names which should always be accepted, separated by a comma good-names=i,j,k,ex,Run,_ # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata # Regular expression which should only match functions or classes name which do # not require a docstring no-docstring-rgx=__.*__ [DESIGN] # Maximum number of arguments for function / method max-args=5 # Argument names that match this expression will be ignored. Default to name # with leading underscore ignored-argument-names=_.* # Maximum number of locals for function / method body max-locals=15 # Maximum number of return / yield for function / method body max-returns=6 # Maximum number of branch for function / method body max-branchs=12 # Maximum number of statements in function / method body max-statements=50 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of attributes for a class (see R0902). max-attributes=7 # Minimum number of public methods for a class (see R0903). min-public-methods=2 # Maximum number of public methods for a class (see R0904). max-public-methods=20 [CLASSES] # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__,__new__,setUp # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls [IMPORTS] # Deprecated modules which should not be used, separated by a comma deprecated-modules=regsub,string,TERMIOS,Bastion,rexec # Create a graph of every (i.e. internal and external) dependencies in the # given file (report RP0402 must not be disabled) import-graph= # Create a graph of external dependencies in the given file (report RP0402 must # not be disabled) ext-import-graph= # Create a graph of internal dependencies in the given file (report RP0402 must # not be disabled) int-import-graph= [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" overgeneral-exceptions=Exception dfdatetime-20170103/utils/review.py000077500000000000000000001747641303277242400171240ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Script to manage code reviews.""" from __future__ import print_function import argparse import json import logging import os import re import shlex import subprocess import sys import time # pylint: disable=import-error # pylint: disable=no-name-in-module if sys.version_info[0] < 3: # Use urllib2 here since this code should be able to be used by a default # Python set up. Otherwise usage of requests is preferred. import urllib as urllib_parse import urllib2 as urllib_error import urllib2 as urllib_request else: import urllib.error as urllib_error import urllib.parse as urllib_parse import urllib.request as urllib_request # Change PYTHONPATH to include utils. sys.path.insert(0, u'.') import utils.upload class CLIHelper(object): """Class that defines CLI helper functions.""" def RunCommand(self, command): """Runs a command. Args: command (str): command to run. Returns: tuple[int,file,file]: exit code, stdout and stderr file-like objects. """ arguments = shlex.split(command) process = subprocess.Popen( arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE) if not process: logging.error(u'Running: "{0:s}" failed.'.format(command)) return 1, None, None output, error = process.communicate() if process.returncode != 0: logging.error(u'Running: "{0:s}" failed with error: {1:s}.'.format( command, error)) return process.returncode, output, error class CodeReviewHelper(CLIHelper): """Class that defines codereview helper functions.""" _REVIEWERS = frozenset([ u'jberggren@gmail.com', u'joachim.metz@gmail.com', u'onager@deerpie.com', u'romaing@google.com']) _REVIEWERS_CC = frozenset([ u'kiddi@kiddaland.net', u'log2timeline-dev@googlegroups.com']) def __init__(self, email_address, no_browser=False): """Initializes a codereview helper object. Args: email_address (str): email address. no_browser (Optional[bool]): True if the functionality to use the webbrowser to get the OAuth token should be disabled. """ super(CodeReviewHelper, self).__init__() self._access_token = None self._email_address = email_address self._no_browser = no_browser self._upload_py_path = os.path.join(u'utils', u'upload.py') self._xsrf_token = None def AddMergeMessage(self, issue_number, message): """Adds a merge message to the code review issue. Where the merge is a commit to the main project git repository. Args: issue_number (int|str): codereview issue number. message (str): message to add to the code review issue. Returns: bool: merge message was added to the code review issue. """ codereview_access_token = self.GetAccessToken() xsrf_token = self.GetXSRFToken() if not codereview_access_token or not xsrf_token: return False codereview_url = b'https://codereview.appspot.com/{0!s}/publish'.format( issue_number) post_data = urllib_parse.urlencode({ u'add_as_reviewer': u'False', u'message': message, u'message_only': u'True', u'no_redirect': 'True', u'send_mail': 'True', u'xsrf_token': xsrf_token}) request = urllib_request.Request(codereview_url) # Add header: Authorization: OAuth request.add_header( u'Authorization', u'OAuth {0:s}'.format(codereview_access_token)) # This will change the request into a POST. request.add_data(post_data) try: url_object = urllib_request.urlopen(request) except urllib_error.HTTPError as exception: logging.error( u'Failed publish to codereview issue: {0!s} with error: {1:s}'.format( issue_number, exception)) return False if url_object.code not in (200, 201): logging.error(( u'Failed publish to codereview issue: {0!s} with status code: ' u'{1:d}').format(issue_number, url_object.code)) return False return True def CloseIssue(self, issue_number): """Closes a code review issue. Args: issue_number (int|str): codereview issue number. Returns: bool: True if the code review was closed. """ codereview_access_token = self.GetAccessToken() xsrf_token = self.GetXSRFToken() if not codereview_access_token or not xsrf_token: return False codereview_url = b'https://codereview.appspot.com/{0!s}/close'.format( issue_number) post_data = urllib_parse.urlencode({ u'xsrf_token': xsrf_token}) request = urllib_request.Request(codereview_url) # Add header: Authorization: OAuth request.add_header( u'Authorization', u'OAuth {0:s}'.format(codereview_access_token)) # This will change the request into a POST. request.add_data(post_data) try: url_object = urllib_request.urlopen(request) except urllib_error.HTTPError as exception: logging.error( u'Failed closing codereview issue: {0!s} with error: {1:s}'.format( issue_number, exception)) return False if url_object.code != 200: logging.error(( u'Failed closing codereview issue: {0!s} with status code: ' u'{1:d}').format(issue_number, url_object.code)) return False return True def CreateIssue(self, diffbase, description): """Creates a new codereview issue. Args: diffbase (str): diffbase. description (str): description. Returns: int: codereview issue number or None. """ reviewers = list(self._REVIEWERS) reviewers_cc = list(self._REVIEWERS_CC) try: # Remove self from reviewers list. reviewers.remove(self._email_address) except ValueError: pass try: # Remove self from reviewers CC list. reviewers_cc.remove(self._email_address) except ValueError: pass reviewers = u','.join(reviewers) reviewers_cc = u','.join(reviewers_cc) command = u'{0:s} {1:s} --oauth2'.format( sys.executable, self._upload_py_path) if self._no_browser: command = u'{0:s} --no_oauth2_webbrowser'.format(command) command = ( u'{0:s} --send_mail -r {1:s} --cc {2:s} -t "{3:s}" -y -- ' u'{4:s}').format( command, reviewers, reviewers_cc, description, diffbase) if self._no_browser: print( u'Upload server: codereview.appspot.com (change with -s/--server)\n' u'Go to the following link in your browser:\n' u'\n' u' https://codereview.appspot.com/get-access-token\n' u'\n' u'and copy the access token.\n' u'\n') print(u'Enter access token:', end=u' ') sys.stdout.flush() exit_code, output, _ = self.RunCommand(command) print(output) if exit_code != 0: return issue_url_line_start = ( u'Issue created. URL: http://codereview.appspot.com/') for line in output.split(b'\n'): if issue_url_line_start in line: _, _, issue_number = line.rpartition(issue_url_line_start) try: return int(issue_number, 10) except ValueError: pass def GetAccessToken(self): """Retrieves the OAuth access token. Returns: str: codereview access token. """ if not self._access_token: # TODO: add support to get access token directly from user. self._access_token = utils.upload.GetAccessToken() if not self._access_token: logging.error(u'Unable to retrieve access token.') return self._access_token def GetXSRFToken(self): """Retrieves the XSRF token. Returns: str: codereview XSRF token or None if the token could not be obtained. """ if not self._xsrf_token: codereview_access_token = self.GetAccessToken() if not codereview_access_token: return codereview_url = b'https://codereview.appspot.com/xsrf_token' request = urllib_request.Request(codereview_url) # Add header: Authorization: OAuth request.add_header( u'Authorization', u'OAuth {0:s}'.format(codereview_access_token)) request.add_header(u'X-Requesting-XSRF-Token', u'1') try: url_object = urllib_request.urlopen(request) except urllib_error.HTTPError as exception: logging.error( u'Failed retrieving codereview XSRF token with error: {0:s}'.format( exception)) return if url_object.code != 200: logging.error(( u'Failed retrieving codereview XSRF token with status code: ' u'{0:d}').format(url_object.code)) return self._xsrf_token = url_object.read() return self._xsrf_token def QueryIssue(self, issue_number): """Queries the information of a code review issue. The query returns JSON data that contains: { "description":str, "cc":[str], "reviewers":[str], "owner_email":str, "private":bool, "base_url":str, "owner":str, "subject":str, "created":str, "patchsets":[int], "modified":str, "project":str, "closed":bool, "issue":int } Where the "created" and "modified" strings are formatted as: "YYYY-MM-DD hh:mm:ss.######" Args: issue_number (int|str): codereview issue number. Returns: dict[str,object]: JSON response or None. """ codereview_url = b'https://codereview.appspot.com/api/{0!s}'.format( issue_number) request = urllib_request.Request(codereview_url) try: url_object = urllib_request.urlopen(request) except urllib_error.HTTPError as exception: logging.error( u'Failed querying codereview issue: {0!s} with error: {1:s}'.format( issue_number, exception)) return if url_object.code != 200: logging.error(( u'Failed querying codereview issue: {0!s} with status code: ' u'{1:d}').format(issue_number, url_object.code)) return response_data = url_object.read() return json.loads(response_data) def UpdateIssue(self, issue_number, diffbase, description): """Updates a code review issue. Args: issue_number (int|str): codereview issue number. diffbase (str): diffbase. description (str): description. Returns: bool: True if the code review was updated. """ command = u'{0:s} {1:s} --oauth2'.format( sys.executable, self._upload_py_path) if self._no_browser: command = u'{0:s} --no_oauth2_webbrowser'.format(command) command = ( u'{0:s} -i {1!s} -m "Code updated." -t "{2:s}" -y -- ' u'{3:s}').format(command, issue_number, description, diffbase) if self._no_browser: print( u'Upload server: codereview.appspot.com (change with -s/--server)\n' u'Go to the following link in your browser:\n' u'\n' u' https://codereview.appspot.com/get-access-token\n' u'\n' u'and copy the access token.\n' u'\n') print(u'Enter access token:', end=u' ') sys.stdout.flush() exit_code, output, _ = self.RunCommand(command) print(output) return exit_code == 0 class GitHelper(CLIHelper): """Class that defines git helper functions.""" def __init__(self, git_repo_url): """Initializes a git helper object. Args: git_repo_url (str): git repo URL. """ super(GitHelper, self).__init__() self._git_repo_url = git_repo_url self._remotes = [] def _GetRemotes(self): """Retrieves the git repository remotes. Returns: list[str]: git repository remotes or None. """ if not self._remotes: exit_code, output, _ = self.RunCommand(u'git remote -v') if exit_code == 0: self._remotes = output.split(b'\n') return self._remotes def AddPath(self, path): """Adds a specific path to be managed by git. Args: path (str): path. Returns: bool: True if the path was added. """ command = u'git add -A {0:s}'.format(path) exit_code, _, _ = self.RunCommand(command) return exit_code == 0 def CheckHasBranch(self, branch): """Checks if the git repo has a specific branch. Args: branch (str): name of the feature branch. Returns: bool: True if git repo has the specific branch. """ exit_code, output, _ = self.RunCommand(u'git branch') if exit_code != 0: return False # Check for remote entries starting with upstream. for line in output.split(b'\n'): # Ignore the first 2 characters of the line. if line[2:] == branch: return True return False def CheckHasProjectOrigin(self): """Checks if the git repo has the project remote origin defined. Returns: bool: True if the git repo has the project origin defined. """ origin_git_repo_url = self.GetRemoteOrigin() return origin_git_repo_url == self._git_repo_url def CheckHasProjectUpstream(self): """Checks if the git repo has the project remote upstream defined. Returns: bool: True if the git repo has the project remote upstream defined. """ # Check for remote entries starting with upstream. for remote in self._GetRemotes(): if remote.startswith(b'upstream\t{0:s}'.format(self._git_repo_url)): return True return False def CheckHasUncommittedChanges(self): """Checks if the git repo has uncommitted changes. Returns: bool: True if the git repo has uncommitted changes. """ exit_code, output, _ = self.RunCommand(u'git status -s') if exit_code != 0: return False # Check if 'git status -s' yielded any output. for line in output.split(b'\n'): if line: return True return False def CheckSynchronizedWithUpstream(self): """Checks if the git repo is synchronized with upstream. Returns: bool: True if the git repo is synchronized with upstream. """ # Fetch the entire upstream repo information not only that of # the master branch. Otherwise the information about the current # upstream HEAD is not updated. exit_code, _, _ = self.RunCommand(u'git fetch upstream') if exit_code != 0: return False # The result of "git log HEAD..upstream/master --oneline" should be empty # if the git repo is synchronized with upstream. exit_code, output, _ = self.RunCommand( u'git log HEAD..upstream/master --oneline') return exit_code == 0 and not output def CommitToOriginInNameOf( self, codereview_issue_number, author, description): """Commits changes in name of an author to the master branch of origin. Args: codereview_issue_number (int|str): codereview issue number. author (str): full name and email address of the author, formatted as: "Full Name ". description (str): description of the commit. Returns: bool: True if the changes were committed to the git repository. """ command = ( u'git commit -a --author="{0:s}" ' u'-m "Code review: {1:s}: {2:s}"').format( author, codereview_issue_number, description) exit_code, _, _ = self.RunCommand(command) if exit_code != 0: return False exit_code, _, _ = self.RunCommand(u'git push origin master') if exit_code != 0: return False return True def DropUncommittedChanges(self): """Drops the uncommitted changes.""" self.RunCommand(u'git stash') self.RunCommand(u'git stash drop') def GetActiveBranch(self): """Retrieves the active branch. Returns: str: name of the active branch or None. """ exit_code, output, _ = self.RunCommand(u'git branch') if exit_code != 0: return False # Check for remote entries starting with upstream. for line in output.split(b'\n'): if line.startswith(b'* '): # Ignore the first 2 characters of the line. return line[2:] return def GetChangedFiles(self, diffbase=None): """Retrieves the changed files. Args: diffbase (Optional[str]): git diffbase, for example "upstream/master". Returns: list[str]: names of the changed files. """ if diffbase: command = u'git diff --name-only {0:s}'.format(diffbase) else: command = u'git ls-files' exit_code, output, _ = self.RunCommand(command) if exit_code != 0: return [] return output.split(b'\n') def GetChangedPythonFiles(self, diffbase=None): """Retrieves the changed Python files. Note that several Python files are excluded: * Python files generated by the protobuf compiler (*_pb2.py) * Python files used as test data (test_data/*.py) * Python files used for sphinx (docs/*.py) * setup.py and utils/upload.py Args: diffbase (Optional[str]): git diffbase, for example "upstream/master". Returns: list[str]: names of the changed Python files. """ upload_path = os.path.join(u'utils', u'upload.py') python_files = [] for changed_file in self.GetChangedFiles(diffbase=diffbase): if (not changed_file.endswith(u'.py') or changed_file.endswith(u'_pb2.py') or not os.path.exists(changed_file) or changed_file.startswith(u'docs') or changed_file.startswith(u'test_data') or changed_file in (u'setup.py', upload_path)): continue python_files.append(changed_file) return python_files def GetEmailAddress(self): """Retrieves the email address. Returns: str: email address or None. """ exit_code, output, _ = self.RunCommand(u'git config user.email') if exit_code != 0: return output_lines = output.split(b'\n') if not output_lines: return return output_lines[0] def GetLastCommitMessage(self): """Retrieves the last commit message. Returns: str: last commit message or None. """ exit_code, output, _ = self.RunCommand(u'git log -1') if exit_code != 0: return # Expecting 6 lines of output where the 5th line contains # the commit message. output_lines = output.split(b'\n') if len(output_lines) != 6: return return output_lines[4].strip() def GetRemoteOrigin(self): """Retrieves the remote origin. Returns: str: git repository URL or None. """ # Check for remote entries starting with origin. for remote in self._GetRemotes(): if remote.startswith(b'origin\t'): values = remote.split() if len(values) == 3: return values[1] def PullFromFork(self, git_repo_url, branch): """Pulls changes from a feature branch on a fork. Args: git_repo_url (str): git repository URL of the fork. branch (str): name of the feature branch of the fork. Returns: bool: True if the pull was successful. """ command = u'git pull --squash {0:s} {1:s}'.format(git_repo_url, branch) exit_code, _, _ = self.RunCommand(command) return exit_code == 0 def PushToOrigin(self, branch, force=False): """Forces a push of the active branch of the git repo to origin. Args: branch (str): name of the feature branch. force (Optional[bool]): True if the push should be forced. Returns: bool: True if the push was successful. """ if force: command = u'git push --set-upstream origin {0:s}'.format(branch) else: command = u'git push -f --set-upstream origin {0:s}'.format(branch) exit_code, _, _ = self.RunCommand(command) return exit_code == 0 def RemoveFeatureBranch(self, branch): """Removes the git feature branch both local and from origin. Args: branch (str): name of the feature branch. """ if branch == u'master': return self.RunCommand(u'git push origin --delete {0:s}'.format(branch)) self.RunCommand(u'git branch -D {0:s}'.format(branch)) def SynchronizeWithOrigin(self): """Synchronizes git with origin. Returns: bool: True if the git repository has synchronized with origin. """ exit_code, _, _ = self.RunCommand(u'git fetch origin') if exit_code != 0: return False exit_code, _, _ = self.RunCommand( u'git pull --no-edit origin master') return exit_code == 0 def SynchronizeWithUpstream(self): """Synchronizes git with upstream. Returns: bool: True if the git repository has synchronized with upstream. """ exit_code, _, _ = self.RunCommand(u'git fetch upstream') if exit_code != 0: return False exit_code, _, _ = self.RunCommand( u'git pull --no-edit --rebase upstream master') if exit_code != 0: return False exit_code, _, _ = self.RunCommand(u'git push') return exit_code == 0 def SwitchToMasterBranch(self): """Switches git to the master branch. Returns: bool: True if the git repository has switched to the master branch. """ exit_code, _, _ = self.RunCommand(u'git checkout master') return exit_code != 0 class GitHubHelper(object): """Class that defines github helper functions.""" def __init__(self, organization, project): """Initializes a github helper object. Args: organization (str): github organization name. project (str): github project name. """ super(GitHubHelper, self).__init__() self._organization = organization self._project = project def CreatePullRequest( self, access_token, codereview_issue_number, origin, description): """Creates a pull request. Args: access_token (str): github access token. codereview_issue_number (int|str): codereview issue number. origin (str): origin of the pull request, formatted as: "username:feature". description (str): description. Returns: bool: True if the pull request was created. """ title = b'{0!s}: {1:s}'.format(codereview_issue_number, description) body = ( b'[Code review: {0!s}: {1:s}]' b'(https://codereview.appspot.com/{0!s}/)').format( codereview_issue_number, description) post_data = ( b'{{\n' b' "title": "{0:s}",\n' b' "body": "{1:s}",\n' b' "head": "{2:s}",\n' b' "base": "master"\n' b'}}\n').format(title, body, origin) github_url = ( u'https://api.github.com/repos/{0:s}/{1:s}/pulls?' u'access_token={2:s}').format( self._organization, self._project, access_token) request = urllib_request.Request(github_url) # This will change the request into a POST. request.add_data(post_data) try: url_object = urllib_request.urlopen(request) except urllib_error.HTTPError as exception: logging.error( u'Failed creating pull request: {0!s} with error: {1:s}'.format( codereview_issue_number, exception)) return False if url_object.code not in (200, 201): logging.error( u'Failed creating pull request: {0!s} with status code: {1:d}'.format( codereview_issue_number, url_object.code)) return False return True def GetForkGitRepoUrl(self, username): """Retrieves the git repository URL of a fork. Args: username (str): github username of the fork. Returns: str: git repository URL or None. """ return u'https://github.com/{0:s}/{1:s}.git'.format(username, self._project) def QueryUser(self, username): """Queries a github user. Args: username (str): github user name. Returns: dict[str,object]: JSON response or None. """ github_url = b'https://api.github.com/users/{0:s}'.format(username) request = urllib_request.Request(github_url) try: url_object = urllib_request.urlopen(request) except urllib_error.HTTPError as exception: logging.error( u'Failed querying github user: {0:s} with error: {1:s}'.format( username, exception)) return if url_object.code != 200: logging.error( u'Failed querying github user: {0:d} with status code: {1:d}'.format( username, url_object.code)) return response_data = url_object.read() return json.loads(response_data) class ProjectHelper(CLIHelper): """Class that defines project helper functions. Attributes: project_name (str): name of the project. """ _AUTHORS_FILE_HEADER = [ u'# Names should be added to this file with this pattern:', u'#', u'# For individuals:', u'# Name (email address)', u'#', u'# For organizations:', u'# Organization (fnmatch pattern)', u'#', u'# See python fnmatch module documentation for more information.', u'', u'Google Inc. (*@google.com)'] SUPPORTED_PROJECTS = frozenset([ u'dfdatetime', u'dfvfs', u'dfwinreg', u'l2tdevtools', u'l2tdocs', u'plaso']) def __init__(self, script_path): """Initializes a project helper object. Args: script_path (str): path to the script. Raises: ValueError: if the project name is not supported. """ super(ProjectHelper, self).__init__() self.project_name = self._GetProjectName(script_path) @property def version_file_path(self): """str: path of the version file.""" return os.path.join(self.project_name, u'__init__.py') def _GetProjectName(self, script_path): """Retrieves the project name from the script path. Args: script_path (str): path to the script. Returns: str: project name. Raises: ValueError: if the project name is not supported. """ project_name = os.path.abspath(script_path) project_name = os.path.dirname(project_name) project_name = os.path.dirname(project_name) project_name = os.path.basename(project_name) for supported_project_name in self.SUPPORTED_PROJECTS: if supported_project_name in project_name: return supported_project_name raise ValueError( u'Unsupported project name: {0:s}.'.format(project_name)) def _ReadFileContents(self, path): """Reads the contents of a file. Args: filename (str): path of the file. Returns: bytes: file content or None. """ if not os.path.exists(path): logging.error(u'Missing file: {0:s}'.format(path)) return try: with open(path, u'rb') as file_object: file_contents = file_object.read() except IOError as exception: logging.error(u'Unable to read file with error: {0:s}'.format(exception)) return try: file_contents = file_contents.decode(u'utf-8') except UnicodeDecodeError as exception: logging.error( u'Unable to read file with error: {0:s}'.format(exception)) return return file_contents def GetVersion(self): """Retrieves the project version from the version file. Returns: str: project version or None. """ version_file_contents = self._ReadFileContents(self.version_file_path) if not version_file_contents: return # The version is formatted as: # __version__ = 'VERSION' version_line_prefix = u'__version__ = \'' lines = version_file_contents.split(u'\n') for line in lines: if line.startswith(version_line_prefix): return line[len(version_line_prefix):-1] return def UpdateDpkgChangelogFile(self): """Updates the dpkg changelog file. Returns: bool: True if the dpkg changelog file was updated or if the dpkg changelog file does not exists. """ project_version = self.GetVersion() dpkg_changelog_path = os.path.join(u'config', u'dpkg', u'changelog') if not os.path.exists(dpkg_changelog_path): return True dpkg_maintainter = u'Log2Timeline ' dpkg_date = time.strftime(u'%a, %d %b %Y %H:%M:%S %z') dpkg_changelog_content = u'\n'.join([ u'{0:s} ({1:s}-1) unstable; urgency=low'.format( self.project_name, project_version), u'', u' * Auto-generated', u'', u' -- {0:s} {1:s}'.format(dpkg_maintainter, dpkg_date)]) try: dpkg_changelog_content = dpkg_changelog_content.encode(u'utf-8') except UnicodeEncodeError as exception: logging.error( u'Unable to write dpkg changelog file with error: {0:s}'.format( exception)) return False try: with open(dpkg_changelog_path, u'wb') as file_object: file_object.write(dpkg_changelog_content) except IOError as exception: logging.error( u'Unable to write dpkg changelog file with error: {0:s}'.format( exception)) return False return True def UpdateAuthorsFile(self): """Updates the AUTHORS file. Returns: bool: True if the AUTHORS file update was successful. """ exit_code, output, _ = self.RunCommand(u'git log --format="%aN (%aE)"') if exit_code != 0: return False lines = output.split(b'\n') # Reverse the lines since we want the oldest commits first. lines.reverse() authors_by_commit = [] authors = {} for author in lines: name, _, email_address = author[:-1].rpartition(u'(') if email_address in authors: if name != authors[email_address]: logging.warning(u'Detected name mismatch for author: {0:d}.'.format( email_address)) continue authors[email_address] = name authors_by_commit.append(author) file_content = [] file_content.extend(self._AUTHORS_FILE_HEADER) file_content.extend(authors_by_commit) file_content = u'\n'.join(file_content) file_content = file_content.encode(u'utf-8') with open(u'AUTHORS', 'wb') as file_object: file_object.write(file_content) return True def UpdateVersionFile(self): """Updates the version file. Returns: bool: True if the file was updated. """ version_file_contents = self._ReadFileContents(self.version_file_path) if not version_file_contents: logging.error(u'Unable to read version file.') return False date_version = time.strftime(u'%Y%m%d') lines = version_file_contents.split(u'\n') for line_index, line in enumerate(lines): if (self.project_name == u'plaso' and line.startswith(u'VERSION_DATE = ')): version_string = u'VERSION_DATE = \'{0:s}\''.format(date_version) lines[line_index] = version_string elif (self.project_name != u'plaso' and line.startswith(u'__version__ = ')): version_string = u'__version__ = \'{0:s}\''.format(date_version) lines[line_index] = version_string version_file_contents = u'\n'.join(lines) try: version_file_contents = version_file_contents.encode(u'utf-8') except UnicodeEncodeError as exception: logging.error( u'Unable to write version file with error: {0:s}'.format(exception)) return False try: with open(self.version_file_path, u'wb') as file_object: file_object.write(version_file_contents) except IOError as exception: logging.error( u'Unable to write version file with error: {0:s}'.format(exception)) return False return True class PylintHelper(CLIHelper): """Class that defines pylint helper functions.""" _MINIMUM_VERSION_TUPLE = (1, 5, 0) def CheckFiles(self, filenames): """Checks if the linting of the files is correct using pylint. Args: filenames (list[str]): names of the files to lint. Returns: bool: True if the files were linted without errors. """ print(u'Running linter on changed files.') failed_filenames = [] for filename in filenames: print(u'Checking: {0:s}'.format(filename)) command = u'pylint --rcfile=utils/pylintrc {0:s}'.format(filename) exit_code = subprocess.call(command, shell=True) if exit_code != 0: failed_filenames.append(filename) if failed_filenames: print(u'\nFiles with linter errors:\n{0:s}\n'.format( u'\n'.join(failed_filenames))) return False return True def CheckUpToDateVersion(self): """Checks if the pylint version is up to date. Returns: bool: True if the pylint version is up to date. """ exit_code, output, _ = self.RunCommand(u'pylint --version') if exit_code != 0: return False version_tuple = (0, 0, 0) for line in output.split(b'\n'): if line.startswith(b'pylint '): _, _, version = line.partition(b' ') # Remove a trailing comma. version, _, _ = version.partition(b',') version_tuple = tuple([int(digit) for digit in version.split(b'.')]) return version_tuple >= self._MINIMUM_VERSION_TUPLE class ReadTheDocsHelper(object): """Class that defines readthedocs helper functions.""" def __init__(self, project): """Initializes a readthedocs helper object. Args: project (str): github project name. """ super(ReadTheDocsHelper, self).__init__() self._project = project def TriggerBuild(self): """Triggers readthedocs to build the docs of the project. Returns: bool: True if the build was triggered. """ readthedocs_url = u'https://readthedocs.org/build/{0:s}'.format( self._project) request = urllib_request.Request(readthedocs_url) # This will change the request into a POST. request.add_data(b'') try: url_object = urllib_request.urlopen(request) except urllib_error.HTTPError as exception: logging.error( u'Failed triggering build with error: {0:s}'.format( exception)) return False if url_object.code != 200: logging.error( u'Failed triggering build with status code: {1:d}'.format( url_object.code)) return False return True class SphinxAPIDocHelper(CLIHelper): """Class that defines sphinx-apidoc helper functions.""" _MINIMUM_VERSION_TUPLE = (1, 2, 0) def __init__(self, project): """Initializes a sphinx-apidoc helper object. Args: project (str): github project name. """ super(SphinxAPIDocHelper, self).__init__() self._project = project def CheckUpToDateVersion(self): """Checks if the sphinx-apidoc version is up to date. Returns: bool: True if the sphinx-apidoc version is up to date. """ exit_code, output, _ = self.RunCommand(u'sphinx-apidoc --version') if exit_code != 0: return False version_tuple = (0, 0, 0) for line in output.split(b'\n'): if line.startswith(b'Sphinx (sphinx-apidoc) '): _, _, version = line.rpartition(b' ') version_tuple = tuple([int(digit) for digit in version.split(b'.')]) return version_tuple >= self._MINIMUM_VERSION_TUPLE def UpdateAPIDocs(self): """Updates the API docs. Returns: bool: True if the API docs have been updated. """ command = u'sphinx-apidoc -f -o docs {0:s}'.format(self._project) exit_code, output, _ = self.RunCommand(command) print(output) return exit_code == 0 class NetRCFile(object): """Class that defines a .netrc file.""" _NETRC_SEPARATOR_RE = re.compile(r'[^ \t\n]+') def __init__(self): """Initializes a .netrc file object.""" super(NetRCFile, self).__init__() self._contents = None self._values = None home_path = os.path.expanduser(u'~') self._path = os.path.join(home_path, u'.netrc') if not os.path.exists(self._path): return with open(self._path, 'r') as file_object: self._contents = file_object.read() def _GetGitHubValues(self): """Retrieves the github values. Returns: list[str]: .netrc values for github.com or None. """ if not self._contents: return # Note that according to GNU's manual on .netrc file, the credential # tokens "may be separated by spaces, tabs, or new-lines". if not self._values: self._values = self._NETRC_SEPARATOR_RE.findall(self._contents) for value_index, value in enumerate(self._values): if value == u'github.com' and self._values[value_index - 1] == u'machine': return self._values[value_index + 1:] def GetGitHubAccessToken(self): """Retrieves the github access token. Returns: str: github access token or None. """ values = self._GetGitHubValues() if not values: return for value_index, value in enumerate(values): if value == u'password': return values[value_index + 1] def GetGitHubUsername(self): """Retrieves the github username. Returns: str: github username or None. """ values = self._GetGitHubValues() if not values: return login_value = None for value_index, value in enumerate(values): if value == u'login': login_value = values[value_index + 1] # If the next field is 'password' we assume the login field is empty. if login_value != u'password': return login_value class ReviewFile(object): """Class that defines a review file. A review file is use to track code review relevant information like the codereview issue number. It is stored in the .review subdirectory and named after the feature branch e.g. ".review/feature". """ def __init__(self, branch_name): """Initializes a review file object. Args: branch_name (str): name of the feature branch of the review. """ super(ReviewFile, self).__init__() self._contents = None self._path = os.path.join(u'.review', branch_name) if os.path.exists(self._path): with open(self._path, 'r') as file_object: self._contents = file_object.read() def Create(self, codereview_issue_number): """Creates a new review file. If the .review directory does not exist, it will be created. Args: codereview_issue_number (int|str): codereview issue number. Returns: bool: True if the review file was created. """ if not os.path.exists(u'.review'): os.mkdir(u'.review') with open(self._path, 'w') as file_object: file_object.write(u'{0!s}'.format(codereview_issue_number)) def Exists(self): """Determines if the review file exists. Returns: bool: True if review file exists. """ return os.path.exists(self._path) def GetCodeReviewIssueNumber(self): """Retrieves the codereview issue number. Returns: int: codereview issue number. """ if not self._contents: return try: return int(self._contents, 10) except ValueError: pass def Remove(self): """Removes the review file.""" if not os.path.exists(self._path): return os.remove(self._path) class ReviewHelper(object): """Class that defines review helper functions.""" _PROJECT_NAME_PREFIX_REGEX = re.compile( r'\[({0:s})\] '.format(u'|'.join(ProjectHelper.SUPPORTED_PROJECTS))) def __init__( self, command, github_origin, feature_branch, diffbase, all_files=False, no_browser=False, no_confirm=False): """Initializes a review helper object. Args: command (str): user provided command, for example "create", "lint". github_origin (str): github origin. feature_branch (str): feature branch. diffbase (str): diffbase. all_files (Optional[bool]): True if the command should apply to all files. Currently this only affects the lint command. no_browser (Optional[bool]): True if the functionality to use the webbrowser to get the OAuth token should be disabled. no_confirm (Optional[bool]): True if the defaults should be applied without confirmation. """ super(ReviewHelper, self).__init__() self._active_branch = None self._all_files = all_files self._codereview_helper = None self._command = command self._diffbase = diffbase self._feature_branch = feature_branch self._git_helper = None self._git_repo_url = None self._github_helper = None self._github_origin = github_origin self._fork_feature_branch = None self._fork_username = None self._merge_author = None self._merge_description = None self._no_browser = no_browser self._no_confirm = no_confirm self._project_helper = None self._project_name = None self._sphinxapidoc_helper = None def CheckLocalGitState(self): """Checks the state of the local git repository. Returns: bool: True if the state of the local git repository is sane. """ if self._command in (u'close', u'create', u'lint', u'update'): if not self._git_helper.CheckHasProjectUpstream(): print(u'{0:s} aborted - missing project upstream.'.format( self._command.title())) print(u'Run: git remote add upstream {0:s}'.format(self._git_repo_url)) return False elif self._command == u'merge': if not self._git_helper.CheckHasProjectOrigin(): print(u'{0:s} aborted - missing project origin.'.format( self._command.title())) return False if self._command not in ( u'lint', u'test', u'update-version', u'update_version'): if self._git_helper.CheckHasUncommittedChanges(): print(u'{0:s} aborted - detected uncommitted changes.'.format( self._command.title())) print(u'Run: git commit') return False self._active_branch = self._git_helper.GetActiveBranch() if self._command in (u'create', u'update'): if self._active_branch == u'master': print(u'{0:s} aborted - active branch is master.'.format( self._command.title())) return False elif self._command == u'close': if self._feature_branch == u'master': print(u'{0:s} aborted - feature branch cannot be master.'.format( self._command.title())) return False if self._active_branch != u'master': self._git_helper.SwitchToMasterBranch() self._active_branch = u'master' return True def CheckRemoteGitState(self): """Checks the state of the remote git repository. Returns: bool: True if the state of the remote git repository is sane. """ if self._command == u'close': if not self._git_helper.SynchronizeWithUpstream(): print(( u'{0:s} aborted - unable to synchronize with ' u'upstream/master.').format(self._command.title())) return False elif self._command in (u'create', u'update'): if not self._git_helper.CheckSynchronizedWithUpstream(): if not self._git_helper.SynchronizeWithUpstream(): print(( u'{0:s} aborted - unable to synchronize with ' u'upstream/master.').format(self._command.title())) return False force_push = True else: force_push = False if not self._git_helper.PushToOrigin( self._active_branch, force=force_push): print(u'{0:s} aborted - unable to push updates to origin/{1:s}.'.format( self._command.title(), self._active_branch)) return False elif self._command == u'lint': self._git_helper.CheckSynchronizedWithUpstream() elif self._command == u'merge': if not self._git_helper.SynchronizeWithOrigin(): print(( u'{0:s} aborted - unable to synchronize with ' u'origin/master.').format(self._command.title())) return False return True def Close(self): """Closes a review. Returns: bool: True if the close was successful. """ review_file = ReviewFile(self._feature_branch) if not review_file.Exists(): print(u'Review file missing for branch: {0:s}'.format( self._feature_branch)) return False if not self._git_helper.CheckHasBranch(self._feature_branch): print(u'No such feature branch: {0:s}'.format(self._feature_branch)) else: self._git_helper.RemoveFeatureBranch(self._feature_branch) codereview_issue_number = review_file.GetCodeReviewIssueNumber() review_file.Remove() if codereview_issue_number: if not self._codereview_helper.CloseIssue(codereview_issue_number): print(u'Unable to close code review: {0!s}'.format( codereview_issue_number)) print(( u'Close it manually on: https://codereview.appspot.com/' u'{0!s}').format(codereview_issue_number)) return True def Create(self): """Creates a review. Returns: bool: True if the create was successful. """ review_file = ReviewFile(self._active_branch) if review_file.Exists(): print(u'Review file already exists for branch: {0:s}'.format( self._active_branch)) return False git_origin = self._git_helper.GetRemoteOrigin() if not git_origin.startswith(u'https://github.com/'): print(u'{0:s} aborted - unsupported git remote origin: {1:s}'.format( self._command.title(), git_origin)) print(u'Make sure the git remote origin is hosted on github.com') return False git_origin, _, _ = git_origin[len(u'https://github.com/'):].rpartition(u'/') netrc_file = NetRCFile() github_access_token = netrc_file.GetGitHubAccessToken() if not github_access_token: print(u'{0:s} aborted - unable to determine github access token.'.format( self._command.title())) print(u'Make sure .netrc is configured with a github access token.') return False last_commit_message = self._git_helper.GetLastCommitMessage() print(u'Automatic generated description of code review:') print(last_commit_message) print(u'') if self._no_confirm: user_input = None else: print(u'Enter a description for the code review or hit enter to use the') print(u'automatic generated one:') user_input = sys.stdin.readline() user_input = user_input.strip() if not user_input: description = last_commit_message else: description = user_input # Prefix the description with the project name for code review to make it # easier to distinguish between projects. code_review_description = u'[{0:s}] {1:s}'.format( self._project_name, description) codereview_issue_number = self._codereview_helper.CreateIssue( self._diffbase, code_review_description) if not codereview_issue_number: print(u'{0:s} aborted - unable to create codereview issue.'.format( self._command.title())) return False if not os.path.isdir(u'.review'): os.mkdir(u'.review') review_file.Create(codereview_issue_number) create_github_origin = u'{0:s}:{1:s}'.format( git_origin, self._active_branch) if not self._github_helper.CreatePullRequest( github_access_token, codereview_issue_number, create_github_origin, description): print(u'Unable to create pull request.') return True def InitializeHelpers(self): """Initializes the helper objects. Returns: bool: True if the helper initialization was successful. """ script_path = os.path.abspath(__file__) self._project_helper = ProjectHelper(script_path) self._project_name = self._project_helper.project_name if not self._project_name: print(u'{0:s} aborted - unable to determine project name.'.format( self._command.title())) return False self._git_repo_url = b'https://github.com/log2timeline/{0:s}.git'.format( self._project_name) self._git_helper = GitHelper(self._git_repo_url) self._github_helper = GitHubHelper(u'log2timeline', self._project_name) if self._command in (u'close', u'create', u'merge', u'update'): email_address = self._git_helper.GetEmailAddress() self._codereview_helper = CodeReviewHelper( email_address, no_browser=self._no_browser) if self._command == u'merge': self._sphinxapidoc_helper = SphinxAPIDocHelper( self._project_name) # TODO: disable the version check for now since sphinx-apidoc 1.2.2 # on Unbuntu 14.04 does not have the --version option. Re-enable when # sphinx-apidoc 1.2.3 or later is introduced. # if not self._sphinxapidoc_helper.CheckUpToDateVersion(): # print(( # u'{0:s} aborted - sphinx-apidoc verion 1.2.0 or later ' # u'required.').format(self._command.title())) # return False return True def Lint(self): """Lints a review. Returns: bool: True if linting was successful. """ if self._project_name == u'l2tdocs': return True if self._command not in (u'create', u'merge', u'lint', u'update'): return True pylint_helper = PylintHelper() if not pylint_helper.CheckUpToDateVersion(): print(u'{0:s} aborted - pylint verion 1.5.0 or later required.'.format( self._command.title())) return False if self._command == u'merge': fork_git_repo_url = self._github_helper.GetForkGitRepoUrl( self._fork_username) if not self._git_helper.PullFromFork( fork_git_repo_url, self._fork_feature_branch): print(u'{0:s} aborted - unable to pull changes from fork.'.format( self._command.title())) return False if self._all_files: diffbase = None elif self._command == u'merge': diffbase = u'origin/master' else: diffbase = self._diffbase changed_python_files = self._git_helper.GetChangedPythonFiles( diffbase=diffbase) if not pylint_helper.CheckFiles(changed_python_files): print(u'{0:s} aborted - unable to pass linter.'.format( self._command.title())) if self._command == u'merge': self._git_helper.DropUncommittedChanges() return False return True def Merge(self, codereview_issue_number): """Merges a review. Args: codereview_issue_number (int|str): codereview issue number. Returns: bool: True if the merge was successful. """ if not self._project_helper.UpdateVersionFile(): print(u'Unable to update version file.') self._git_helper.DropUncommittedChanges() return False if not self._project_helper.UpdateDpkgChangelogFile(): print(u'Unable to update dpkg changelog file.') self._git_helper.DropUncommittedChanges() return False apidoc_config_path = os.path.join(u'docs', u'conf.py') if os.path.exists(apidoc_config_path): self._sphinxapidoc_helper.UpdateAPIDocs() self._git_helper.AddPath(u'docs') readthedocs_helper = ReadTheDocsHelper(self._project_name) # The project wiki repo contains the documentation and # has no trigger on update webhook for readthedocs. # So we trigger readthedocs directly to build the docs. readthedocs_helper.TriggerBuild() if not self._git_helper.CommitToOriginInNameOf( codereview_issue_number, self._merge_author, self._merge_description): print(u'Unable to commit changes.') self._git_helper.DropUncommittedChanges() return False commit_message = ( u'Changes have been merged with master branch. ' u'To close the review and clean up the feature branch you can run: ' u'python ./utils/review.py close {0:s}').format( self._fork_feature_branch) self._codereview_helper.AddMergeMessage( codereview_issue_number, commit_message) return True def Open(self, codereview_issue_number): """Opens a review. Args: codereview_issue_number (int|str): codereview issue number. Returns: bool: True if the open was successful. """ # TODO: implement. # * check if feature branch exists # * check if review file exists # * check if issue number corresponds to branch by checking PR? # * create feature branch and pull changes from origin # * create review file _ = codereview_issue_number return False def PrepareMerge(self, codereview_issue_number): """Prepares a merge. Args: codereview_issue_number (int|str): codereview issue number. Returns: bool: True if the prepare were successful. """ codereview_information = self._codereview_helper.QueryIssue( codereview_issue_number) if not codereview_information: print(( u'{0:s} aborted - unable to retrieve code review: {1!s} ' u'information.').format( self._command.title(), codereview_issue_number)) return False self._merge_description = codereview_information.get(u'subject', None) if not self._merge_description: print(( u'{0:s} aborted - unable to determine description of code review: ' u'{1!s}.').format( self._command.title(), codereview_issue_number)) return False # When merging remove the project name ("[project]") prefix from # the code review description. self._merge_description = self._PROJECT_NAME_PREFIX_REGEX.sub( u'', self._merge_description) merge_email_address = codereview_information.get(u'owner_email', None) if not merge_email_address: print(( u'{0:s} aborted - unable to determine email address of owner of ' u'code review: {1!s}.').format( self._command.title(), codereview_issue_number)) return False self._fork_username, _, self._fork_feature_branch = ( self._github_origin.partition(u':')) github_user_information = self._github_helper.QueryUser( self._fork_username) if not github_user_information: print(( u'{0:s} aborted - unable to retrieve github user: {1:s} ' u'information.').format( self._command.title(), self._fork_username)) return False merge_fullname = github_user_information.get(u'name', None) if not merge_fullname: merge_fullname = codereview_information.get(u'owner', None) if not merge_fullname: merge_fullname = github_user_information.get(u'company', None) if not merge_fullname: print(( u'{0:s} aborted - unable to determine full name.').format( self._command.title())) return False self._merge_author = u'{0:s} <{1:s}>'.format( merge_fullname, merge_email_address) return True def Test(self): """Tests a review. Returns: bool: True if the tests were successful. """ if self._project_name == u'l2tdocs': return True if self._command not in (u'create', u'merge', u'test', u'update'): return True # TODO: determine why this alters the behavior of argparse. # Currently affects this script being used in plaso. command = u'{0:s} run_tests.py --fail-unless-has-test-file'.format( sys.executable) exit_code = subprocess.call(command, shell=True) if exit_code != 0: print(u'{0:s} aborted - unable to pass tests.'.format( self._command.title())) if self._command == u'merge': self._git_helper.DropUncommittedChanges() return False return True def Update(self): """Updates a review. Returns: bool: True if the update was successful. """ review_file = ReviewFile(self._active_branch) if not review_file.Exists(): print(u'Review file missing for branch: {0:s}'.format( self._active_branch)) return False codereview_issue_number = review_file.GetCodeReviewIssueNumber() last_commit_message = self._git_helper.GetLastCommitMessage() print(u'Automatic generated description of the update:') print(last_commit_message) print(u'') if self._no_confirm: user_input = None else: print(u'Enter a description for the update or hit enter to use the') print(u'automatic generated one:') user_input = sys.stdin.readline() user_input = user_input.strip() if not user_input: description = last_commit_message else: description = user_input if not self._codereview_helper.UpdateIssue( codereview_issue_number, self._diffbase, description): print(u'Unable to update code review: {0!s}'.format( codereview_issue_number)) return False return True def UpdateAuthors(self): """Updates the authors. Returns: bool: True if the authors update was successful. """ if self._project_name == u'l2tdocs': return True if not self._project_helper.UpdateAuthorsFile(): print(u'Unable to update authors file.') return False return True def UpdateVersion(self): """Updates the version. Returns: bool: True if the version update was successful. """ if self._project_name == u'l2tdocs': return True if not self._project_helper.UpdateVersionFile(): print(u'Unable to update version file.') return False if not self._project_helper.UpdateDpkgChangelogFile(): print(u'Unable to update dpkg changelog file.') return False return True def Main(): """The main program function. Returns: bool: True if successful or False if not. """ argument_parser = argparse.ArgumentParser( description=u'Script to manage code reviews.') # TODO: add option to directly pass code review issue number. argument_parser.add_argument( u'--allfiles', u'--all-files', u'--all_files', dest=u'all_files', action=u'store_true', default=False, help=( u'Apply command to all files, currently only affects the lint ' u'command.')) argument_parser.add_argument( u'--diffbase', dest=u'diffbase', action=u'store', type=str, metavar=u'DIFFBASE', default=u'upstream/master', help=( u'The diffbase the default is upstream/master. This options is used ' u'to indicate to what "base" the code changes are relative to and ' u'can be used to "chain" code reviews.')) argument_parser.add_argument( u'--nobrowser', u'--no-browser', u'--no_browser', dest=u'no_browser', action=u'store_true', default=False, help=( u'Disable the functionality to use the webbrowser to get the OAuth ' u'token should be disabled.')) argument_parser.add_argument( u'--noconfirm', u'--no-confirm', u'--no_confirm', dest=u'no_confirm', action=u'store_true', default=False, help=( u'Do not ask for confirmation apply defaults.\n' u'WARNING: only use this when you are familiar with the defaults.')) argument_parser.add_argument( u'--offline', dest=u'offline', action=u'store_true', default=False, help=( u'The review script is running offline and any online check is ' u'skipped.')) commands_parser = argument_parser.add_subparsers(dest=u'command') close_command_parser = commands_parser.add_parser(u'close') # TODO: add this to help output. close_command_parser.add_argument( u'branch', action=u'store', metavar=u'BRANCH', default=None, help=u'name of the corresponding feature branch.') commands_parser.add_parser(u'create') merge_command_parser = commands_parser.add_parser(u'merge') # TODO: add this to help output. merge_command_parser.add_argument( u'codereview_issue_number', action=u'store', metavar=u'CODEREVIEW_ISSUE_NUMBER', default=None, help=u'the codereview issue number to be merged.') # TODO: add this to help output. merge_command_parser.add_argument( u'github_origin', action=u'store', metavar=u'GITHUB_ORIGIN', default=None, help=u'the github origin to merged e.g. username:feature.') commands_parser.add_parser(u'lint') open_command_parser = commands_parser.add_parser(u'open') # TODO: add this to help output. open_command_parser.add_argument( u'codereview_issue_number', action=u'store', metavar=u'CODEREVIEW_ISSUE_NUMBER', default=None, help=u'the codereview issue number to be opened.') # TODO: add this to help output. open_command_parser.add_argument( u'branch', action=u'store', metavar=u'BRANCH', default=None, help=u'name of the corresponding feature branch.') # TODO: add submit option? commands_parser.add_parser(u'test') # TODO: add dry-run option to run merge without commit. # useful to test pending CLs. commands_parser.add_parser(u'update') commands_parser.add_parser(u'update-authors') commands_parser.add_parser(u'update_authors') commands_parser.add_parser(u'update-version') commands_parser.add_parser(u'update_version') options = argument_parser.parse_args() codereview_issue_number = None feature_branch = None github_origin = None print_help_on_error = False if options.command in (u'close', u'open'): feature_branch = getattr(options, u'branch', None) if not feature_branch: print(u'Feature branch value is missing.') print_help_on_error = True # Support "username:branch" notation. if u':' in feature_branch: _, _, feature_branch = feature_branch.rpartition(u':') if options.command in (u'merge', u'open'): codereview_issue_number = getattr( options, u'codereview_issue_number', None) if not codereview_issue_number: print(u'Codereview issue number value is missing.') print_help_on_error = True if options.command == u'merge': github_origin = getattr(options, u'github_origin', None) if not github_origin: print(u'Github origin value is missing.') print_help_on_error = True if options.offline and options.command not in (u'lint', u'test'): print(u'Cannot run: {0:s} in offline mode.'.format(options.command)) print_help_on_error = True if print_help_on_error: print(u'') argument_parser.print_help() print(u'') return False home_path = os.path.expanduser(u'~') netrc_path = os.path.join(home_path, u'.netrc') if not os.path.exists(netrc_path): print(u'{0:s} aborted - unable to find .netrc.'.format( options.command.title())) return False review_helper = ReviewHelper( options.command, github_origin, feature_branch, options.diffbase, all_files=options.all_files, no_browser=options.no_browser, no_confirm=options.no_confirm) if not review_helper.InitializeHelpers(): return False if not review_helper.CheckLocalGitState(): return False if not options.offline and not review_helper.CheckRemoteGitState(): return False if options.command == u'merge': if not review_helper.PrepareMerge(codereview_issue_number): return False if not review_helper.Lint(): return False if not review_helper.Test(): return False result = False if options.command == u'create': result = review_helper.Create() elif options.command == u'close': result = review_helper.Close() elif options.command in (u'lint', u'test'): result = True elif options.command == u'merge': result = review_helper.Merge(codereview_issue_number) elif options.command == u'open': result = review_helper.Open(codereview_issue_number) elif options.command == u'update': result = review_helper.Update() elif options.command in (u'update-authors', u'update_authors'): result = review_helper.UpdateAuthors() elif options.command in (u'update-version', u'update_version'): result = review_helper.UpdateVersion() return result if __name__ == u'__main__': if not Main(): sys.exit(1) else: sys.exit(0) dfdatetime-20170103/utils/upload.py000066400000000000000000002772321303277242400170760ustar00rootroot00000000000000#!/usr/bin/env python # coding: utf-8 # # This file originates from the Rietveld project: # https://code.google.com/p/rietveld/ # # Copyright 2007 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tool for uploading diffs from a version control system to the codereview app. Usage summary: upload.py [options] [-- diff_options] [path...] Diff options are passed to the diff command of the underlying system. Supported version control systems: Git Mercurial Subversion Perforce CVS It is important for Git/Mercurial users to specify a tree/node/branch to diff against by using the '--rev' option. """ # This code is derived from appcfg.py in the App Engine SDK (open source), # and from ASPN recipe #146306. import BaseHTTPServer import ConfigParser import cookielib import errno import fnmatch import getpass import logging import marshal import mimetypes import optparse import os import re import socket import subprocess import sys import urllib import urllib2 import urlparse import webbrowser # The md5 module was deprecated in Python 2.5. try: from hashlib import md5 except ImportError: from md5 import md5 try: import readline except ImportError: pass try: import keyring except ImportError: keyring = None # The logging verbosity: # 0: Errors only. # 1: Status messages. # 2: Info logs. # 3: Debug logs. verbosity = 1 # The account type used for authentication. # This line could be changed by the review server (see handler for # upload.py). AUTH_ACCOUNT_TYPE = "GOOGLE" # URL of the default review server. As for AUTH_ACCOUNT_TYPE, this line could be # changed by the review server (see handler for upload.py). DEFAULT_REVIEW_SERVER = "codereview.appspot.com" # Max size of patch or base file. MAX_UPLOAD_SIZE = 900 * 1024 # Constants for version control names. Used by GuessVCSName. VCS_GIT = "Git" VCS_MERCURIAL = "Mercurial" VCS_SUBVERSION = "Subversion" VCS_PERFORCE = "Perforce" VCS_CVS = "CVS" VCS_UNKNOWN = "Unknown" VCS_ABBREVIATIONS = { VCS_MERCURIAL.lower(): VCS_MERCURIAL, "hg": VCS_MERCURIAL, VCS_SUBVERSION.lower(): VCS_SUBVERSION, "svn": VCS_SUBVERSION, VCS_PERFORCE.lower(): VCS_PERFORCE, "p4": VCS_PERFORCE, VCS_GIT.lower(): VCS_GIT, VCS_CVS.lower(): VCS_CVS, } # OAuth 2.0-Related Constants LOCALHOST_IP = '127.0.0.1' DEFAULT_OAUTH2_PORT = 8001 ACCESS_TOKEN_PARAM = 'access_token' ERROR_PARAM = 'error' OAUTH_DEFAULT_ERROR_MESSAGE = 'OAuth 2.0 error occurred.' OAUTH_PATH = '/get-access-token' OAUTH_PATH_PORT_TEMPLATE = OAUTH_PATH + '?port=%(port)d' AUTH_HANDLER_RESPONSE = """\ Authentication Status

The authentication flow has completed.

""" # Borrowed from google-api-python-client OPEN_LOCAL_MESSAGE_TEMPLATE = """\ Your browser has been opened to visit: %s If your browser is on a different machine then exit and re-run upload.py with the command-line parameter --no_oauth2_webbrowser """ NO_OPEN_LOCAL_MESSAGE_TEMPLATE = """\ Go to the following link in your browser: %s and copy the access token. """ # The result of parsing Subversion's [auto-props] setting. svn_auto_props_map = None def GetEmail(prompt): """Prompts the user for their email address and returns it. The last used email address is saved to a file and offered up as a suggestion to the user. If the user presses enter without typing in anything the last used email address is used. If the user enters a new address, it is saved for next time we prompt. """ last_email_file_name = os.path.expanduser("~/.last_codereview_email_address") last_email = "" if os.path.exists(last_email_file_name): try: last_email_file = open(last_email_file_name, "r") last_email = last_email_file.readline().strip("\n") last_email_file.close() prompt += " [%s]" % last_email except IOError, e: pass email = raw_input(prompt + ": ").strip() if email: try: last_email_file = open(last_email_file_name, "w") last_email_file.write(email) last_email_file.close() except IOError, e: pass else: email = last_email return email def StatusUpdate(msg): """Print a status message to stdout. If 'verbosity' is greater than 0, print the message. Args: msg: The string to print. """ if verbosity > 0: print msg def ErrorExit(msg): """Print an error message to stderr and exit.""" print >>sys.stderr, msg sys.exit(1) class ClientLoginError(urllib2.HTTPError): """Raised to indicate there was an error authenticating with ClientLogin.""" def __init__(self, url, code, msg, headers, args): urllib2.HTTPError.__init__(self, url, code, msg, headers, None) self.args = args self._reason = args["Error"] self.info = args.get("Info", None) @property def reason(self): # reason is a property on python 2.7 but a member variable on <=2.6. # self.args is modified so it cannot be used as-is so save the value in # self._reason. return self._reason class AbstractRpcServer(object): """Provides a common interface for a simple RPC server.""" def __init__(self, host, auth_function, host_override=None, extra_headers=None, save_cookies=False, account_type=AUTH_ACCOUNT_TYPE): """Creates a new AbstractRpcServer. Args: host: The host to send requests to. auth_function: A function that takes no arguments and returns an (email, password) tuple when called. Will be called if authentication is required. host_override: The host header to send to the server (defaults to host). extra_headers: A dict of extra headers to append to every request. save_cookies: If True, save the authentication cookies to local disk. If False, use an in-memory cookiejar instead. Subclasses must implement this functionality. Defaults to False. account_type: Account type used for authentication. Defaults to AUTH_ACCOUNT_TYPE. """ self.host = host if (not self.host.startswith("http://") and not self.host.startswith("https://")): self.host = "http://" + self.host self.host_override = host_override self.auth_function = auth_function self.authenticated = False self.extra_headers = extra_headers or {} self.save_cookies = save_cookies self.account_type = account_type self.opener = self._GetOpener() if self.host_override: logging.info("Server: %s; Host: %s", self.host, self.host_override) else: logging.info("Server: %s", self.host) def _GetOpener(self): """Returns an OpenerDirector for making HTTP requests. Returns: A urllib2.OpenerDirector object. """ raise NotImplementedError() def _CreateRequest(self, url, data=None): """Creates a new urllib request.""" logging.debug("Creating request for: '%s' with payload:\n%s", url, data) req = urllib2.Request(url, data=data, headers={"Accept": "text/plain"}) if self.host_override: req.add_header("Host", self.host_override) for key, value in self.extra_headers.iteritems(): req.add_header(key, value) return req def _GetAuthToken(self, email, password): """Uses ClientLogin to authenticate the user, returning an auth token. Args: email: The user's email address password: The user's password Raises: ClientLoginError: If there was an error authenticating with ClientLogin. HTTPError: If there was some other form of HTTP error. Returns: The authentication token returned by ClientLogin. """ account_type = self.account_type if self.host.endswith(".google.com"): # Needed for use inside Google. account_type = "HOSTED" req = self._CreateRequest( url="https://www.google.com/accounts/ClientLogin", data=urllib.urlencode({ "Email": email, "Passwd": password, "service": "ah", "source": "rietveld-codereview-upload", "accountType": account_type, }), ) try: response = self.opener.open(req) response_body = response.read() response_dict = dict(x.split("=") for x in response_body.split("\n") if x) return response_dict["Auth"] except urllib2.HTTPError, e: if e.code == 403: body = e.read() response_dict = dict(x.split("=", 1) for x in body.split("\n") if x) raise ClientLoginError(req.get_full_url(), e.code, e.msg, e.headers, response_dict) else: raise def _GetAuthCookie(self, auth_token): """Fetches authentication cookies for an authentication token. Args: auth_token: The authentication token returned by ClientLogin. Raises: HTTPError: If there was an error fetching the authentication cookies. """ # This is a dummy value to allow us to identify when we're successful. continue_location = "http://localhost/" args = {"continue": continue_location, "auth": auth_token} req = self._CreateRequest("%s/_ah/login?%s" % (self.host, urllib.urlencode(args))) try: response = self.opener.open(req) except urllib2.HTTPError, e: response = e if (response.code != 302 or response.info()["location"] != continue_location): raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, response.headers, response.fp) self.authenticated = True def _Authenticate(self): """Authenticates the user. The authentication process works as follows: 1) We get a username and password from the user 2) We use ClientLogin to obtain an AUTH token for the user (see http://code.google.com/apis/accounts/AuthForInstalledApps.html). 3) We pass the auth token to /_ah/login on the server to obtain an authentication cookie. If login was successful, it tries to redirect us to the URL we provided. If we attempt to access the upload API without first obtaining an authentication cookie, it returns a 401 response (or a 302) and directs us to authenticate ourselves with ClientLogin. """ for i in range(3): credentials = self.auth_function() try: auth_token = self._GetAuthToken(credentials[0], credentials[1]) except ClientLoginError, e: print >>sys.stderr, '' if e.reason == "BadAuthentication": if e.info == "InvalidSecondFactor": print >>sys.stderr, ( "Use an application-specific password instead " "of your regular account password.\n" "See http://www.google.com/" "support/accounts/bin/answer.py?answer=185833") else: print >>sys.stderr, "Invalid username or password." elif e.reason == "CaptchaRequired": print >>sys.stderr, ( "Please go to\n" "https://www.google.com/accounts/DisplayUnlockCaptcha\n" "and verify you are a human. Then try again.\n" "If you are using a Google Apps account the URL is:\n" "https://www.google.com/a/yourdomain.com/UnlockCaptcha") elif e.reason == "NotVerified": print >>sys.stderr, "Account not verified." elif e.reason == "TermsNotAgreed": print >>sys.stderr, "User has not agreed to TOS." elif e.reason == "AccountDeleted": print >>sys.stderr, "The user account has been deleted." elif e.reason == "AccountDisabled": print >>sys.stderr, "The user account has been disabled." break elif e.reason == "ServiceDisabled": print >>sys.stderr, ("The user's access to the service has been " "disabled.") elif e.reason == "ServiceUnavailable": print >>sys.stderr, "The service is not available; try again later." else: # Unknown error. raise print >>sys.stderr, '' continue self._GetAuthCookie(auth_token) return def Send(self, request_path, payload=None, content_type="application/octet-stream", timeout=None, extra_headers=None, **kwargs): """Sends an RPC and returns the response. Args: request_path: The path to send the request to, eg /api/appversion/create. payload: The body of the request, or None to send an empty request. content_type: The Content-Type header to use. timeout: timeout in seconds; default None i.e. no timeout. (Note: for large requests on OS X, the timeout doesn't work right.) extra_headers: Dict containing additional HTTP headers that should be included in the request (string header names mapped to their values), or None to not include any additional headers. kwargs: Any keyword arguments are converted into query string parameters. Returns: The response body, as a string. """ # TODO: Don't require authentication. Let the server say # whether it is necessary. if not self.authenticated: self._Authenticate() old_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(timeout) try: tries = 0 while True: tries += 1 args = dict(kwargs) url = "%s%s" % (self.host, request_path) if args: url += "?" + urllib.urlencode(args) req = self._CreateRequest(url=url, data=payload) req.add_header("Content-Type", content_type) if extra_headers: for header, value in extra_headers.items(): req.add_header(header, value) try: f = self.opener.open(req) response = f.read() f.close() return response except urllib2.HTTPError, e: if tries > 3: raise elif e.code == 401 or e.code == 302: self._Authenticate() elif e.code == 301: # Handle permanent redirect manually. url = e.info()["location"] url_loc = urlparse.urlparse(url) self.host = '%s://%s' % (url_loc[0], url_loc[1]) elif e.code >= 500: ErrorExit(e.read()) else: raise finally: socket.setdefaulttimeout(old_timeout) class HttpRpcServer(AbstractRpcServer): """Provides a simplified RPC-style interface for HTTP requests.""" def _Authenticate(self): """Save the cookie jar after authentication.""" if isinstance(self.auth_function, OAuth2Creds): access_token = self.auth_function() if access_token is not None: self.extra_headers['Authorization'] = 'OAuth %s' % (access_token,) self.authenticated = True else: super(HttpRpcServer, self)._Authenticate() if self.save_cookies: StatusUpdate("Saving authentication cookies to %s" % self.cookie_file) self.cookie_jar.save() def _GetOpener(self): """Returns an OpenerDirector that supports cookies and ignores redirects. Returns: A urllib2.OpenerDirector object. """ opener = urllib2.OpenerDirector() opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) if self.save_cookies: self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies") self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file) if os.path.exists(self.cookie_file): try: self.cookie_jar.load() self.authenticated = True StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file) except (cookielib.LoadError, IOError): # Failed to load cookies - just ignore them. pass else: # Create an empty cookie file with mode 600 fd = os.open(self.cookie_file, os.O_CREAT, 0600) os.close(fd) # Always chmod the cookie file os.chmod(self.cookie_file, 0600) else: # Don't save cookies across runs of update.py. self.cookie_jar = cookielib.CookieJar() opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) return opener class CondensedHelpFormatter(optparse.IndentedHelpFormatter): """Frees more horizontal space by removing indentation from group options and collapsing arguments between short and long, e.g. '-o ARG, --opt=ARG' to -o --opt ARG""" def format_heading(self, heading): return "%s:\n" % heading def format_option(self, option): self.dedent() res = optparse.HelpFormatter.format_option(self, option) self.indent() return res def format_option_strings(self, option): self.set_long_opt_delimiter(" ") optstr = optparse.HelpFormatter.format_option_strings(self, option) optlist = optstr.split(", ") if len(optlist) > 1: if option.takes_value(): # strip METAVAR from all but the last option optlist = [x.split()[0] for x in optlist[:-1]] + optlist[-1:] optstr = " ".join(optlist) return optstr parser = optparse.OptionParser( usage=("%prog [options] [-- diff_options] [path...]\n" "See also: http://code.google.com/p/rietveld/wiki/UploadPyUsage"), add_help_option=False, formatter=CondensedHelpFormatter() ) parser.add_option("-h", "--help", action="store_true", help="Show this help message and exit.") parser.add_option("-y", "--assume_yes", action="store_true", dest="assume_yes", default=False, help="Assume that the answer to yes/no questions is 'yes'.") # Logging group = parser.add_option_group("Logging options") group.add_option("-q", "--quiet", action="store_const", const=0, dest="verbose", help="Print errors only.") group.add_option("-v", "--verbose", action="store_const", const=2, dest="verbose", default=1, help="Print info level logs.") group.add_option("--noisy", action="store_const", const=3, dest="verbose", help="Print all logs.") group.add_option("--print_diffs", dest="print_diffs", action="store_true", help="Print full diffs.") # Review server group = parser.add_option_group("Review server options") group.add_option("-s", "--server", action="store", dest="server", default=DEFAULT_REVIEW_SERVER, metavar="SERVER", help=("The server to upload to. The format is host[:port]. " "Defaults to '%default'.")) group.add_option("-e", "--email", action="store", dest="email", metavar="EMAIL", default=None, help="The username to use. Will prompt if omitted.") group.add_option("-H", "--host", action="store", dest="host", metavar="HOST", default=None, help="Overrides the Host header sent with all RPCs.") group.add_option("--no_cookies", action="store_false", dest="save_cookies", default=True, help="Do not save authentication cookies to local disk.") group.add_option("--oauth2", action="store_true", dest="use_oauth2", default=False, help="Use OAuth 2.0 instead of a password.") group.add_option("--oauth2_port", action="store", type="int", dest="oauth2_port", default=DEFAULT_OAUTH2_PORT, help=("Port to use to handle OAuth 2.0 redirect. Must be an " "integer in the range 1024-49151, defaults to " "'%default'.")) group.add_option("--no_oauth2_webbrowser", action="store_false", dest="open_oauth2_local_webbrowser", default=True, help="Don't open a browser window to get an access token.") group.add_option("--account_type", action="store", dest="account_type", metavar="TYPE", default=AUTH_ACCOUNT_TYPE, choices=["GOOGLE", "HOSTED"], help=("Override the default account type " "(defaults to '%default', " "valid choices are 'GOOGLE' and 'HOSTED').")) # Issue group = parser.add_option_group("Issue options") group.add_option("-t", "--title", action="store", dest="title", help="New issue subject or new patch set title") group.add_option("-m", "--message", action="store", dest="message", default=None, help="New issue description or new patch set message") group.add_option("-F", "--file", action="store", dest="file", default=None, help="Read the message above from file.") group.add_option("-r", "--reviewers", action="store", dest="reviewers", metavar="REVIEWERS", default=None, help="Add reviewers (comma separated email addresses).") group.add_option("--cc", action="store", dest="cc", metavar="CC", default='log2timeline-dev@googlegroups.com', help="Add CC (comma separated email addresses).") group.add_option("--private", action="store_true", dest="private", default=False, help="Make the issue restricted to reviewers and those CCed") # Upload options group = parser.add_option_group("Patch options") group.add_option("-i", "--issue", type="int", action="store", metavar="ISSUE", default=None, help="Issue number to which to add. Defaults to new issue.") group.add_option("--cache", action="store_true", dest="add_cache", default=False, help="Add git cache parameter for new files.") group.add_option("--base_url", action="store", dest="base_url", default=None, help="Base URL path for files (listed as \"Base URL\" when " "viewing issue). If omitted, will be guessed automatically " "for SVN repos and left blank for others.") group.add_option("--download_base", action="store_true", dest="download_base", default=False, help="Base files will be downloaded by the server " "(side-by-side diffs may not work on files with CRs).") group.add_option("--rev", action="store", dest="revision", metavar="REV", default=None, help="Base revision/branch/tree to diff against. Use " "rev1:rev2 range to review already committed changeset.") group.add_option("--send_mail", action="store_true", dest="send_mail", default=False, help="Send notification email to reviewers.") group.add_option("-p", "--send_patch", action="store_true", dest="send_patch", default=False, help="Same as --send_mail, but include diff as an " "attachment, and prepend email subject with 'PATCH:'.") group.add_option("--vcs", action="store", dest="vcs", metavar="VCS", default=None, help=("Version control system (optional, usually upload.py " "already guesses the right VCS).")) group.add_option("--emulate_svn_auto_props", action="store_true", dest="emulate_svn_auto_props", default=False, help=("Emulate Subversion's auto properties feature.")) # Git-specific group = parser.add_option_group("Git-specific options") group.add_option("--git_similarity", action="store", dest="git_similarity", metavar="SIM", type="int", default=50, help=("Set the minimum similarity index for detecting renames " "and copies. See `git diff -C`. (default 50).")) group.add_option("--git_no_find_copies", action="store_false", default=True, dest="git_find_copies", help=("Prevents git from looking for copies (default off).")) # Perforce-specific group = parser.add_option_group("Perforce-specific options " "(overrides P4 environment variables)") group.add_option("--p4_port", action="store", dest="p4_port", metavar="P4_PORT", default=None, help=("Perforce server and port (optional)")) group.add_option("--p4_changelist", action="store", dest="p4_changelist", metavar="P4_CHANGELIST", default=None, help=("Perforce changelist id")) group.add_option("--p4_client", action="store", dest="p4_client", metavar="P4_CLIENT", default=None, help=("Perforce client/workspace")) group.add_option("--p4_user", action="store", dest="p4_user", metavar="P4_USER", default=None, help=("Perforce user")) # OAuth 2.0 Methods and Helpers class ClientRedirectServer(BaseHTTPServer.HTTPServer): """A server for redirects back to localhost from the associated server. Waits for a single request and parses the query parameters for an access token or an error and then stops serving. """ access_token = None error = None class ClientRedirectHandler(BaseHTTPServer.BaseHTTPRequestHandler): """A handler for redirects back to localhost from the associated server. Waits for a single request and parses the query parameters into the server's access_token or error and then stops serving. """ def SetResponseValue(self): """Stores the access token or error from the request on the server. Will only do this if exactly one query parameter was passed in to the request and that query parameter used 'access_token' or 'error' as the key. """ query_string = urlparse.urlparse(self.path).query query_params = urlparse.parse_qs(query_string) if len(query_params) == 1: if query_params.has_key(ACCESS_TOKEN_PARAM): access_token_list = query_params[ACCESS_TOKEN_PARAM] if len(access_token_list) == 1: self.server.access_token = access_token_list[0] else: error_list = query_params.get(ERROR_PARAM, []) if len(error_list) == 1: self.server.error = error_list[0] def do_GET(self): """Handle a GET request. Parses and saves the query parameters and prints a message that the server has completed its lone task (handling a redirect). Note that we can't detect if an error occurred. """ self.send_response(200) self.send_header('Content-type', 'text/html') self.end_headers() self.SetResponseValue() self.wfile.write(AUTH_HANDLER_RESPONSE) def log_message(self, format, *args): """Do not log messages to stdout while running as command line program.""" pass def OpenOAuth2ConsentPage(server=DEFAULT_REVIEW_SERVER, port=DEFAULT_OAUTH2_PORT): """Opens the OAuth 2.0 consent page or prints instructions how to. Uses the webbrowser module to open the OAuth server side page in a browser. Args: server: String containing the review server URL. Defaults to DEFAULT_REVIEW_SERVER. port: Integer, the port where the localhost server receiving the redirect is serving. Defaults to DEFAULT_OAUTH2_PORT. Returns: A boolean indicating whether the page opened successfully. """ path = OAUTH_PATH_PORT_TEMPLATE % {'port': port} parsed_url = urlparse.urlparse(server) scheme = parsed_url[0] or 'https' if scheme != 'https': ErrorExit('Using OAuth requires a review server with SSL enabled.') # If no scheme was given on command line the server address ends up in # parsed_url.path otherwise in netloc. host = parsed_url[1] or parsed_url[2] page = '%s://%s%s' % (scheme, host, path) page_opened = webbrowser.open(page, new=1, autoraise=True) if page_opened: print OPEN_LOCAL_MESSAGE_TEMPLATE % (page,) return page_opened def WaitForAccessToken(port=DEFAULT_OAUTH2_PORT): """Spins up a simple HTTP Server to handle a single request. Intended to handle a single redirect from the production server after the user authenticated via OAuth 2.0 with the server. Args: port: Integer, the port where the localhost server receiving the redirect is serving. Defaults to DEFAULT_OAUTH2_PORT. Returns: The access token passed to the localhost server, or None if no access token was passed. """ httpd = ClientRedirectServer((LOCALHOST_IP, port), ClientRedirectHandler) # Wait to serve just one request before deferring control back # to the caller of wait_for_refresh_token httpd.handle_request() if httpd.access_token is None: ErrorExit(httpd.error or OAUTH_DEFAULT_ERROR_MESSAGE) return httpd.access_token def GetAccessToken(server=DEFAULT_REVIEW_SERVER, port=DEFAULT_OAUTH2_PORT, open_local_webbrowser=True): """Gets an Access Token for the current user. Args: server: String containing the review server URL. Defaults to DEFAULT_REVIEW_SERVER. port: Integer, the port where the localhost server receiving the redirect is serving. Defaults to DEFAULT_OAUTH2_PORT. open_local_webbrowser: Boolean, defaults to True. If set, opens a page in the user's browser. Returns: A string access token that was sent to the local server. If the serving page via WaitForAccessToken does not receive an access token, this method returns None. """ access_token = None if open_local_webbrowser: page_opened = OpenOAuth2ConsentPage(server=server, port=port) if page_opened: try: access_token = WaitForAccessToken(port=port) except socket.error, e: print 'Can\'t start local webserver. Socket Error: %s\n' % (e.strerror,) if access_token is None: # TODO(dhermes): Offer to add to clipboard using xsel, xclip, pbcopy, etc. page = 'https://%s%s' % (server, OAUTH_PATH) print NO_OPEN_LOCAL_MESSAGE_TEMPLATE % (page,) access_token = raw_input('Enter access token: ').strip() return access_token class KeyringCreds(object): def __init__(self, server, host, email): self.server = server # Explicitly cast host to str to work around bug in old versions of Keyring # (versions before 0.10). Even though newer versions of Keyring fix this, # some modern linuxes (such as Ubuntu 12.04) still bundle a version with # the bug. self.host = str(host) self.email = email self.accounts_seen = set() def GetUserCredentials(self): """Prompts the user for a username and password. Only use keyring on the initial call. If the keyring contains the wrong password, we want to give the user a chance to enter another one. """ # Create a local alias to the email variable to avoid Python's crazy # scoping rules. global keyring email = self.email if email is None: email = GetEmail("Email (login for uploading to %s)" % self.server) password = None if keyring and not email in self.accounts_seen: try: password = keyring.get_password(self.host, email) except: # Sadly, we have to trap all errors here as # gnomekeyring.IOError inherits from object. :/ print "Failed to get password from keyring" keyring = None if password is not None: print "Using password from system keyring." self.accounts_seen.add(email) else: password = getpass.getpass("Password for %s: " % email) if keyring: answer = raw_input("Store password in system keyring?(y/N) ").strip() if answer == "y": keyring.set_password(self.host, email, password) self.accounts_seen.add(email) return (email, password) class OAuth2Creds(object): """Simple object to hold server and port to be passed to GetAccessToken.""" def __init__(self, server, port, open_local_webbrowser=True): self.server = server self.port = port self.open_local_webbrowser = open_local_webbrowser def __call__(self): """Uses stored server and port to retrieve OAuth 2.0 access token.""" return GetAccessToken(server=self.server, port=self.port, open_local_webbrowser=self.open_local_webbrowser) def GetRpcServer(server, email=None, host_override=None, save_cookies=True, account_type=AUTH_ACCOUNT_TYPE, use_oauth2=False, oauth2_port=DEFAULT_OAUTH2_PORT, open_oauth2_local_webbrowser=True): """Returns an instance of an AbstractRpcServer. Args: server: String containing the review server URL. email: String containing user's email address. host_override: If not None, string containing an alternate hostname to use in the host header. save_cookies: Whether authentication cookies should be saved to disk. account_type: Account type for authentication, either 'GOOGLE' or 'HOSTED'. Defaults to AUTH_ACCOUNT_TYPE. use_oauth2: Boolean indicating whether OAuth 2.0 should be used for authentication. oauth2_port: Integer, the port where the localhost server receiving the redirect is serving. Defaults to DEFAULT_OAUTH2_PORT. open_oauth2_local_webbrowser: Boolean, defaults to True. If True and using OAuth, this opens a page in the user's browser to obtain a token. Returns: A new HttpRpcServer, on which RPC calls can be made. """ # If this is the dev_appserver, use fake authentication. host = (host_override or server).lower() if re.match(r'(http://)?localhost([:/]|$)', host): if email is None: email = "test@example.com" logging.info("Using debug user %s. Override with --email" % email) server = HttpRpcServer( server, lambda: (email, "password"), host_override=host_override, extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email}, save_cookies=save_cookies, account_type=account_type) # Don't try to talk to ClientLogin. server.authenticated = True return server positional_args = [server] if use_oauth2: positional_args.append( OAuth2Creds(server, oauth2_port, open_oauth2_local_webbrowser)) else: positional_args.append(KeyringCreds(server, host, email).GetUserCredentials) return HttpRpcServer(*positional_args, host_override=host_override, save_cookies=save_cookies, account_type=account_type) def EncodeMultipartFormData(fields, files): """Encode form fields for multipart/form-data. Args: fields: A sequence of (name, value) elements for regular form fields. files: A sequence of (name, filename, value) elements for data to be uploaded as files. Returns: (content_type, body) ready for httplib.HTTP instance. Source: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306 """ BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' CRLF = '\r\n' lines = [] for (key, value) in fields: lines.append('--' + BOUNDARY) lines.append('Content-Disposition: form-data; name="%s"' % key) lines.append('') if isinstance(value, unicode): value = value.encode('utf-8') lines.append(value) for (key, filename, value) in files: lines.append('--' + BOUNDARY) lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) lines.append('Content-Type: %s' % GetContentType(filename)) lines.append('') if isinstance(value, unicode): value = value.encode('utf-8') lines.append(value) lines.append('--' + BOUNDARY + '--') lines.append('') body = CRLF.join(lines) content_type = 'multipart/form-data; boundary=%s' % BOUNDARY return content_type, body def GetContentType(filename): """Helper to guess the content-type from the filename.""" return mimetypes.guess_type(filename)[0] or 'application/octet-stream' # Use a shell for subcommands on Windows to get a PATH search. use_shell = sys.platform.startswith("win") def RunShellWithReturnCodeAndStderr(command, print_output=False, universal_newlines=True, env=os.environ): """Executes a command and returns the output from stdout, stderr and the return code. Args: command: Command to execute. print_output: If True, the output is printed to stdout. If False, both stdout and stderr are ignored. universal_newlines: Use universal_newlines flag (default: True). Returns: Tuple (stdout, stderr, return code) """ logging.info("Running %s", command) env = env.copy() env['LC_MESSAGES'] = 'C' p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=use_shell, universal_newlines=universal_newlines, env=env) if print_output: output_array = [] while True: line = p.stdout.readline() if not line: break print line.strip("\n") output_array.append(line) output = "".join(output_array) else: output = p.stdout.read() p.wait() errout = p.stderr.read() if print_output and errout: print >>sys.stderr, errout p.stdout.close() p.stderr.close() return output, errout, p.returncode def RunShellWithReturnCode(command, print_output=False, universal_newlines=True, env=os.environ): """Executes a command and returns the output from stdout and the return code.""" out, err, retcode = RunShellWithReturnCodeAndStderr(command, print_output, universal_newlines, env) return out, retcode def RunShell(command, silent_ok=False, universal_newlines=True, print_output=False, env=os.environ): data, retcode = RunShellWithReturnCode(command, print_output, universal_newlines, env) if retcode: ErrorExit("Got error status from %s:\n%s" % (command, data)) if not silent_ok and not data: ErrorExit("No output from %s" % command) return data class VersionControlSystem(object): """Abstract base class providing an interface to the VCS.""" def __init__(self, options): """Constructor. Args: options: Command line options. """ self.options = options def GetGUID(self): """Return string to distinguish the repository from others, for example to query all opened review issues for it""" raise NotImplementedError( "abstract method -- subclass %s must override" % self.__class__) def PostProcessDiff(self, diff): """Return the diff with any special post processing this VCS needs, e.g. to include an svn-style "Index:".""" return diff def GenerateDiff(self, args): """Return the current diff as a string. Args: args: Extra arguments to pass to the diff command. """ raise NotImplementedError( "abstract method -- subclass %s must override" % self.__class__) def GetUnknownFiles(self): """Return a list of files unknown to the VCS.""" raise NotImplementedError( "abstract method -- subclass %s must override" % self.__class__) def CheckForUnknownFiles(self): """Show an "are you sure?" prompt if there are unknown files.""" unknown_files = self.GetUnknownFiles() if unknown_files: print "The following files are not added to version control:" for line in unknown_files: print line prompt = "Are you sure to continue?(y/N) " answer = raw_input(prompt).strip() if answer != "y": ErrorExit("User aborted") def GetBaseFile(self, filename): """Get the content of the upstream version of a file. Returns: A tuple (base_content, new_content, is_binary, status) base_content: The contents of the base file. new_content: For text files, this is empty. For binary files, this is the contents of the new file, since the diff output won't contain information to reconstruct the current file. is_binary: True iff the file is binary. status: The status of the file. """ raise NotImplementedError( "abstract method -- subclass %s must override" % self.__class__) def GetBaseFiles(self, diff): """Helper that calls GetBase file for each file in the patch. Returns: A dictionary that maps from filename to GetBaseFile's tuple. Filenames are retrieved based on lines that start with "Index:" or "Property changes on:". """ files = {} for line in diff.splitlines(True): if line.startswith('Index:') or line.startswith('Property changes on:'): unused, filename = line.split(':', 1) # On Windows if a file has property changes its filename uses '\' # instead of '/'. filename = filename.strip().replace('\\', '/') files[filename] = self.GetBaseFile(filename) return files def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options, files): """Uploads the base files (and if necessary, the current ones as well).""" def UploadFile(filename, file_id, content, is_binary, status, is_base): """Uploads a file to the server.""" file_too_large = False if is_base: type = "base" else: type = "current" if len(content) > MAX_UPLOAD_SIZE: print ("Not uploading the %s file for %s because it's too large." % (type, filename)) file_too_large = True content = "" checksum = md5(content).hexdigest() if options.verbose > 0 and not file_too_large: print "Uploading %s file for %s" % (type, filename) url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id) form_fields = [("filename", filename), ("status", status), ("checksum", checksum), ("is_binary", str(is_binary)), ("is_current", str(not is_base)), ] if file_too_large: form_fields.append(("file_too_large", "1")) if options.email: form_fields.append(("user", options.email)) ctype, body = EncodeMultipartFormData(form_fields, [("data", filename, content)]) response_body = rpc_server.Send(url, body, content_type=ctype) if not response_body.startswith("OK"): StatusUpdate(" --> %s" % response_body) sys.exit(1) patches = dict() [patches.setdefault(v, k) for k, v in patch_list] for filename in patches.keys(): base_content, new_content, is_binary, status = files[filename] file_id_str = patches.get(filename) if file_id_str.find("nobase") != -1: base_content = None file_id_str = file_id_str[file_id_str.rfind("_") + 1:] file_id = int(file_id_str) if base_content != None: UploadFile(filename, file_id, base_content, is_binary, status, True) if new_content != None: UploadFile(filename, file_id, new_content, is_binary, status, False) def IsImage(self, filename): """Returns true if the filename has an image extension.""" mimetype = mimetypes.guess_type(filename)[0] if not mimetype: return False return mimetype.startswith("image/") and not mimetype.startswith("image/svg") def IsBinaryData(self, data): """Returns true if data contains a null byte.""" # Derived from how Mercurial's heuristic, see # http://selenic.com/hg/file/848a6658069e/mercurial/util.py#l229 return bool(data and "\0" in data) class SubversionVCS(VersionControlSystem): """Implementation of the VersionControlSystem interface for Subversion.""" def __init__(self, options): super(SubversionVCS, self).__init__(options) if self.options.revision: match = re.match(r"(\d+)(:(\d+))?", self.options.revision) if not match: ErrorExit("Invalid Subversion revision %s." % self.options.revision) self.rev_start = match.group(1) self.rev_end = match.group(3) else: self.rev_start = self.rev_end = None # Cache output from "svn list -r REVNO dirname". # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev). self.svnls_cache = {} # Base URL is required to fetch files deleted in an older revision. # Result is cached to not guess it over and over again in GetBaseFile(). required = self.options.download_base or self.options.revision is not None self.svn_base = self._GuessBase(required) def GetGUID(self): return self._GetInfo("Repository UUID") def GuessBase(self, required): """Wrapper for _GuessBase.""" return self.svn_base def _GuessBase(self, required): """Returns base URL for current diff. Args: required: If true, exits if the url can't be guessed, otherwise None is returned. """ url = self._GetInfo("URL") if url: scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) guess = "" # TODO(anatoli) - repository specific hacks should be handled by server if netloc == "svn.python.org" and scheme == "svn+ssh": path = "projects" + path scheme = "http" guess = "Python " elif netloc.endswith(".googlecode.com"): scheme = "http" guess = "Google Code " path = path + "/" base = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) logging.info("Guessed %sbase = %s", guess, base) return base if required: ErrorExit("Can't find URL in output from svn info") return None def _GetInfo(self, key): """Parses 'svn info' for current dir. Returns value for key or None""" for line in RunShell(["svn", "info"]).splitlines(): if line.startswith(key + ": "): return line.split(":", 1)[1].strip() def _EscapeFilename(self, filename): """Escapes filename for SVN commands.""" if "@" in filename and not filename.endswith("@"): filename = "%s@" % filename return filename def GenerateDiff(self, args): cmd = ["svn", "diff"] if self.options.revision: cmd += ["-r", self.options.revision] cmd.extend(args) data = RunShell(cmd) count = 0 for line in data.splitlines(): if line.startswith("Index:") or line.startswith("Property changes on:"): count += 1 logging.info(line) if not count: ErrorExit("No valid patches found in output from svn diff") return data def _CollapseKeywords(self, content, keyword_str): """Collapses SVN keywords.""" # svn cat translates keywords but svn diff doesn't. As a result of this # behavior patching.PatchChunks() fails with a chunk mismatch error. # This part was originally written by the Review Board development team # who had the same problem (http://reviews.review-board.org/r/276/). # Mapping of keywords to known aliases svn_keywords = { # Standard keywords 'Date': ['Date', 'LastChangedDate'], 'Revision': ['Revision', 'LastChangedRevision', 'Rev'], 'Author': ['Author', 'LastChangedBy'], 'HeadURL': ['HeadURL', 'URL'], 'Id': ['Id'], # Aliases 'LastChangedDate': ['LastChangedDate', 'Date'], 'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'], 'LastChangedBy': ['LastChangedBy', 'Author'], 'URL': ['URL', 'HeadURL'], } def repl(m): if m.group(2): return "$%s::%s$" % (m.group(1), " " * len(m.group(3))) return "$%s$" % m.group(1) keywords = [keyword for name in keyword_str.split(" ") for keyword in svn_keywords.get(name, [])] return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content) def GetUnknownFiles(self): status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True) unknown_files = [] for line in status.split("\n"): if line and line[0] == "?": unknown_files.append(line) return unknown_files def ReadFile(self, filename): """Returns the contents of a file.""" file = open(filename, 'rb') result = "" try: result = file.read() finally: file.close() return result def GetStatus(self, filename): """Returns the status of a file.""" if not self.options.revision: status = RunShell(["svn", "status", "--ignore-externals", self._EscapeFilename(filename)]) if not status: ErrorExit("svn status returned no output for %s" % filename) status_lines = status.splitlines() # If file is in a cl, the output will begin with # "\n--- Changelist 'cl_name':\n". See # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt if (len(status_lines) == 3 and not status_lines[0] and status_lines[1].startswith("--- Changelist")): status = status_lines[2] else: status = status_lines[0] # If we have a revision to diff against we need to run "svn list" # for the old and the new revision and compare the results to get # the correct status for a file. else: dirname, relfilename = os.path.split(filename) if dirname not in self.svnls_cache: cmd = ["svn", "list", "-r", self.rev_start, self._EscapeFilename(dirname) or "."] out, err, returncode = RunShellWithReturnCodeAndStderr(cmd) if returncode: # Directory might not yet exist at start revison # svn: Unable to find repository location for 'abc' in revision nnn if re.match('^svn: Unable to find repository location for .+ in revision \d+', err): old_files = () else: ErrorExit("Failed to get status for %s:\n%s" % (filename, err)) else: old_files = out.splitlines() args = ["svn", "list"] if self.rev_end: args += ["-r", self.rev_end] cmd = args + [self._EscapeFilename(dirname) or "."] out, returncode = RunShellWithReturnCode(cmd) if returncode: ErrorExit("Failed to run command %s" % cmd) self.svnls_cache[dirname] = (old_files, out.splitlines()) old_files, new_files = self.svnls_cache[dirname] if relfilename in old_files and relfilename not in new_files: status = "D " elif relfilename in old_files and relfilename in new_files: status = "M " else: status = "A " return status def GetBaseFile(self, filename): status = self.GetStatus(filename) base_content = None new_content = None # If a file is copied its status will be "A +", which signifies # "addition-with-history". See "svn st" for more information. We need to # upload the original file or else diff parsing will fail if the file was # edited. if status[0] == "A" and status[3] != "+": # We'll need to upload the new content if we're adding a binary file # since diff's output won't contain it. mimetype = RunShell(["svn", "propget", "svn:mime-type", self._EscapeFilename(filename)], silent_ok=True) base_content = "" is_binary = bool(mimetype) and not mimetype.startswith("text/") if is_binary: new_content = self.ReadFile(filename) elif (status[0] in ("M", "D", "R") or (status[0] == "A" and status[3] == "+") or # Copied file. (status[0] == " " and status[1] == "M")): # Property change. args = [] if self.options.revision: # filename must not be escaped. We already add an ampersand here. url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start) else: # Don't change filename, it's needed later. url = filename args += ["-r", "BASE"] cmd = ["svn"] + args + ["propget", "svn:mime-type", url] mimetype, returncode = RunShellWithReturnCode(cmd) if returncode: # File does not exist in the requested revision. # Reset mimetype, it contains an error message. mimetype = "" else: mimetype = mimetype.strip() get_base = False # this test for binary is exactly the test prescribed by the # official SVN docs at # http://subversion.apache.org/faq.html#binary-files is_binary = (bool(mimetype) and not mimetype.startswith("text/") and mimetype not in ("image/x-xbitmap", "image/x-xpixmap")) if status[0] == " ": # Empty base content just to force an upload. base_content = "" elif is_binary: get_base = True if status[0] == "M": if not self.rev_end: new_content = self.ReadFile(filename) else: url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end) new_content = RunShell(["svn", "cat", url], universal_newlines=True, silent_ok=True) else: get_base = True if get_base: if is_binary: universal_newlines = False else: universal_newlines = True if self.rev_start: # "svn cat -r REV delete_file.txt" doesn't work. cat requires # the full URL with "@REV" appended instead of using "-r" option. url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start) base_content = RunShell(["svn", "cat", url], universal_newlines=universal_newlines, silent_ok=True) else: base_content, ret_code = RunShellWithReturnCode( ["svn", "cat", self._EscapeFilename(filename)], universal_newlines=universal_newlines) if ret_code and status[0] == "R": # It's a replaced file without local history (see issue208). # The base file needs to be fetched from the server. url = "%s/%s" % (self.svn_base, filename) base_content = RunShell(["svn", "cat", url], universal_newlines=universal_newlines, silent_ok=True) elif ret_code: ErrorExit("Got error status from 'svn cat %s'" % filename) if not is_binary: args = [] if self.rev_start: url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start) else: url = filename args += ["-r", "BASE"] cmd = ["svn"] + args + ["propget", "svn:keywords", url] keywords, returncode = RunShellWithReturnCode(cmd) if keywords and not returncode: base_content = self._CollapseKeywords(base_content, keywords) else: StatusUpdate("svn status returned unexpected output: %s" % status) sys.exit(1) return base_content, new_content, is_binary, status[0:5] class GitVCS(VersionControlSystem): """Implementation of the VersionControlSystem interface for Git.""" def __init__(self, options): super(GitVCS, self).__init__(options) # Map of filename -> (hash before, hash after) of base file. # Hashes for "no such file" are represented as None. self.hashes = {} # Map of new filename -> old filename for renames. self.renames = {} def GetGUID(self): revlist = RunShell("git rev-list --parents HEAD".split()).splitlines() # M-A: Return the 1st root hash, there could be multiple when a # subtree is merged. In that case, more analysis would need to # be done to figure out which HEAD is the 'most representative'. for r in revlist: if ' ' not in r: return r def PostProcessDiff(self, gitdiff): """Converts the diff output to include an svn-style "Index:" line as well as record the hashes of the files, so we can upload them along with our diff.""" # Special used by git to indicate "no such content". NULL_HASH = "0"*40 def IsFileNew(filename): return filename in self.hashes and self.hashes[filename][0] is None def AddSubversionPropertyChange(filename): """Add svn's property change information into the patch if given file is new file. We use Subversion's auto-props setting to retrieve its property. See http://svnbook.red-bean.com/en/1.1/ch07.html#svn-ch-7-sect-1.3.2 for Subversion's [auto-props] setting. """ if self.options.emulate_svn_auto_props and IsFileNew(filename): svnprops = GetSubversionPropertyChanges(filename) if svnprops: svndiff.append("\n" + svnprops + "\n") svndiff = [] filecount = 0 filename = None for line in gitdiff.splitlines(): match = re.match(r"diff --git a/(.*) b/(.*)$", line) if match: # Add auto property here for previously seen file. if filename is not None: AddSubversionPropertyChange(filename) filecount += 1 # Intentionally use the "after" filename so we can show renames. filename = match.group(2) svndiff.append("Index: %s\n" % filename) if match.group(1) != match.group(2): self.renames[match.group(2)] = match.group(1) else: # The "index" line in a git diff looks like this (long hashes elided): # index 82c0d44..b2cee3f 100755 # We want to save the left hash, as that identifies the base file. match = re.match(r"index (\w+)\.\.(\w+)", line) if match: before, after = (match.group(1), match.group(2)) if before == NULL_HASH: before = None if after == NULL_HASH: after = None self.hashes[filename] = (before, after) svndiff.append(line + "\n") if not filecount: ErrorExit("No valid patches found in output from git diff") # Add auto property for the last seen file. assert filename is not None AddSubversionPropertyChange(filename) return "".join(svndiff) def GenerateDiff(self, extra_args): extra_args = extra_args[:] if self.options.revision: if ":" in self.options.revision: extra_args = self.options.revision.split(":", 1) + extra_args else: extra_args = [self.options.revision] + extra_args # --no-ext-diff is broken in some versions of Git, so try to work around # this by overriding the environment (but there is still a problem if the # git config key "diff.external" is used). env = os.environ.copy() if "GIT_EXTERNAL_DIFF" in env: del env["GIT_EXTERNAL_DIFF"] # -M/-C will not print the diff for the deleted file when a file is renamed. # This is confusing because the original file will not be shown on the # review when a file is renamed. So, get a diff with ONLY deletes, then # append a diff (with rename detection), without deletes. cmd = [ "git", "diff", "--no-color", "--no-ext-diff", "--full-index", "--ignore-submodules", ] diff = RunShell( cmd + ["--no-renames", "--diff-filter=D"] + extra_args, env=env, silent_ok=True) if self.options.git_find_copies: similarity_options = ["--find-copies-harder", "-l100000", "-C%s" % self.options.git_similarity ] else: similarity_options = ["-M%s" % self.options.git_similarity ] diff += RunShell( cmd + ["--diff-filter=AMCRT"] + similarity_options + extra_args, env=env, silent_ok=True) # Added by Kristinn. if self.options.add_cache: diff += RunShell(cmd + ["--cached"], env=env, silent_ok=True) # The CL could be only file deletion or not. So accept silent diff for both # commands then check for an empty diff manually. if not diff: ErrorExit("No output from %s" % (cmd + extra_args)) return diff def GetUnknownFiles(self): status = RunShell(["git", "ls-files", "--exclude-standard", "--others"], silent_ok=True) return status.splitlines() def GetFileContent(self, file_hash, is_binary): """Returns the content of a file identified by its git hash.""" data, retcode = RunShellWithReturnCode(["git", "show", file_hash], universal_newlines=not is_binary) if retcode: ErrorExit("Got error status from 'git show %s'" % file_hash) return data def GetBaseFile(self, filename): hash_before, hash_after = self.hashes.get(filename, (None,None)) base_content = None new_content = None status = None if filename in self.renames: status = "A +" # Match svn attribute name for renames. if filename not in self.hashes: # If a rename doesn't change the content, we never get a hash. base_content = RunShell( ["git", "show", "HEAD:" + filename], silent_ok=True) elif not hash_before: status = "A" base_content = "" elif not hash_after: status = "D" else: status = "M" is_image = self.IsImage(filename) is_binary = self.IsBinaryData(base_content) or is_image # Grab the before/after content if we need it. # Grab the base content if we don't have it already. if base_content is None and hash_before: base_content = self.GetFileContent(hash_before, is_binary) # Only include the "after" file if it's an image; otherwise it # it is reconstructed from the diff. if is_image and hash_after: new_content = self.GetFileContent(hash_after, is_binary) return (base_content, new_content, is_binary, status) class CVSVCS(VersionControlSystem): """Implementation of the VersionControlSystem interface for CVS.""" def __init__(self, options): super(CVSVCS, self).__init__(options) def GetGUID(self): """For now we don't know how to get repository ID for CVS""" return def GetOriginalContent_(self, filename): RunShell(["cvs", "up", filename], silent_ok=True) # TODO need detect file content encoding content = open(filename).read() return content.replace("\r\n", "\n") def GetBaseFile(self, filename): base_content = None new_content = None status = "A" output, retcode = RunShellWithReturnCode(["cvs", "status", filename]) if retcode: ErrorExit("Got error status from 'cvs status %s'" % filename) if output.find("Status: Locally Modified") != -1: status = "M" temp_filename = "%s.tmp123" % filename os.rename(filename, temp_filename) base_content = self.GetOriginalContent_(filename) os.rename(temp_filename, filename) elif output.find("Status: Locally Added"): status = "A" base_content = "" elif output.find("Status: Needs Checkout"): status = "D" base_content = self.GetOriginalContent_(filename) return (base_content, new_content, self.IsBinaryData(base_content), status) def GenerateDiff(self, extra_args): cmd = ["cvs", "diff", "-u", "-N"] if self.options.revision: cmd += ["-r", self.options.revision] cmd.extend(extra_args) data, retcode = RunShellWithReturnCode(cmd) count = 0 if retcode in [0, 1]: for line in data.splitlines(): if line.startswith("Index:"): count += 1 logging.info(line) if not count: ErrorExit("No valid patches found in output from cvs diff") return data def GetUnknownFiles(self): data, retcode = RunShellWithReturnCode(["cvs", "diff"]) if retcode not in [0, 1]: ErrorExit("Got error status from 'cvs diff':\n%s" % (data,)) unknown_files = [] for line in data.split("\n"): if line and line[0] == "?": unknown_files.append(line) return unknown_files class MercurialVCS(VersionControlSystem): """Implementation of the VersionControlSystem interface for Mercurial.""" def __init__(self, options, repo_dir): super(MercurialVCS, self).__init__(options) # Absolute path to repository (we can be in a subdir) self.repo_dir = os.path.normpath(repo_dir) # Compute the subdir cwd = os.path.normpath(os.getcwd()) assert cwd.startswith(self.repo_dir) self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/") if self.options.revision: self.base_rev = self.options.revision else: self.base_rev = RunShell(["hg", "parent", "-q"]).split(':')[1].strip() def GetGUID(self): # See chapter "Uniquely identifying a repository" # http://hgbook.red-bean.com/read/customizing-the-output-of-mercurial.html info = RunShell("hg log -r0 --template {node}".split()) return info.strip() def _GetRelPath(self, filename): """Get relative path of a file according to the current directory, given its logical path in the repo.""" absname = os.path.join(self.repo_dir, filename) return os.path.relpath(absname) def GenerateDiff(self, extra_args): cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args data = RunShell(cmd, silent_ok=True) svndiff = [] filecount = 0 for line in data.splitlines(): m = re.match("diff --git a/(\S+) b/(\S+)", line) if m: # Modify line to make it look like as it comes from svn diff. # With this modification no changes on the server side are required # to make upload.py work with Mercurial repos. # NOTE: for proper handling of moved/copied files, we have to use # the second filename. filename = m.group(2) svndiff.append("Index: %s" % filename) svndiff.append("=" * 67) filecount += 1 logging.info(line) else: svndiff.append(line) if not filecount: ErrorExit("No valid patches found in output from hg diff") return "\n".join(svndiff) + "\n" def GetUnknownFiles(self): """Return a list of files unknown to the VCS.""" args = [] status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."], silent_ok=True) unknown_files = [] for line in status.splitlines(): st, fn = line.split(" ", 1) if st == "?": unknown_files.append(fn) return unknown_files def GetBaseFile(self, filename): # "hg status" and "hg cat" both take a path relative to the current subdir, # but "hg diff" has given us the path relative to the repo root. base_content = "" new_content = None is_binary = False oldrelpath = relpath = self._GetRelPath(filename) # "hg status -C" returns two lines for moved/copied files, one otherwise out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath]) out = out.splitlines() # HACK: strip error message about missing file/directory if it isn't in # the working copy if out[0].startswith('%s: ' % relpath): out = out[1:] status, _ = out[0].split(' ', 1) if len(out) > 1 and status == "A": # Moved/copied => considered as modified, use old filename to # retrieve base contents oldrelpath = out[1].strip() status = "M" if ":" in self.base_rev: base_rev = self.base_rev.split(":", 1)[0] else: base_rev = self.base_rev if status != "A": base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], silent_ok=True) is_binary = self.IsBinaryData(base_content) if status != "R": new_content = open(relpath, "rb").read() is_binary = is_binary or self.IsBinaryData(new_content) if is_binary and base_content: # Fetch again without converting newlines base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], silent_ok=True, universal_newlines=False) if not is_binary: new_content = None return base_content, new_content, is_binary, status class PerforceVCS(VersionControlSystem): """Implementation of the VersionControlSystem interface for Perforce.""" def __init__(self, options): def ConfirmLogin(): # Make sure we have a valid perforce session while True: data, retcode = self.RunPerforceCommandWithReturnCode( ["login", "-s"], marshal_output=True) if not data: ErrorExit("Error checking perforce login") if not retcode and (not "code" in data or data["code"] != "error"): break print "Enter perforce password: " self.RunPerforceCommandWithReturnCode(["login"]) super(PerforceVCS, self).__init__(options) self.p4_changelist = options.p4_changelist if not self.p4_changelist: ErrorExit("A changelist id is required") if (options.revision): ErrorExit("--rev is not supported for perforce") self.p4_port = options.p4_port self.p4_client = options.p4_client self.p4_user = options.p4_user ConfirmLogin() if not options.title: description = self.RunPerforceCommand(["describe", self.p4_changelist], marshal_output=True) if description and "desc" in description: # Rietveld doesn't support multi-line descriptions raw_title = description["desc"].strip() lines = raw_title.splitlines() if len(lines): options.title = lines[0] def GetGUID(self): """For now we don't know how to get repository ID for Perforce""" return def RunPerforceCommandWithReturnCode(self, extra_args, marshal_output=False, universal_newlines=True): args = ["p4"] if marshal_output: # -G makes perforce format its output as marshalled python objects args.extend(["-G"]) if self.p4_port: args.extend(["-p", self.p4_port]) if self.p4_client: args.extend(["-c", self.p4_client]) if self.p4_user: args.extend(["-u", self.p4_user]) args.extend(extra_args) data, retcode = RunShellWithReturnCode( args, print_output=False, universal_newlines=universal_newlines) if marshal_output and data: data = marshal.loads(data) return data, retcode def RunPerforceCommand(self, extra_args, marshal_output=False, universal_newlines=True): # This might be a good place to cache call results, since things like # describe or fstat might get called repeatedly. data, retcode = self.RunPerforceCommandWithReturnCode( extra_args, marshal_output, universal_newlines) if retcode: ErrorExit("Got error status from %s:\n%s" % (extra_args, data)) return data def GetFileProperties(self, property_key_prefix = "", command = "describe"): description = self.RunPerforceCommand(["describe", self.p4_changelist], marshal_output=True) changed_files = {} file_index = 0 # Try depotFile0, depotFile1, ... until we don't find a match while True: file_key = "depotFile%d" % file_index if file_key in description: filename = description[file_key] change_type = description[property_key_prefix + str(file_index)] changed_files[filename] = change_type file_index += 1 else: break return changed_files def GetChangedFiles(self): return self.GetFileProperties("action") def GetUnknownFiles(self): # Perforce doesn't detect new files, they have to be explicitly added return [] def IsBaseBinary(self, filename): base_filename = self.GetBaseFilename(filename) return self.IsBinaryHelper(base_filename, "files") def IsPendingBinary(self, filename): return self.IsBinaryHelper(filename, "describe") def IsBinaryHelper(self, filename, command): file_types = self.GetFileProperties("type", command) if not filename in file_types: ErrorExit("Trying to check binary status of unknown file %s." % filename) # This treats symlinks, macintosh resource files, temporary objects, and # unicode as binary. See the Perforce docs for more details: # http://www.perforce.com/perforce/doc.current/manuals/cmdref/o.ftypes.html return not file_types[filename].endswith("text") def GetFileContent(self, filename, revision, is_binary): file_arg = filename if revision: file_arg += "#" + revision # -q suppresses the initial line that displays the filename and revision return self.RunPerforceCommand(["print", "-q", file_arg], universal_newlines=not is_binary) def GetBaseFilename(self, filename): actionsWithDifferentBases = [ "move/add", # p4 move "branch", # p4 integrate (to a new file), similar to hg "add" "add", # p4 integrate (to a new file), after modifying the new file ] # We only see a different base for "add" if this is a downgraded branch # after a file was branched (integrated), then edited. if self.GetAction(filename) in actionsWithDifferentBases: # -Or shows information about pending integrations/moves fstat_result = self.RunPerforceCommand(["fstat", "-Or", filename], marshal_output=True) baseFileKey = "resolveFromFile0" # I think it's safe to use only file0 if baseFileKey in fstat_result: return fstat_result[baseFileKey] return filename def GetBaseRevision(self, filename): base_filename = self.GetBaseFilename(filename) have_result = self.RunPerforceCommand(["have", base_filename], marshal_output=True) if "haveRev" in have_result: return have_result["haveRev"] def GetLocalFilename(self, filename): where = self.RunPerforceCommand(["where", filename], marshal_output=True) if "path" in where: return where["path"] def GenerateDiff(self, args): class DiffData: def __init__(self, perforceVCS, filename, action): self.perforceVCS = perforceVCS self.filename = filename self.action = action self.base_filename = perforceVCS.GetBaseFilename(filename) self.file_body = None self.base_rev = None self.prefix = None self.working_copy = True self.change_summary = None def GenerateDiffHeader(diffData): header = [] header.append("Index: %s" % diffData.filename) header.append("=" * 67) if diffData.base_filename != diffData.filename: if diffData.action.startswith("move"): verb = "rename" else: verb = "copy" header.append("%s from %s" % (verb, diffData.base_filename)) header.append("%s to %s" % (verb, diffData.filename)) suffix = "\t(revision %s)" % diffData.base_rev header.append("--- " + diffData.base_filename + suffix) if diffData.working_copy: suffix = "\t(working copy)" header.append("+++ " + diffData.filename + suffix) if diffData.change_summary: header.append(diffData.change_summary) return header def GenerateMergeDiff(diffData, args): # -du generates a unified diff, which is nearly svn format diffData.file_body = self.RunPerforceCommand( ["diff", "-du", diffData.filename] + args) diffData.base_rev = self.GetBaseRevision(diffData.filename) diffData.prefix = "" # We have to replace p4's file status output (the lines starting # with +++ or ---) to match svn's diff format lines = diffData.file_body.splitlines() first_good_line = 0 while (first_good_line < len(lines) and not lines[first_good_line].startswith("@@")): first_good_line += 1 diffData.file_body = "\n".join(lines[first_good_line:]) return diffData def GenerateAddDiff(diffData): fstat = self.RunPerforceCommand(["fstat", diffData.filename], marshal_output=True) if "headRev" in fstat: diffData.base_rev = fstat["headRev"] # Re-adding a deleted file else: diffData.base_rev = "0" # Brand new file diffData.working_copy = False rel_path = self.GetLocalFilename(diffData.filename) diffData.file_body = open(rel_path, 'r').read() # Replicate svn's list of changed lines line_count = len(diffData.file_body.splitlines()) diffData.change_summary = "@@ -0,0 +1" if line_count > 1: diffData.change_summary += ",%d" % line_count diffData.change_summary += " @@" diffData.prefix = "+" return diffData def GenerateDeleteDiff(diffData): diffData.base_rev = self.GetBaseRevision(diffData.filename) is_base_binary = self.IsBaseBinary(diffData.filename) # For deletes, base_filename == filename diffData.file_body = self.GetFileContent(diffData.base_filename, None, is_base_binary) # Replicate svn's list of changed lines line_count = len(diffData.file_body.splitlines()) diffData.change_summary = "@@ -1" if line_count > 1: diffData.change_summary += ",%d" % line_count diffData.change_summary += " +0,0 @@" diffData.prefix = "-" return diffData changed_files = self.GetChangedFiles() svndiff = [] filecount = 0 for (filename, action) in changed_files.items(): svn_status = self.PerforceActionToSvnStatus(action) if svn_status == "SKIP": continue diffData = DiffData(self, filename, action) # Is it possible to diff a branched file? Stackoverflow says no: # http://stackoverflow.com/questions/1771314/in-perforce-command-line-how-to-diff-a-file-reopened-for-add if svn_status == "M": diffData = GenerateMergeDiff(diffData, args) elif svn_status == "A": diffData = GenerateAddDiff(diffData) elif svn_status == "D": diffData = GenerateDeleteDiff(diffData) else: ErrorExit("Unknown file action %s (svn action %s)." % \ (action, svn_status)) svndiff += GenerateDiffHeader(diffData) for line in diffData.file_body.splitlines(): svndiff.append(diffData.prefix + line) filecount += 1 if not filecount: ErrorExit("No valid patches found in output from p4 diff") return "\n".join(svndiff) + "\n" def PerforceActionToSvnStatus(self, status): # Mirroring the list at http://permalink.gmane.org/gmane.comp.version-control.mercurial.devel/28717 # Is there something more official? return { "add" : "A", "branch" : "A", "delete" : "D", "edit" : "M", # Also includes changing file types. "integrate" : "M", "move/add" : "M", "move/delete": "SKIP", "purge" : "D", # How does a file's status become "purge"? }[status] def GetAction(self, filename): changed_files = self.GetChangedFiles() if not filename in changed_files: ErrorExit("Trying to get base version of unknown file %s." % filename) return changed_files[filename] def GetBaseFile(self, filename): base_filename = self.GetBaseFilename(filename) base_content = "" new_content = None status = self.PerforceActionToSvnStatus(self.GetAction(filename)) if status != "A": revision = self.GetBaseRevision(base_filename) if not revision: ErrorExit("Couldn't find base revision for file %s" % filename) is_base_binary = self.IsBaseBinary(base_filename) base_content = self.GetFileContent(base_filename, revision, is_base_binary) is_binary = self.IsPendingBinary(filename) if status != "D" and status != "SKIP": relpath = self.GetLocalFilename(filename) if is_binary: new_content = open(relpath, "rb").read() return base_content, new_content, is_binary, status # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync. def SplitPatch(data): """Splits a patch into separate pieces for each file. Args: data: A string containing the output of svn diff. Returns: A list of 2-tuple (filename, text) where text is the svn diff output pertaining to filename. """ patches = [] filename = None diff = [] for line in data.splitlines(True): new_filename = None if line.startswith('Index:'): unused, new_filename = line.split(':', 1) new_filename = new_filename.strip() elif line.startswith('Property changes on:'): unused, temp_filename = line.split(':', 1) # When a file is modified, paths use '/' between directories, however # when a property is modified '\' is used on Windows. Make them the same # otherwise the file shows up twice. temp_filename = temp_filename.strip().replace('\\', '/') if temp_filename != filename: # File has property changes but no modifications, create a new diff. new_filename = temp_filename if new_filename: if filename and diff: patches.append((filename, ''.join(diff))) filename = new_filename diff = [line] continue if diff is not None: diff.append(line) if filename and diff: patches.append((filename, ''.join(diff))) return patches def UploadSeparatePatches(issue, rpc_server, patchset, data, options): """Uploads a separate patch for each file in the diff output. Returns a list of [patch_key, filename] for each file. """ patches = SplitPatch(data) rv = [] for patch in patches: if len(patch[1]) > MAX_UPLOAD_SIZE: print ("Not uploading the patch for " + patch[0] + " because the file is too large.") continue form_fields = [("filename", patch[0])] if not options.download_base: form_fields.append(("content_upload", "1")) files = [("data", "data.diff", patch[1])] ctype, body = EncodeMultipartFormData(form_fields, files) url = "/%d/upload_patch/%d" % (int(issue), int(patchset)) print "Uploading patch for " + patch[0] response_body = rpc_server.Send(url, body, content_type=ctype) lines = response_body.splitlines() if not lines or lines[0] != "OK": StatusUpdate(" --> %s" % response_body) sys.exit(1) rv.append([lines[1], patch[0]]) return rv def GuessVCSName(options): """Helper to guess the version control system. This examines the current directory, guesses which VersionControlSystem we're using, and returns an string indicating which VCS is detected. Returns: A pair (vcs, output). vcs is a string indicating which VCS was detected and is one of VCS_GIT, VCS_MERCURIAL, VCS_SUBVERSION, VCS_PERFORCE, VCS_CVS, or VCS_UNKNOWN. Since local perforce repositories can't be easily detected, this method will only guess VCS_PERFORCE if any perforce options have been specified. output is a string containing any interesting output from the vcs detection routine, or None if there is nothing interesting. """ for attribute, value in options.__dict__.iteritems(): if attribute.startswith("p4") and value != None: return (VCS_PERFORCE, None) def RunDetectCommand(vcs_type, command): """Helper to detect VCS by executing command. Returns: A pair (vcs, output) or None. Throws exception on error. """ try: out, returncode = RunShellWithReturnCode(command) if returncode == 0: return (vcs_type, out.strip()) except OSError, (errcode, message): if errcode != errno.ENOENT: # command not found code raise # Mercurial has a command to get the base directory of a repository # Try running it, but don't die if we don't have hg installed. # NOTE: we try Mercurial first as it can sit on top of an SVN working copy. res = RunDetectCommand(VCS_MERCURIAL, ["hg", "root"]) if res != None: return res # Subversion from 1.7 has a single centralized .svn folder # ( see http://subversion.apache.org/docs/release-notes/1.7.html#wc-ng ) # That's why we use 'svn info' instead of checking for .svn dir res = RunDetectCommand(VCS_SUBVERSION, ["svn", "info"]) if res != None: return res # Git has a command to test if you're in a git tree. # Try running it, but don't die if we don't have git installed. res = RunDetectCommand(VCS_GIT, ["git", "rev-parse", "--is-inside-work-tree"]) if res != None: return res # detect CVS repos use `cvs status && $? == 0` rules res = RunDetectCommand(VCS_CVS, ["cvs", "status"]) if res != None: return res return (VCS_UNKNOWN, None) def GuessVCS(options): """Helper to guess the version control system. This verifies any user-specified VersionControlSystem (by command line or environment variable). If the user didn't specify one, this examines the current directory, guesses which VersionControlSystem we're using, and returns an instance of the appropriate class. Exit with an error if we can't figure it out. Returns: A VersionControlSystem instance. Exits if the VCS can't be guessed. """ vcs = options.vcs if not vcs: vcs = os.environ.get("CODEREVIEW_VCS") if vcs: v = VCS_ABBREVIATIONS.get(vcs.lower()) if v is None: ErrorExit("Unknown version control system %r specified." % vcs) (vcs, extra_output) = (v, None) else: (vcs, extra_output) = GuessVCSName(options) if vcs == VCS_MERCURIAL: if extra_output is None: extra_output = RunShell(["hg", "root"]).strip() return MercurialVCS(options, extra_output) elif vcs == VCS_SUBVERSION: return SubversionVCS(options) elif vcs == VCS_PERFORCE: return PerforceVCS(options) elif vcs == VCS_GIT: return GitVCS(options) elif vcs == VCS_CVS: return CVSVCS(options) ErrorExit(("Could not guess version control system. " "Are you in a working copy directory?")) def CheckReviewer(reviewer): """Validate a reviewer -- either a nickname or an email addres. Args: reviewer: A nickname or an email address. Calls ErrorExit() if it is an invalid email address. """ if "@" not in reviewer: return # Assume nickname parts = reviewer.split("@") if len(parts) > 2: ErrorExit("Invalid email address: %r" % reviewer) assert len(parts) == 2 if "." not in parts[1]: ErrorExit("Invalid email address: %r" % reviewer) def LoadSubversionAutoProperties(): """Returns the content of [auto-props] section of Subversion's config file as a dictionary. Returns: A dictionary whose key-value pair corresponds the [auto-props] section's key-value pair. In following cases, returns empty dictionary: - config file doesn't exist, or - 'enable-auto-props' is not set to 'true-like-value' in [miscellany]. """ if os.name == 'nt': subversion_config = os.environ.get("APPDATA") + "\\Subversion\\config" else: subversion_config = os.path.expanduser("~/.subversion/config") if not os.path.exists(subversion_config): return {} config = ConfigParser.ConfigParser() config.read(subversion_config) if (config.has_section("miscellany") and config.has_option("miscellany", "enable-auto-props") and config.getboolean("miscellany", "enable-auto-props") and config.has_section("auto-props")): props = {} for file_pattern in config.options("auto-props"): props[file_pattern] = ParseSubversionPropertyValues( config.get("auto-props", file_pattern)) return props else: return {} def ParseSubversionPropertyValues(props): """Parse the given property value which comes from [auto-props] section and returns a list whose element is a (svn_prop_key, svn_prop_value) pair. See the following doctest for example. >>> ParseSubversionPropertyValues('svn:eol-style=LF') [('svn:eol-style', 'LF')] >>> ParseSubversionPropertyValues('svn:mime-type=image/jpeg') [('svn:mime-type', 'image/jpeg')] >>> ParseSubversionPropertyValues('svn:eol-style=LF;svn:executable') [('svn:eol-style', 'LF'), ('svn:executable', '*')] """ key_value_pairs = [] for prop in props.split(";"): key_value = prop.split("=") assert len(key_value) <= 2 if len(key_value) == 1: # If value is not given, use '*' as a Subversion's convention. key_value_pairs.append((key_value[0], "*")) else: key_value_pairs.append((key_value[0], key_value[1])) return key_value_pairs def GetSubversionPropertyChanges(filename): """Return a Subversion's 'Property changes on ...' string, which is used in the patch file. Args: filename: filename whose property might be set by [auto-props] config. Returns: A string like 'Property changes on |filename| ...' if given |filename| matches any entries in [auto-props] section. None, otherwise. """ global svn_auto_props_map if svn_auto_props_map is None: svn_auto_props_map = LoadSubversionAutoProperties() all_props = [] for file_pattern, props in svn_auto_props_map.items(): if fnmatch.fnmatch(filename, file_pattern): all_props.extend(props) if all_props: return FormatSubversionPropertyChanges(filename, all_props) return None def FormatSubversionPropertyChanges(filename, props): """Returns Subversion's 'Property changes on ...' strings using given filename and properties. Args: filename: filename props: A list whose element is a (svn_prop_key, svn_prop_value) pair. Returns: A string which can be used in the patch file for Subversion. See the following doctest for example. >>> print FormatSubversionPropertyChanges('foo.cc', [('svn:eol-style', 'LF')]) Property changes on: foo.cc ___________________________________________________________________ Added: svn:eol-style + LF """ prop_changes_lines = [ "Property changes on: %s" % filename, "___________________________________________________________________"] for key, value in props: prop_changes_lines.append("Added: " + key) prop_changes_lines.append(" + " + value) return "\n".join(prop_changes_lines) + "\n" def RealMain(argv, data=None): """The real main function. Args: argv: Command line arguments. data: Diff contents. If None (default) the diff is generated by the VersionControlSystem implementation returned by GuessVCS(). Returns: A 2-tuple (issue id, patchset id). The patchset id is None if the base files are not uploaded by this script (applies only to SVN checkouts). """ options, args = parser.parse_args(argv[1:]) if options.help: if options.verbose < 2: # hide Perforce options parser.epilog = ( "Use '--help -v' to show additional Perforce options. " "For more help, see " "http://code.google.com/p/rietveld/wiki/CodeReviewHelp" ) parser.option_groups.remove(parser.get_option_group('--p4_port')) parser.print_help() sys.exit(0) global verbosity verbosity = options.verbose if verbosity >= 3: logging.getLogger().setLevel(logging.DEBUG) elif verbosity >= 2: logging.getLogger().setLevel(logging.INFO) vcs = GuessVCS(options) base = options.base_url if isinstance(vcs, SubversionVCS): # Guessing the base field is only supported for Subversion. # Note: Fetching base files may become deprecated in future releases. guessed_base = vcs.GuessBase(options.download_base) if base: if guessed_base and base != guessed_base: print "Using base URL \"%s\" from --base_url instead of \"%s\"" % \ (base, guessed_base) else: base = guessed_base if not base and options.download_base: options.download_base = True logging.info("Enabled upload of base file") if not options.assume_yes: vcs.CheckForUnknownFiles() if data is None: data = vcs.GenerateDiff(args) data = vcs.PostProcessDiff(data) if options.print_diffs: print "Rietveld diff start:*****" print data print "Rietveld diff end:*****" files = vcs.GetBaseFiles(data) if verbosity >= 1: print "Upload server:", options.server, "(change with -s/--server)" if options.use_oauth2: options.save_cookies = False rpc_server = GetRpcServer(options.server, options.email, options.host, options.save_cookies, options.account_type, options.use_oauth2, options.oauth2_port, options.open_oauth2_local_webbrowser) form_fields = [] repo_guid = vcs.GetGUID() if repo_guid: form_fields.append(("repo_guid", repo_guid)) if base: b = urlparse.urlparse(base) username, netloc = urllib.splituser(b.netloc) if username: logging.info("Removed username from base URL") base = urlparse.urlunparse((b.scheme, netloc, b.path, b.params, b.query, b.fragment)) form_fields.append(("base", base)) if options.issue: form_fields.append(("issue", str(options.issue))) if options.email: form_fields.append(("user", options.email)) if options.reviewers: for reviewer in options.reviewers.split(','): CheckReviewer(reviewer) form_fields.append(("reviewers", options.reviewers)) if options.cc: for cc in options.cc.split(','): CheckReviewer(cc) form_fields.append(("cc", options.cc)) # Process --message, --title and --file. message = options.message or "" title = options.title or "" if options.file: if options.message: ErrorExit("Can't specify both message and message file options") file = open(options.file, 'r') message = file.read() file.close() if options.issue: prompt = "Title describing this patch set: " else: prompt = "New issue subject: " title = ( title or message.split('\n', 1)[0].strip() or raw_input(prompt).strip()) if not title and not options.issue: ErrorExit("A non-empty title is required for a new issue") # For existing issues, it's fine to give a patchset an empty name. Rietveld # doesn't accept that so use a whitespace. title = title or " " if len(title) > 100: title = title[:99] + '…' if title and not options.issue: message = message or title form_fields.append(("subject", title)) # If it's a new issue send message as description. Otherwise a new # message is created below on upload_complete. if message and not options.issue: form_fields.append(("description", message)) # Send a hash of all the base file so the server can determine if a copy # already exists in an earlier patchset. base_hashes = "" for file, info in files.iteritems(): if not info[0] is None: checksum = md5(info[0]).hexdigest() if base_hashes: base_hashes += "|" base_hashes += checksum + ":" + file form_fields.append(("base_hashes", base_hashes)) if options.private: if options.issue: print "Warning: Private flag ignored when updating an existing issue." else: form_fields.append(("private", "1")) if options.send_patch: options.send_mail = True if not options.download_base: form_fields.append(("content_upload", "1")) if len(data) > MAX_UPLOAD_SIZE: print "Patch is large, so uploading file patches separately." uploaded_diff_file = [] form_fields.append(("separate_patches", "1")) else: uploaded_diff_file = [("data", "data.diff", data)] ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file) response_body = rpc_server.Send("/upload", body, content_type=ctype) patchset = None if not options.download_base or not uploaded_diff_file: lines = response_body.splitlines() if len(lines) >= 2: msg = lines[0] patchset = lines[1].strip() patches = [x.split(" ", 1) for x in lines[2:]] else: msg = response_body else: msg = response_body StatusUpdate(msg) if not response_body.startswith("Issue created.") and \ not response_body.startswith("Issue updated."): sys.exit(0) issue = msg[msg.rfind("/")+1:] if not uploaded_diff_file: result = UploadSeparatePatches(issue, rpc_server, patchset, data, options) if not options.download_base: patches = result if not options.download_base: vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files) payload = {} # payload for final request if options.send_mail: payload["send_mail"] = "yes" if options.send_patch: payload["attach_patch"] = "yes" if options.issue and message: payload["message"] = message payload = urllib.urlencode(payload) rpc_server.Send("/" + issue + "/upload_complete/" + (patchset or ""), payload=payload) return issue, patchset def main(): try: logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:" "%(lineno)s %(message)s ")) os.environ['LC_ALL'] = 'C' RealMain(sys.argv) except KeyboardInterrupt: print StatusUpdate("Interrupted.") sys.exit(1) if __name__ == "__main__": main()