svn-load-1.3/0000755000175000017500000000000011536457535012054 5ustar dannfdannfsvn-load-1.3/svn-load0000755000175000017500000006034411535502765013527 0ustar dannfdannf#!/usr/bin/python # # Copyright (c) 2007-2008 Hewlett-Packard Development Company, L.P. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above # copyright notice, this list of conditions and the following # disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials # provided with the distribution. # # * Neither the name of the Hewlett-Packard Co. nor the names # of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY HEWLETT-PACKARD DEVELOPMENT COMPANY # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL # HEWLETT-PACKARD DEVELOPMENT COMPANY BE LIABLE FOR ANY DIRECT, # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # from optparse import OptionParser import sys import os.path, shutil, stat import tempfile import pysvn import re import types import getpass class NotifiedClient: def __init__(self, username, password): self.client = pysvn.Client() self.client.callback_notify = self.notify self.client.callback_ssl_client_cert_password_prompt = self.ssl_client_cert_password_prompt self.client.callback_get_login = self.get_login self.client.callback_ssl_client_cert_prompt = self.ssl_client_cert_prompt self.client.callback_ssl_server_prompt = self.ssl_server_prompt self.client.callback_ssl_server_trust_prompt = self.ssl_server_trust_prompt self.username = username self.password = password def notify(self, event_dict): if event_dict['action'] == pysvn.wc_notify_action.delete: sys.stdout.write("Removing %s\n" % (event_dict['path'])) elif event_dict['action'] == pysvn.wc_notify_action.add: sys.stdout.write("Adding %s\n" % (event_dict['path'])) elif event_dict['action'] == pysvn.wc_notify_action.copy: sys.stdout.write("Copying %s\n" % (event_dict['path'])) def ssl_client_cert_password_prompt(realm, may_save): return True, getpass.getpass("Passphrase for '%s': " % (realm)), False def get_login(realm, username, may_save): if not self.password: self.password = raw_input("Password for %s (svn): " % username) return (True, username, password, False) ## pysvn supports a number of callbacks for scenarios I've yet to ## encounter. For now, just emit a warning to hopefully clue the user ## in about what went wrong - maybe they'll send a patch! :) def nocallback(): sys.stderr.write("Warning: Unimplemented callback: %s\n" % (sys._getframe(1).f_code.co_name)) def ssl_client_cert_prompt(realm, may_save): nocallback() def ssl_server_prompt(): nocallback() def ssl_server_trust_prompt(trust_dict): nocallback() class TagClient: def __init__(self, client, url, svndir, tagdir): self.url = url self.svndir = svndir self.tagdir = tagdir self.message = "Tag %s as %s" % (svndir, tagdir) self.svnclient = client self.svnclient.callback_get_log_message = self.get_tag_message def get_tag_message(self): return True, self.message def doIt(self): self.svnclient.copy(os.path.join(self.url, self.svndir), os.path.join(self.url, self.tagdir)) class MoveMenu(object): """A menu allowing the user to indicate whether deleted/added files were moved, to conserve space in the repository """ def __init__(self, workingdir, newdir, moved=None, interactive=True): """Create a MoveMenu instance Params: workingdir: temporary working directory newdir: directory to load moved: If not None, should be a regex-indexed dictionary of functions of m (an re.match object), which map deleted files to added files, e.g. moved = {re.compile('^src/(?P\S+)\.gif') : lambda m: 'images/%s.gif' % m.group('name')} which maps any file ending with .gif under 'src' to a file of the same name under 'images'. It's easy enough to do one-to-one mappings, as well: moved = {re.compile('^foo/bar$') : lambda m: 'bar/baz'} interactive: Menus are only actually displayed if this is True """ self.workingdir = workingdir self.newdir = newdir self.moved = moved self.interactive = interactive # make these variables in case of future localization self.delcolhead = 'Deleted' self.addcolhead = 'Added' self.deleted = unique_nodes(workingdir, newdir) self.added = unique_nodes(newdir, workingdir) self.menu_pair = re.compile("^(?P\d+)\s+(?P\d+)$") def go(self): """Go through all the differences and perform requested operations""" while self.deleted and self.added: # The deleted column should be as wide as the longest path # in that column or the length of the 'Deleted' string, whichever # is greater self.delcollen = max([len(i) for i in self.deleted] + [len(self.delcolhead)]) + 1 keep_going, answer = self.prompt() if not keep_going: break if not answer: continue else: srcindex, destindex = answer if srcindex >= len(self.deleted) or destindex >= len(self.added): sys.stderr.write("Error: Invalid index.\n") continue src = self.deleted[srcindex] dest = self.added[destindex] move_node(client, self.workingdir, src, dest) del self.deleted[srcindex] # If we moved a node into a subtree that didn't yet exist, # then move_node politely created it for us. That was nice of her. # Let's remove those directories from our 'added' list - we can't # move a directory to a name that already exists. head = dest while head: if head in self.added: self.added.remove(head) (head, tail) = os.path.split(head) # If we just moved a directory, its subtree went with it and # can't move again. Remove subtree nodes from the deleted list so # the user can't try it. If this proves to be a desired feature, # we'll need to do multiple commits. Otherwise, users should # move subtree components first, and then move the whole directory if os.path.isdir(os.path.join(workingdir, dest)): i = 0 while i < len(self.deleted): if self.deleted[i][:len(src)+1] == src + '/': self.deleted.pop(i) else: i = i + 1 def generate_header(self): """Generate the header line, returning it as a string""" delcollen = max(self.delcollen, len(self.delcolhead)) header = " " * 5 header = header + self.delcolhead header = header + (delcollen - len(self.delcolhead) + 1) * " " header = header + self.addcolhead + "\n" return header def generate_row(self, num, delfile, addfile): """Return a string for a row""" deleted = delfile + '_' * (self.delcollen - len(delfile) - 1) return("%4d %s %s\n" % (num, deleted, addfile)) def display(self): """Display the menu, row-by-row""" for i in range(max([len(self.deleted), len(self.added)])): delcell = "" if len(self.deleted) > i: delcell = self.deleted[i] addcell = "" if len(self.added) > i: addcell = self.added[i] row = self.generate_row(i, delcell, addcell) sys.stdout.write(row) def _prompt(): """Return a prompt as a string""" return("Enter two indexes for each column to rename, (R)elist, or (F)inish: ") _prompt = staticmethod(_prompt) def prompt(self): """Prompt the user for an answer, and return an answer tuple If self.moved is set, mappings from there are exhausted before resorting to asking the user. Return value is of the form (continue_listing, answer_tuple). Cases: If user answers "F", then continue_listing is False, and answer_tuple is None. If user answers "R" or the input is invalid, then continue_listing is True, and answer_tuple is None. If user answers with a valid tuple (or one is found using self.moved), then continue_listing is True, and answer_tuple is the tuple of indices If self.interactive is False, no prompts are printed. """ ask = self.interactive ret = (False, None) if self.moved is not None: # Check in the dictionary mapping deleted files to added files found = False for node in self.deleted: for pattern, func in self.moved.items(): m = pattern.match(node) if m: dest = func(m) try: destindex = self.added.index(dest) except ValueError: # This file was probably deleted. Let's let other # patterns try to match it before giving up. continue else: srcindex = self.deleted.index(node) ret = (True, (srcindex, destindex)) found = True break if found: ask = False else: # We've exhausted our mapping if self.interactive: ask = True else: ask = False ret = (False, None) if ask: # Ask the user header = self.generate_header() sys.stdout.write(header) self.display() prompt = self._prompt() sys.stdout.write(prompt) input = sys.stdin.readline()[:-1] if input in ['r', 'R']: ret = (True, None) elif input in ['f', 'F']: ret = (False, None) else: m = self.menu_pair.match(input) if m: srcindex = int(m.group('src')) destindex = int(m.group('dest')) ret = (True, (srcindex, destindex)) else: sys.stderr.write("Error: Invalid input.\n") ret = (True, None) return ret ## ## Check to see if a node (path) exists. If so, returns an entry oject for it. ## def svn_path_exists(client, svn_url, svn_dir): try: entry = client.info2(os.path.join(svn_url, svn_dir), recurse = False)[0] return entry except pysvn._pysvn.ClientError: return None ## ## Create a directory in svn (and any parents, if necesary) ## def make_svn_dirs(client, svn_url, svn_import_dir): entry = svn_path_exists(client, svn_url, svn_import_dir) if entry: if entry[1]['kind'] == pysvn.node_kind.dir: return True else: sys.stderr.write("\nError: %s exists but is not a directory.\n\n" % (svn_import_dir)) raise pysvn.ClientError else: make_svn_dirs(client, svn_url, os.path.dirname(svn_import_dir)) client.mkdir(os.path.join(svn_url, svn_import_dir), "Creating directory for import") def contains_svn_metadata(dir): for root, dirs, files in os.walk(dir): if '.svn' in dirs or '.svn' in files: return True return False ## ## Checkout an svn dir to a temporary directory, and return that directory ## def checkout_to_temp(client, svn_url, svn_dir): workingdir = tempfile.mkdtemp(prefix="svn-load") client.checkout(os.path.join(svn_url, svn_dir), os.path.join(workingdir, 'working')) return workingdir ## ## return a list of files that exist only in dir1 ## def unique_nodes(dir1, dir2): unique = [] for root, dirs, files in os.walk(dir1): if '.svn' in dirs: dirs.remove('.svn') for path in files + dirs: relpath = os.path.join(root, path)[len(dir1)+1:] counterpath = os.path.join(dir2, relpath) if not os.path.lexists(counterpath): unique.append(relpath) return unique def move_node(client, workingdir, src, dest): make_svn_dirs(client, "", os.path.dirname(os.path.join(workingdir, dest))) client.move(os.path.join(workingdir, src), os.path.join(workingdir, dest)) # ## Clear out the removed files # shutil.rmtree(os.path.join(workingdir, src)) def remove_nodes(client, workingdir, newdir): dellist = unique_nodes(workingdir, newdir) fqdellist = [ os.path.join(workingdir, p) for p in dellist ] client.remove(fqdellist) ## ## Overlay the new tree on top of our working directory, adding any ## new nodes along the way ## def overlay_files(client, workingdir, newdir): for root, dirs, files in os.walk(newdir): # treat links to directories as files so that # we create a link instead of duplicating a subtree for d in dirs: if os.path.islink(os.path.join(root, d)): files.append(d) dirs.remove(d) for f in files: fullpath = os.path.join(root, f) relpath = fullpath[len(newdir)+1:] counterpath = os.path.join(workingdir, relpath) if os.path.isdir(counterpath) and not os.path.islink(counterpath): sys.stderr.write("Can't replace directory %s with file %s.\n" % (counterpath, fullpath)) return False needs_add = False if not os.path.lexists(counterpath): needs_add = True # shutil.copy follows symlinks, so we need to handle them # separately if os.path.lexists(counterpath) and \ (os.path.islink(counterpath) or os.path.islink(fullpath)): os.unlink(counterpath) if os.path.islink(fullpath): os.symlink(os.readlink(fullpath), counterpath) else: shutil.copy(fullpath, counterpath) if needs_add: client.add(counterpath, ignore=False) # Force accurate symlink settings if os.path.islink(counterpath): client.propset('svn:special', '*', counterpath) else: client.propdel("svn:special", counterpath) # We have to use a counter instead of something like 'for d in dirs' # because we might be removing elements - removing elements in an # iterator causes us to skip over some i = 0 while i < len(dirs): fullpath = os.path.join(root, dirs[i]) relpath = fullpath[len(newdir)+1:] counterpath = os.path.join(workingdir, relpath) if not os.path.exists(counterpath): shutil.copytree(fullpath, counterpath, symlinks=True) client.add(counterpath, ignore=False) dirs.pop(i) continue if not os.path.isdir(counterpath): sys.stderr.write("Can't replace file %s with dir %s.\n" % (counterpath, fullpath)) return False i = i + 1 ## treats u+x as the canonical decider for svn:executable ## should probably see if svn import does it differently.. def is_executable(f): if os.path.islink(f): return False s = os.lstat(f) return s[stat.ST_MODE] & 0500 == 0500 def svn_is_executable(client, file): for path, prop_list in client.proplist(file): if prop_list.has_key('svn:executable'): return True return False def svn_set_exec(client, file): client.propset('svn:executable', '*', file) def svn_clear_exec(client, file): client.propdel('svn:executable', file) def sync_exec_flags(client, workingdir): for root, dirs, files in os.walk(workingdir): if '.svn' in dirs: dirs.remove('.svn') for f in files: path = os.path.join(root, f) if is_executable(path) and not svn_is_executable(client, path): svn_set_exec(client, path) if not is_executable(path) and svn_is_executable(client, path): svn_clear_exec(client, path) def strip_slashes(path): path = os.path.normpath(path) while os.path.isabs(path): path = path[1:] return path def expand_dirs(dirs): ## Globs get expanded by the application on windows if sys.platform == 'win32': import glob newdirs = [] for d in dirs: __dirs = glob.glob(d) __dirs.sort(lambda x, y: cmp(x.lower(), y.lower())) newdirs.extend(__dirs) return newdirs else: return dirs def parse_move_map(filename): """Read in mappings from filename, return a dictionary Example file entries: ^src/(?P\S+)\.gif$ lambda m: "images/%s.gif" % m.group("name") ^foo/bar$ "bar/baz" Essentially, the first field must be a pattern that explicitly matches ^ and $. The second field (separated by whitespace) is a lambda function of one variable, of the type returned by re.match(). Alternately, the second field can be an explicit string--in the second example, the following function would be constructed automatically: lambda m: "bar/baz" If you specify an explicit string, it must be enclosed in quotes. After parsing, return a dictionary which maps the objects returned by re.compile() to the lambda functions of their match objects. """ f = open(filename, 'r') map = {} for line in f: if not line.strip(): # Ignore blanks continue if not line.startswith('^'): sys.stderr.write("Error: Regular expression in map must explicitly " "match ^ and $\n") sys.exit(1) keep_searching = True pos = 0 while keep_searching: pos = line.find(r'$', pos+1) if pos == -1: sys.stderr.write("Error: Regular expression in map must " "explicitly match ^ and $\n") sys.exit(1) elif line[pos-1] == '\\': # Escaped, so keep looking! continue else: # Found the end keep_searching = False pattern = re.compile(line[:pos+1]) rest = line[pos+1:].strip() # Evaluate with nothing in the scope value = eval(rest, {}, {}) # If it's actually a string, let's turn it into a trivial lambda if type(value) == types.StringType: func = lambda m, v=value: v elif type(value) != types.FunctionType: sys.stderr.write("Error: right field in map must be a lambda or " "a string\n") sys.exit(1) else: func = value map[pattern] = func f.close() return map if __name__ == '__main__': usage = "usage: %prog [options] svn_url svn_import_dir dir_v1 [dir_v2 [..]]" parser = OptionParser(usage=usage) parser.add_option("-t", dest="tagdir", help="create a tag copy in tag_dir, relative to svn_url", metavar="tag_dir") parser.add_option("-u", "--svn_username", dest="username", metavar="USER", help="Username for accessing the svn repository") parser.add_option("-p", "--svn_password", dest="password", metavar="USER", help="Password for accessing the svn repository") parser.add_option("--no-prompt", action="store_true", dest="noprompt", default=False, help="non-interactive mode - don't ask any questions") parser.add_option("--wc", dest="working_copy", help="use the already checked-out working copy at path " "instead of checking out a fresh working copy", metavar="working_copy") parser.add_option("-m", "--move-map", metavar="FILE", help="Load a mapping of regular expression patterns to " "lambda functions of match objects from FILE") parser.add_option('--auto-props', action='store_true', dest='auto_props', help='enable automatic properties') parser.add_option('--no-auto-props', action='store_false', dest='auto_props', default=True, help='disable automatic properties') (options, args) = parser.parse_args() if len(args) < 3: sys.stderr.write("Invalid syntax.\n") parser.print_help() sys.exit(1) url = args[0] client = NotifiedClient(options.username, options.password).client if not client.is_url(url): sys.stderr.write("Error: %s is not a valid svn url.\n" % url) sys.exit(1) if not svn_path_exists(client, url, ''): sys.stderr.write("Error connecting or no such repository: %s\n" % url) sys.exit(1) client.set_auto_props(options.auto_props) import_dir = strip_slashes(args[1]) make_svn_dirs(client, url, import_dir) dirs = expand_dirs(args[2:]) # Check to make sure the user isn't trying to import a non-existent dir or # an svn working dir for d in dirs: if not os.path.isdir(d): sys.stderr.write("Error: %s does not exist or is not a directory\n" % d) sys.exit(1) if contains_svn_metadata(d): sys.stderr.write("Error: %s contains .svn dirs or files\n" % (d)) sys.exit(1) if options.move_map: moved = parse_move_map(options.move_map) else: moved = None if options.tagdir: make_svn_dirs(client, url, os.path.dirname(options.tagdir)) if options.working_copy: workingdir = os.path.abspath(options.working_copy) if not contains_svn_metadata(workingdir): sys.stderr.write("Error: %s isn't an svn working directory\n" % (workingdir)) sys.exit(2) else: workingparent = checkout_to_temp(client, url, import_dir) workingdir = os.path.join(workingparent, 'working') for d in dirs: d = os.path.abspath(d) menu = MoveMenu(workingdir, d, interactive=not options.noprompt, moved=moved) menu.go() remove_nodes(client, workingdir, d) overlay_files(client, workingdir, d) sync_exec_flags(client, workingdir) client.checkin(workingdir, "Load %s into %s." % (os.path.basename(d), import_dir)) if options.tagdir: t = TagClient(client, url, import_dir, options.tagdir) t.doIt() if not options.working_copy: shutil.rmtree(workingparent) svn-load-1.3/test.py0000755000175000017500000004702411535502765013412 0ustar dannfdannf#!/usr/bin/python # # This file is a test harness for svn-load # # Copyright (c) 2007, Hewlett-Packard Development Company, L.P. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above # copyright notice, this list of conditions and the following # disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials # provided with the distribution. # # * Neither the name of the Hewlett-Packard Co. nor the names # of its contributors may be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY HEWLETT-PACKARD DEVELOPMENT COMPANY # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL # HEWLETT-PACKARD DEVELOPMENT COMPANY BE LIABLE FOR ANY DIRECT, # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # import unittest import tempfile import shutil import os import pysvn import filecmp import subprocess import time ## This case simply imports an empty directory into the toplevel of an ## empty repository. But, it provides the framework for other classes ## to inherit and do more complex things class BaseLoadCase(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() self.importDirSetUp() self.loadDirsSetUp() self.exportparent = os.path.join(self.tempdir, 'exportparent') os.mkdir(self.exportparent) self.repo = self.createRepository() self.url = 'file://' + self.repo ## idir is the relative path in repo where things get imported self.idir = '' self.username = None self.password = None self.extra_args = [] self.extra_svn_args = [] def importDirSetUp(self): self.importdir = os.path.join(self.tempdir, 'importdir') os.mkdir(self.importdir) def loadDirsSetUp(self): self.loaddirs = [os.path.join(self.tempdir, 'loaddir0')] os.mkdir(self.loaddirs[0]) def tearDown(self): shutil.rmtree(self.tempdir) def createRepository(self): repo = os.path.join(self.tempdir, 'repo') os.mkdir(repo) subprocess.call(['/usr/bin/svnadmin', 'create', repo]) return repo def exportRepository(self): args = [ '/usr/bin/svn', 'export', '--quiet' ] args = args + self.extra_svn_args args = args + [self.url, os.path.join(self.exportparent, 'export')] ret = subprocess.call(args) if ret == 0: return True else: return False def loaddir(self, ldir): args = ['./svn-load', self.url, self.idir, ldir, '--no-prompt'] args = args + self.extra_args ret = subprocess.call(args) if ret == 0: return True else: return False ## This should allow us to accept cases that dircmp considers "funny". ## The only known case so far is where both trees have the same identical ## symlink def funny_check(self, a, b, funny): haha = True strange = False for f in funny: funnyA = os.path.join(a, f) funnyB = os.path.join(b, f) if os.path.islink(funnyA) and os.path.islink(funnyB): if os.readlink(funnyA) == os.readlink(funnyB): continue else: return strange return haha def compare_dirs(self, a, b): if os.path.islink(a) and not os.path.islink(b) or \ not os.path.islink(a) and os.path.islink(b): return False c = filecmp.dircmp(a, b) funny_ok = self.funny_check(a, b, c.common_funny) if c.left_only or c.right_only or c.diff_files or not funny_ok: return False else: return c.common_dirs def compare_trees(self, a, b): subdirs = self.compare_dirs(a, b) if subdirs == False: return False else: for d in subdirs: if not self.compare_trees(os.path.join(a, d), os.path.join(b, d)): return False return True def testLoad(self): client = pysvn.Client() if self.username: client.set_default_username(self.username) if self.password: client.set_default_password(self.password) client.import_(self.importdir, self.url, 'Initial import') for d in self.loaddirs: self.failUnless(self.loaddir(d)) self.failUnless(self.exportRepository()) self.failUnless(self.compare_trees(d, os.path.join(self.exportparent, 'export'))) class EmptyToSingleFile(BaseLoadCase): def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) f = open(os.path.join(self.loaddirs[0], 'foo'), 'w') f.write("baz\n") f.close() class EmptyToSingleFile3LevelsDown(EmptyToSingleFile): def loadDirsSetUp(self): EmptyToSingleFile.loadDirsSetUp(self) os.makedirs(os.path.join(self.loaddirs[0], '1/2/3')) f = open(os.path.join(self.loaddirs[0], '1/2/3/foo'), 'w') f.write("baz\n") f.close() class EmptyToSymLink(BaseLoadCase): def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) os.symlink('baz', os.path.join(self.loaddirs[0], 'foo')) class EmptyToSingleFileAndSymLink(EmptyToSingleFile): def loadDirsSetUp(self): EmptyToSingleFile.loadDirsSetUp(self) os.symlink('foo', os.path.join(self.loaddirs[0], 'baz')) class SingleFileContentChange(BaseLoadCase): def importDirSetUp(self): BaseLoadCase.importDirSetUp(self) f = open(os.path.join(self.importdir, 'foo'), 'w') f.write("bar\n") f.close() def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) f = open(os.path.join(self.loaddirs[0], 'foo'), 'w') f.write("baz\n") f.close() ### We'd need to be able to control the user's svn config file to test this. #class AutoProps(BaseLoadCase): # def loadDirsSetUp(self): # BaseLoadCase.loadDirsSetUp(self) # f = open(os.path.join(self.loaddirs[0], 'foo.txt'), 'w') # f.write("baz\n") # f.close() # # def testLoad(self): # BaseLoadCase.testLoad(self) # client = pysvn.Client() # proplist = client.proplist('%s/foo.txt' % (self.url)) # self.failUnless() class BaseMoveCase(BaseLoadCase): # files_old: list of files in "old" side # files_new: list of files in "new" side, must be identical in length to # files_old, and files_new[i] is the new path for files_old[i] files_old = [] files_new = [] def setUp(self): BaseLoadCase.setUp(self) self.movemap = tempfile.mkstemp()[1] self.makeMoveMap() self.extra_args.extend(['--move-map', self.movemap]) self.filesmap_forward = dict(zip(self.files_old, self.files_new)) self.filesmap_backwards = dict(zip(self.files_new, self.files_old)) def tearDown(self): BaseLoadCase.tearDown(self) os.unlink(self.movemap) def makeMoveMap(self): # Override this method to make the move map f = open(self.movemap, 'w') f.write('\n') f.close() def makeFiles(dir, files): for filename in files: # strip off leading slashes, just in case filename = filename.lstrip('/') try: d = os.path.join(dir, os.path.dirname(filename)) os.makedirs(d) except OSError: # already made pass f = open(os.path.join(dir, filename), 'w') f.write('foo\n') f.close() makeFiles = staticmethod(makeFiles) def importDirSetUp(self): BaseLoadCase.importDirSetUp(self) self.makeFiles(self.importdir, self.files_old) def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) self.makeFiles(self.loaddirs[0], self.files_new) def testLoad(self): # superclass's testLoad method does some stuff that would conflict with # testFilesMoved pass def testFilesMoved(self): if not (self.files_old and self.files_new): # If empty (like on the BaseMoveCase), just ignore return client = pysvn.Client() client.import_(self.importdir, self.url, 'Initial import') self.failUnless(self.loaddir(self.loaddirs[0])) log = client.log(os.path.join(self.url, self.idir), limit=1, discover_changed_paths=True)[0] changed_paths = log['changed_paths'] for path_d in changed_paths: rel_path = path_d['path'][1 + len(self.idir):] if rel_path in self.files_old: # File was moved (changes in this class are only copies and # deletes, no actual content changes) self.assertEqual(path_d['action'], 'D') elif rel_path in self.files_new: old_path = os.path.join('/', self.idir, self.filesmap_backwards[rel_path]) self.assertEqual(path_d['action'], 'A') self.assertEqual(path_d['copyfrom_path'], old_path) class SingleFileMove(BaseMoveCase): files_old = ['foo'] files_new = ['bar'] def makeMoveMap(self): f = open(self.movemap, 'w') f.write("^foo$ 'bar'\n") f.close() class GraphicsFilesMove(BaseMoveCase): files_old = [ 'src/main.c', 'src/foo.jpg', 'src/bar.gif', 'src/baz.png', 'src/bang.png', 'src/blast/boom.jpg', ] files_new = [ 'src/main.c', 'graphics/foo.jpg', 'graphics/bar.gif', 'graphics/baz.png', 'graphics/bang.png', 'graphics/blast/boom.jpg', ] def makeMoveMap(self): f = open(self.movemap, 'w') f.write("^src/(?P.+\.(gif|jpg|png))$ " "lambda m: 'graphics/%s' % m.group('filename')\n") f.close() class MultipleStringMoves(BaseMoveCase): files_old = [ 'foo', 'bar', 'baz', ] files_new = [ 'foo', 'bang', 'eek', ] def makeMoveMap(self): f = open(self.movemap, 'w') f.write("^bar$ 'bang'\n") f.write("^baz$ 'eek'\n") class MoveMapWithDollarSign(BaseMoveCase): files_old = [ 'foo', 'bar$baz', ] files_new = [ 'foo', 'baz', ] def makeMoveMap(self): f = open(self.movemap, 'w') f.write(r"^bar\$baz$ 'baz'") f.write("\n") class SingleFileToBrokenSymLink(BaseLoadCase): def importDirSetUp(self): BaseLoadCase.importDirSetUp(self) f = open(os.path.join(self.importdir, 'foo'), 'w') f.write("bar\n") f.close() def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) os.symlink('broken', os.path.join(self.loaddirs[0], 'foo')) class SingleFileToSymLink(BaseLoadCase): def importDirSetUp(self): BaseLoadCase.importDirSetUp(self) f = open(os.path.join(self.importdir, 'foo'), 'w') f.close() f = open(os.path.join(self.importdir, 'bar'), 'w') f.close() def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) f = open(os.path.join(self.loaddirs[0], 'foo'), 'w') f.close() os.symlink('foo', os.path.join(self.loaddirs[0], 'bar')) class SingleFileToDirectorySymLink(BaseLoadCase): def importDirSetUp(self): BaseLoadCase.importDirSetUp(self) os.mkdir(os.path.join(self.importdir, 'foo')) open(os.path.join(self.importdir, 'bar'), 'w').close() def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) os.mkdir(os.path.join(self.loaddirs[0], 'foo')) os.symlink('foo', os.path.join(self.loaddirs[0], 'bar')) class BrokenSymLinkToFile(BaseLoadCase): def importDirSetUp(self): BaseLoadCase.importDirSetUp(self) os.symlink('broken', os.path.join(self.importdir, 'foo')) def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) f = open(os.path.join(self.loaddirs[0], 'foo'), 'w') f.write("bar\n") f.close() class SymLinkToFile(BaseLoadCase): def importDirSetUp(self): BaseLoadCase.importDirSetUp(self) open(os.path.join(self.importdir, 'foo'), 'w').close() os.symlink('foo', os.path.join(self.importdir, 'bar')) def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) open(os.path.join(self.loaddirs[0], 'foo'), 'w').close() open(os.path.join(self.loaddirs[0], 'bar'), 'w').close() class DirectorySymLinkToFile(BaseLoadCase): def importDirSetUp(self): BaseLoadCase.importDirSetUp(self) os.mkdir(os.path.join(self.importdir, 'foo')) os.symlink('foo', os.path.join(self.importdir, 'bar')) def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) os.mkdir(os.path.join(self.loaddirs[0], 'foo')) open(os.path.join(self.loaddirs[0], 'bar'), 'w').close() class ReimportSymlinkBase(BaseLoadCase): def _fillDirectory(self, directory): # overwrite in derived class pass def importDirSetUp(self): BaseLoadCase.importDirSetUp(self) self._fillDirectory(self.importdir) def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) self._fillDirectory(self.loaddirs[0]) # create files one alphabetically before and one after 'bar' open(os.path.join(self.loaddirs[0], 'a-file'), 'w').close() open(os.path.join(self.loaddirs[0], 'zoo-file'), 'w').close() # create a subdirectory hierarchy os.mkdir(os.path.join(self.loaddirs[0], 'zoo')) os.mkdir(os.path.join(self.loaddirs[0], 'zoo', 'subzoo')) os.mkdir(os.path.join(self.loaddirs[0], 'zoo', 'subzoo', 'subsubzoo')) open(os.path.join(self.loaddirs[0], 'zoo', 'subzoo', 'subsubzoo', 'subsubzoo-file'), 'w').close() class ReimportSymlink(ReimportSymlinkBase): def _fillDirectory(self, directory): open(os.path.join(directory, 'foo'), 'w').close() os.symlink('foo', os.path.join(directory, 'bar')) class ReimportSymlinkToDirectory(ReimportSymlinkBase): """svn-load version 0.9 fails on this and _commits_ an _inconsistent_ content """ def _fillDirectory(self, directory): os.mkdir(os.path.join(directory, 'foo')) os.symlink('foo', os.path.join(directory, 'bar')) class ReimportBrokenSymlink(ReimportSymlinkBase): def _fillDirectory(self, directory): os.symlink('foo', os.path.join(directory, 'bar')) class SyblingDirSymlink(BaseLoadCase): def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) os.mkdir(os.path.join(self.loaddirs[0], 'foo')) os.symlink('foo', os.path.join(self.loaddirs[0], 'bar')) class GlobalIgnoreFile(BaseLoadCase): ## As of this writing, svn-load assumes that all files in a directory ## will be added when the subdirectory is added, so it doesn't bother ## descending down a tree it just added. However, some files are always ## ignored by svn, even if no svn:ignore property exists. This is ## controlled by the global-ignores parameter. svn-load blows up ## on these files during its check_permissions pass, as it assumes it ## can query their svn:executable property def loadDirsSetUp(self): BaseLoadCase.loadDirsSetUp(self) os.mkdir(os.path.join(self.loaddirs[0], 'foo')) f = open(os.path.join(self.loaddirs[0], 'foo', 'bar.o'), 'w') f.write("baz\n") f.close() ## This test creates two load dirs: foo and foo2, then attempts to load ## 'foo*'. On UNIX this should work because the shell expands the glob. ## On Windows it should work because svn-load expands the glob class GlobHandling(BaseLoadCase): def loadDirsSetUp(self): self.loaddirparent = os.path.join(self.tempdir, 'loaddirparent') os.mkdir(self.loaddirparent) self.loaddirs = [os.path.join(self.loaddirparent, 'foo'), os.path.join(self.loaddirparent, 'foo2')] for d in self.loaddirs: os.mkdir(d) f = open(os.path.join(d, 'test'), 'w') f.write(d + '\n') f.close() def testLoad(self): svnload = './svn-load' client = pysvn.Client() client.import_(self.importdir, self.url, 'Initial import') os.system('%s %s / %s*' % (svnload, self.url, os.path.join(self.loaddirparent, "foo*"))) self.failUnless(self.exportRepository()) self.failUnless(self.compare_trees(self.loaddirs[-1], os.path.join(self.exportparent, 'export'))) class TempConfigFile: def __init__(self): self.f = tempfile.NamedTemporaryFile(delete=False) self.path = self.f.name def __del__(self): os.unlink(self.path) class SvnServePasswdFile(TempConfigFile): def __init__(self, userDict): TempConfigFile.__init__(self) self.f.write('[users]\n') users = userDict.keys() for u in users: self.f.write('%s = %s\n' % (u, userDict[u])) self.f.close() class SvnServeConf(TempConfigFile): def __init__(self, passwd): TempConfigFile.__init__(self) self.f.write('[general]\n') self.f.write('anon-access = none\n') self.f.write('auth-access = write\n') self.f.write('password-db = %s\n' % (passwd)) self.f.close() class AuthenticatedServer(BaseLoadCase): def startServer(self): self.passwd = SvnServePasswdFile({ self.username: self.password }) self.conf = SvnServeConf(self.passwd.path) self.port = 8686 self.server = subprocess.Popen(['svnserve', '--daemon', '--foreground', '--root=%s' % (self.tempdir), '--listen-host=127.0.0.1', '--listen-port=%d' % (self.port), '--config-file=%s' % (self.conf.path)]) # Give server time to listen() time.sleep(1) def setUp(self): BaseLoadCase.setUp(self) self.username = 'alice' self.password = 'secret' self.extra_args.extend(['--svn_username=%s' % (self.username)]) self.extra_args.extend(['--svn_password=%s' % (self.password)]) self.extra_svn_args.extend(['--username', self.username]) self.extra_svn_args.extend(['--password', self.password]) self.startServer() self.url = 'svn://127.0.0.1:%s/repo' % (self.port) def tearDown(self): self.server.kill() BaseLoadCase.tearDown(self) if __name__ == '__main__': try: defaultTest = sys.argv[1] except: defaultTest = None unittest.main(defaultTest=defaultTest) svn-load-1.3/ChangeLog0000644000175000017500000000360511535771741013627 0ustar dannfdannf2011-03-09 dann frazier * Add support for username/password * Add support for --auto-props/--no-auto-props * Release 1.3 2009-02-15 dann frazier * Check that user-specified directories exist before loading them, patch from John Wright. * Release 1.2 2008-12-12 dann frazier * Remove nocallback warning for ssl_client_cert_password_prompt * Release 1.1 2008-10-28 dann frazier * Add a callback to prompt for client cert passwords * Warn when unimplemented callbacks are triggered * Release 1.0 2008-04-23 dann frazier * The following patches developed by Robert Hunger * Fix issue with broken symlink becoming a regular file * Fix issue with a preexisting non-dangling directory symlink * Fix traceback when using --no-prompt without --move-map, reported in Debian bug #476569 * test.py: permit running a single test from the command line * test.py: Use a common temporary directory in BaseLoadCase.tempdir * test.py: If initial repository load fails, fail the test * test.py: Run svn export in quiet mode * Release 0.10 2008-04-08 dann frazier * Add support for regular files becoming symlinks, reported in Debian bug #436751 * Fix help output to show that at least one load directory is required, reported in Debian bug #475093 * Treat directory symlinks like files when overlaying them to prevent a real directory (and its contents) from being added * Release 0.9 2007-12-18 dann frazier * Release 0.8 2007-12-18 dann frazier * Disregard default ignores and svn:ignore properties. Assume the user really means it. 2007-11-02 dann frazier * Create this file. * Add support for globs on Windows, contributed by Mariano Reingart * Remove duplicate SingleFileToBrokenSymLink definition in test.py, reported by Chaim Krause svn-load-1.3/README.move-map0000644000175000017500000000377710635270553014462 0ustar dannfdannfMoving files according to regular expression matches ============================================================ Motivation ---------- Occasionally, a vendor will decide to make moves that are difficult to go through a file at a time. For instance, consider a project that originally had all of its graphics in its src/ directory, and later decided to make an images/ directory and move all of its graphics out of the src/ directory and into the images/ directory. If there are many images, it's tedious to manually go through each file and decide where each one was moved. Often, it's easier simply to ignore the moves, treating them as deletes and new files. Unfortunately, this wastes a lot of space. Instead, why not map the moves with a regular expression? Here, the pattern ^src/(?P.+\.(gif|jpg|png))$ matches any .gif, .jpg, or .png in src, with the filename in the named group "filename". To actually map the filename to a new filename, we need a function that takes the regular expression object as its only argument and returns the new filename as a string. The following Python lambda function fits the bill: lambda m: "images/%s" % m.group("filename") Implementation -------------- svn-load has a simple move-map file format: one map per line, each line starts with an absolute (i.e. contains ^ and $) pattern and a Python lambda function of an _sre.SRE_Match object (the type returned by re.match) which returns a string. So, the above example would look like ^src/(?P.+\.(gif|jpg|png))$ lambda m: "images/%s" % m.group("filename") You don't have to use named groups in your regular expressions; however, to refer to numbered group n, you will need to use the value of m.group(n) (assuming m is the name of the argument). If you save this file to a file called 'move-images.move-map', you can run svn-load as usual, adding '--move-map move-images.move-map' to the command-line, and it will correctly move all those images before asking you to interactively map any other file moves. svn-load-1.3/COPYING0000644000175000017500000000306210564145774013107 0ustar dannfdannfCopyright (c) 2007, Hewlett-Packard Development Company, L.P. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the Hewlett-Packard Co. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY HEWLETT-PACKARD DEVELOPMENT COMPANY ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL HEWLETT-PACKARD DEVELOPMENT COMPANY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.