ftpgrab-0.1.5/0000750003136300116100000000000011275171421011747 5ustar cevansengftpgrab-0.1.5/fgpickregexpsync.h0000640003136300116100000000213407101741214015470 0ustar cevanseng#ifndef _FGPICKREGEXPSYNC_H #define _FGPICKREGEXPSYNC_H // fgpickregexpsync.h // // Implementation of a file picker that selects all remote files matching // a given regular expression for which we do not have remote copies. // Also, all local files matching the regexp which are not on the remote // site, are removed #ifndef _FGFPICKI_H #include "fgfpicki.h" #endif class FGConnectionInterface; class FGString; struct re_pattern_buffer; typedef struct re_pattern_buffer regex_t; class FGPickRegexpSync : public FGFilePickerInterface { public: // Must be constructed with connection interface and regexp match string FGPickRegexpSync(FGConnectionInterface* pConnIf, const FGString& regexp); virtual ~FGPickRegexpSync(); virtual FGActionList DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir); private: // Banned! FGPickRegexpSync(const FGPickRegexpSync& other); FGPickRegexpSync& operator=(const FGPickRegexpSync& other); // Internal representation of a regexp regex_t* mpRegExp; }; #endif // _FGPICKREGEXP_H ftpgrab-0.1.5/fgfilegrab.h0000640003136300116100000000206706702006053014213 0ustar cevanseng#ifndef _FGFILEGRAB_H #define _FGFILEGRAB_H // fgfilegrab.h // // Definition of in-memory structure describing a single "fetch" operation. // A fetch operation may fetch one file, or many files if it is for example // based on a wildcard rule #ifndef _FGSTRING_H #include "fgstring.h" #endif class FGConnectionInterface; class FGFilePickerInterface; class FGFileGrab { public: // Constructors FGFileGrab(const FGString& ruleName, const FGString& host, const FGString& remDir, const FGString& localDir, const FGString& file); // The all important "go get it" method void GrabIt(void) const; // Getters const FGString& GetRuleName(void) const; void FreeResources(void); private: // Implementation details FGString mRuleName; FGString mRemoteHost; FGString mRemoteDir; FGString mLocalDir; // Interface to the connection protocol this fetch uses FGConnectionInterface* mpConInterface; // Interface to the method of selecting files this fetch uses FGFilePickerInterface* mpPickInterface; }; #endif // _FGFILEGRAB_H ftpgrab-0.1.5/debian/0000750003136300116100000000000007115533142013171 5ustar cevansengftpgrab-0.1.5/debian/copyright0000640003136300116100000000115207115532570015130 0ustar cevansengThis package was debianized by Christian T. Steigies on Sat, 24 Apr 1999 23:11:56 +0200. It was downloaded from ftp.lmh.ox.ac.uk/pub/linux/ftpgrab-0.1.1.tar.gz Upstream Author: Chris Evans Copyright: This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. On Debian GNU/Linux systems, the complete text of the GNU General Public License is found in `/usr/share/common-licenses/GPL'. ftpgrab-0.1.5/debian/rules0000750003136300116100000000331706713623731014264 0ustar cevanseng#!/usr/bin/make -f #-*- makefile -*- # Made with the aid of dh_make, by Craig Small # Sample debian/rules that uses debhelper. GNU copyright 1997 by Joey Hess. # Some lines taken from debmake, by Christoph Lameter. # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 build: build-stamp build-stamp: dh_testdir # Add here commands to compile the package. $(MAKE) touch build-stamp clean: dh_testdir dh_testroot rm -f build-stamp install-stamp # Add here commands to clean up after the build process. -$(MAKE) clean dh_clean install: install-stamp install-stamp: build-stamp dh_testdir dh_testroot dh_clean -k dh_installdirs # Add here commands to install the package into debian/tmp. # $(MAKE) install DESTDIR=`pwd`/debian/tmp install -g root -o root -m 755 ftpgrab `pwd`/debian/tmp/usr/bin touch install-stamp # Build architecture-independent files here. binary-indep: build install # We have nothing to do by default. # Build architecture-dependent files here. binary-arch: build install # dh_testversion dh_testdir dh_testroot dh_installdocs # dh_installdocs BUILD COPYING DESIGN dh_installexamples ftpgrabrc dh_installmenu # dh_installemacsen # dh_installinit dh_installcron dh_installmanpages # dh_undocumented ftpgrab.1 dh_installchangelogs Changelog dh_link dh_strip dh_compress dh_fixperms # You may want to make some executables suid here dh_suidregister # dh_makeshlibs dh_installdeb dh_shlibdeps dh_gencontrol dh_md5sums dh_builddeb source diff: @echo >&2 'source and diff are obsolete - use dpkg-source -b'; false binary: binary-indep binary-arch .PHONY: build clean binary-indep binary-arch binary ftpgrab-0.1.5/debian/docs0000640003136300116100000000011707115532570014050 0ustar cevansengREADME README.URGENT.KERNEL2.0 README.gcc.2.7 README.options TODO BUILD DESIGN ftpgrab-0.1.5/debian/control0000640003136300116100000000162107115532570014601 0ustar cevansengSource: ftpgrab Section: net Priority: optional Maintainer: Christian T. Steigies Standards-Version: 3.0.1 Build-Depends: debhelper, libstdc++2.10-glibc2.1-dev Package: ftpgrab Architecture: any Depends: ${shlibs:Depends} Description: file mirroring utility ftpgrab is a utility for maintaining FTP mirrors. In fact not unlike the "Mirror" perl program. However ftpgrab is oriented towards the smaller site which doesn't have the resources to mirror entire version trees of software. . The primary "plus point" of ftpgrab is that it can base download decisions by parsing version numbers out of filenames. For example, ftpgrab will recognize that the file "linux-2.2.2.tar.gz" is newer than "linux-2.2.1.tar.gz" based on the version string. It will then download the new version and delete the old one when it is done, thus saving you mirroring 10 kernel versions all at >10Mb each. ftpgrab-0.1.5/debian/dirs0000640003136300116100000000002106713623731014056 0ustar cevansengusr/bin usr/sbin ftpgrab-0.1.5/debian/watch0000640003136300116100000000046706713623731014241 0ustar cevanseng# Example watch control file for uscan # Rename this file to "watch" and then you can run the "uscan" command # to check for upstream updates and more. # Site Directory Pattern Version Script #sunsite.unc.edu /pub/Linux/Incomingu ftpgrab-*.tar.gz debian uupdate ftp.lmh.ox.ac.uk /pub/linux ftpgrab-*.tar.gz ftpgrab-0.1.5/debian/changelog0000640003136300116100000000223207115532570015047 0ustar cevansengftpgrab (0.1.2pre1-1) unstable; urgency=low * New upstream release. -- Christian T. Steigies Mon, 15 May 2000 20:31:24 +0200 ftpgrab (0.1.1-1) unstable; urgency=low * new upstream source * added Build-Depends * updated Standards -- Christian T. Steigies Mon, 3 Jan 2000 21:03:03 +0100 ftpgrab (0.1.0-2) unstable; urgency=low * apply a patch required for alpha, thanks Bart. Fixes: Bug#39982. -- Christian T. Steigies Wed, 23 Jun 1999 11:39:37 +0200 ftpgrab (0.1.0-1) unstable; urgency=low * New upstream release. -- Christian T. Steigies Tue, 18 May 1999 14:18:14 +0200 ftpgrab (0.0.6a-2) unstable; urgency=low * updated the manpage -- Christian T. Steigies Thu, 6 May 1999 22:35:12 +0200 ftpgrab (0.0.6a-1) unstable; urgency=low * New upstream release. Read README.URGENT.KERNEL2.0 ! -- Christian T. Steigies Wed, 5 May 1999 21:22:54 +0200 ftpgrab (0.0.4a-1) unstable; urgency=low * Initial Release. -- Christian T. Steigies Sat, 24 Apr 1999 23:11:56 +0200 Local variables: mode: debian-changelog End: ftpgrab-0.1.5/debian/ftpgrab.10000640003136300116100000000266107115532570014712 0ustar cevanseng.TH FTPGRAB 1 "Jan 3rd 2000" "GNU" "Debian GNU/Linux manual" .SH NAME ftpgrab \- a file mirroring utility .SH SYNOPSIS .B ftpgrab .I "[options]" .SH "DESCRIPTION" This manual page documents briefly the .BR ftpgrab command. This manual page was written for the Debian GNU/Linux distribution because the original program does not have a manual page. .PP .B ftpgrab is an utility for maintaining FTP mirrors. In fact not unlike the .B mirror perl program. However ftpgrab is oriented towards the smaller site which doesn't have the resources to mirror entire version trees of software. .SH OPTIONS .TP .I -h usage .TP .I -n Don't log to an output file .TP .I -l Change log file name from default .B (fglog.out) to .TP .I -r Change config file name from default .B (ftpgrabrc) to .TP .I -t Set maximum download threads to .TP .I -v Output verbose progress updates to stdout .\".B ftpgrab .\"does not take any options currently, but is entirely configured by .\"the file ftpgrabrc in the directory where .\".B ftpgrab .\"is called. .\".B ftpgrab .\"does not produce any output currently, but it creates a logfile in the .\"directory where it is called ( .\".B fglog.out .\"). Please see the file /usr/share/doc/ftpgrab/examples/ftpgrabrc for details. .SH AUTHOR This manual page was written by Christian T. Steigies , for the Debian GNU/Linux system (but may be used by others). ftpgrab-0.1.5/debian/README.Debian0000640003136300116100000000153606713623731015247 0ustar cevansengftpgrab for Debian ---------------------- ftpgrab is consired ALPHA software by the author, allthough the examples supplied seem to be working. Citing Chris Evans : OK, I can add an option to output what it is doing to screen in the next release. OK, I will add an option to specify a diffent rc filename in the next release (and probably change the default to a dotfile). Thanks for your comments. I would probably recommend you wait a release or two before making a package - I have a few bugs to fix. Also, the next release will feature regular expression support in download rules. So ftpgrab options and rc files may change in the future, if you have recommendations, just mail Chris: "I am always happy to listen to suggestions!" -- Christian T. Steigies , Sun, 25 Apr 1999 21:24:56 +0200 ftpgrab-0.1.5/main.cc0000640003136300116100000000564511275171365013224 0ustar cevanseng#ifndef _FGSTRING_H #include "fgstring.h" #endif #ifndef _FGFILELIST_H #include "fgfilelist.h" #endif #ifndef _FGLOGGER_H #include "fglogger.h" #endif #include #include #include #include #include #include // Globals #ifndef _FGGLOB_H #include "fgglob.h" #endif #ifndef _FGEXC_H #include "fgexc.h" #endif FGString FGGlob::gHostName; bool FGGlob::gLogDisabled = false; int FGGlob::gMaxThreads = 5; bool FGGlob::gVerbose = false; void usage(void) { fprintf(stderr, "This is ftpgrab v0.1.2\n"); fprintf(stderr, "Usage:\n"); fprintf(stderr, "ftpgrab [-n]\n"); fprintf(stderr, "-h : Display this message\n"); fprintf(stderr, "-n : Do not output information to a log file\n"); fprintf(stderr, "-l : Output log to rather than default location\n"); fprintf(stderr, "-r : Use as rcfile rather than default\n"); fprintf(stderr, "-t : Use up to concurrent threads (1<=num<=50)\n"); fprintf(stderr, "-v : Output progress to console\n"); fflush(stderr); exit(1); } int main(int argc, char* const argv[]) { // Parse command line options char option; FGString rcFileName("ftpgrabrc"); while ((option = getopt(argc, argv, "hnvl:r:t:")) != EOF) { switch (option) { case '?': case ':': case 'h': usage(); break; case 'n': FGGlob::gLogDisabled = true; break; case 'l': FGLogger::SetLogName(optarg); break; case 'r': rcFileName = optarg; break; case 't': { int threads = atoi(optarg); if (threads < 1 || threads > 50) { usage(); } FGGlob::gMaxThreads = threads; } break; case 'v': FGGlob::gVerbose = true; break; } } { // Initialize signal handling struct sigaction pipeAction; // Safety memset(&pipeAction, '\0', sizeof(pipeAction)); pipeAction.sa_handler = SIG_IGN; sigemptyset(&pipeAction.sa_mask); sigaction(SIGPIPE, &pipeAction, NULL); } { // Get the current hostname struct utsname machineDetails; uname(&machineDetails); FGGlob::gHostName = machineDetails.nodename; } try { (void) FGLogger::GetLogger(); } catch (FGException& e) { fprintf(stderr, "Failed to initialize logger - bailing!\n"); exit(1); } FGLogger& log = FGLogger::GetLogger(); log.LogMsg("ftpgrab startup", FGLogger::kFGLLInfo); log.LogMsg("Attempting to parse config file", FGLogger::kFGLLVerbose); FGFileList list; try { list.LoadConfig(rcFileName); } catch (FGException& e) { log.LogException(e); log.LogMsg("Failed to parse config file - bailing", FGLogger::kFGLLErr); exit(1); } log.LogMsg("Sucessfully parsed config file", FGLogger::kFGLLVerbose); list.GetAllFiles(); log.LogMsg("ftpgrab exiting", FGLogger::kFGLLInfo); exit(0); } ftpgrab-0.1.5/fgactioni.h0000640003136300116100000000143706702416232014072 0ustar cevanseng#ifndef _FGACTIONI_H #define _FGACTIONI_H // fgactioni.h // // FGActionInterface // // Interface to individual actions e.g. "get" "delete" etc. // Very simple :-) class FGConnectionInterface; class FGActionInterface { public: // Need virtual destructor for the mm virtual ~FGActionInterface(); // Non-virtual method which basically delegates to VirtualDo() but // handles exceptions by automatically calling abort and re-throwing void Do(void) const; virtual void VirtualDo(void) const = 0; // Abort: called if operation exits abnormally // Example: we are downloading a file when we lose the connection. In // this case we call Abort() and the specific cleanup is to delete the // partially downloaded file virtual void Abort(void) const = 0; }; #endif // _FGACTIONI_H ftpgrab-0.1.5/fgcharcomp.cc0000640003136300116100000000223007022703001014353 0ustar cevanseng// fgcharcomp.cc #include "fgcharcomp.h" #ifndef _FGSTRING_H #include "fgstring.h" #endif #include FGCharacterComponent::FGCharacterComponent(bool isOptional) { mIsOptional = isOptional; } FGCharacterComponent::~FGCharacterComponent() { } bool FGCharacterComponent::MatchAndRankComponent(FGString& fnameRemainder, int* pMatchVal) const { // Initialize to be safe *pMatchVal = -1; // Sanity checks if (fnameRemainder.GetLength() < 1 || !isalpha(fnameRemainder[0])) { // May or may not be a "match" or such. If the character component // is NOT optional, it's not match. Otherwise it IS a match, with // minimal value if (!mIsOptional) { return false; } else { *pMatchVal = 0; return true; } } // The value returned will actually not be zero based, it will // be ASCII "A" based (65??) // It is of no conseqence *pMatchVal = toupper(fnameRemainder[0]); // Chop off the digits we consumed int charsLeft = fnameRemainder.GetLength() - 1; FGString bitLeft = fnameRemainder.Right(charsLeft); fnameRemainder = bitLeft; return true; } ftpgrab-0.1.5/fglogger.cc0000640003136300116100000000605411267250764014071 0ustar cevanseng// fglogger.cc #include "fglogger.h" #ifndef _FGEXC_H #include "fgexc.h" #endif #ifndef _FGGLOB_H #include "fgglob.h" #endif #include #include #include #include #include // Threading: FGLogger is a shared object so we need to serialize access // to it #include // Statics FGString FGLogger::msFileName = "fglog.out"; FGLogger* FGLogger::mspLogger = 0; // Synchronization object // Should be private member static but can't forward declare // pthread_mutex_t static pthread_mutex_t msLogMutex; void FGLogger::SetLogName(const FGString& name) { msFileName = name; } FGLogger& FGLogger::GetLogger(void) { if (mspLogger == 0) { mspLogger = new FGLogger(msFileName); pthread_mutex_init(&msLogMutex, NULL); } return *mspLogger; } FGLogger::FGLogger(const FGString& fileName) { if (!FGGlob::gLogDisabled) { mpFile = fopen(fileName, "a"); if (mpFile == NULL) { FGString details("`"); details += fileName; details += '\''; throw FGException(FGException::kLogFileOpenFailed, details); } } else { mpFile = NULL; } } void FGLogger::LogMsg(const FGString& msg, EFGLogSeverity severity, bool locked) { MutexAutoUnlocker autoLocker; // NB. It locked is set "true", client MUST have called public // ::Lock() method on the logger class! if (!locked) { autoLocker.Prime(&msLogMutex); } if (FGGlob::gVerbose) { printf("%s\n", (const char*)msg); } if (FGGlob::gLogDisabled || severity == kFGLLVerbose) { return; } FGString dateStamp = GetDateStamp(); fprintf(mpFile, "%s %s\n", (const char*)dateStamp, (const char*)msg); fflush(mpFile); } void FGLogger::LogException(const FGException& e, bool locked) { MutexAutoUnlocker autoLocker; // NB. It locked is set "true", client MUST have called public // ::Lock() method on the logger class! if (!locked) { autoLocker.Prime(&msLogMutex); } // {ce} - 07-Dec-99: oops, forgot to log this to stdout FGString dateStamp = GetDateStamp(); FGString excText = e.GetFullDetails(); if (FGGlob::gVerbose) { printf("%s\n", (const char*)excText); } if (FGGlob::gLogDisabled) { return; } fprintf(mpFile, "%s %s\n", (const char*)dateStamp, (const char*)excText); fflush(mpFile); } FGString FGLogger::GetDateStamp(void) const { char dateBuf[64]; struct timeval now; gettimeofday(&now, NULL); struct tm* pTime = localtime(&now.tv_sec); strftime(dateBuf, sizeof(dateBuf), "[%d %b %Y %H:%M]", pTime); FGString ret(dateBuf); return ret; } void FGLogger::Lock(void) { pthread_mutex_lock(&msLogMutex); } void FGLogger::Unlock(void) { pthread_mutex_unlock(&msLogMutex); } // Helper classes FGLogger::MutexAutoUnlocker::MutexAutoUnlocker() : mpMutex(0) { } FGLogger::MutexAutoUnlocker::~MutexAutoUnlocker() { if (mpMutex) { pthread_mutex_unlock(static_cast(mpMutex)); } } void FGLogger::MutexAutoUnlocker::Prime(void* pMutex) { mpMutex = pMutex; pthread_mutex_lock(static_cast(mpMutex)); } ftpgrab-0.1.5/fgfilelist.cc0000640003136300116100000002402011267444232014411 0ustar cevanseng// fgfilelist.cc #include "fgfilelist.h" #ifndef _FGFILEGRAB_H #include "fgfilegrab.h" #endif #ifndef __SGI_STL_VECTOR_H #include #endif #ifndef _FGSTRING_H #define "fgstring.h" #endif #ifndef _FGLOGGER_H #include "fglogger.h" #endif #ifndef _FGEXC_H #include "fgexc.h" #endif #ifndef _FGGLOB_H #include "fgglob.h" #endif #include #include #include // Threading. Very not cool. #include // Class scope variables int FGFileList::msThreadsActive = 0; struct FGFileList::Internal_FGFileList { // The individual rules one by one std::vector mFiles; // Threading synchronization object // Children flag this when they exit to alert // master process to spawn more static pthread_cond_t msChildExitCondition; // Mutex to go with the above static pthread_mutex_t msChildExitMutex; // Mutex to ensure no children run before the parent // has acknowledged the broadcast static pthread_mutex_t msBroadcastSerializer; }; pthread_cond_t FGFileList::Internal_FGFileList::msChildExitCondition; pthread_mutex_t FGFileList::Internal_FGFileList::msChildExitMutex; pthread_mutex_t FGFileList::Internal_FGFileList::msBroadcastSerializer; FGFileList::FGFileList() { // Allocate the implementation mpInternals = new Internal_FGFileList; // Set up the threading objects pthread_cond_init(&Internal_FGFileList::msChildExitCondition, NULL); pthread_mutex_init(&Internal_FGFileList::msChildExitMutex, NULL); pthread_mutex_init(&Internal_FGFileList::msBroadcastSerializer, NULL); } FGFileList::~FGFileList() { if (mpInternals) { // Free allocated resources of each FGFileGrab object std::vector::iterator iFiles; for (iFiles = mpInternals->mFiles.begin(); iFiles != mpInternals->mFiles.end(); iFiles++) { iFiles->FreeResources(); } pthread_mutex_destroy(&Internal_FGFileList::msChildExitMutex); pthread_mutex_destroy(&Internal_FGFileList::msBroadcastSerializer); pthread_cond_destroy(&Internal_FGFileList::msChildExitCondition); delete mpInternals; } // Bug if we've still got threads hanging around assert(msThreadsActive == 0); } FGString FGFileList::GetLineString(void) const { FGString ret("at line "); char intBuf[10]; sprintf(intBuf, "%d", mLine); ret += intBuf; return ret; } void FGFileList::LoadConfig(const FGString& configFile) { FILE* pFile = fopen(configFile, "r"); if (pFile == NULL) { FGString details("`"); details += configFile; details += '\''; throw FGException(FGException::kConfigFileNotFound, details); } // Catch then rethrow exceptions so we can clean up and close file try { // Temp. strings as we build up a file entry FGString ruleName; FGString host; FGString remDir; FGString localDir; FGString file; // This indicates how many bits we have to go before we have a // complete file entry int componentsToFind = 5; mLine = 0; char buf[1024]; while (fgets(buf, sizeof(buf), pFile) != NULL) { mLine++; // Nail carrige returns into end of lines char* pCR = strchr(buf, '\n'); if (pCR != NULL) { *pCR = '\0'; } // Skip whitespace char* pPos = buf; while (*pPos == ' ' || *pPos == '\t') { pPos++; } // 09-Dec-99 - Also ignore whitespace at end of line! char* pStart = pPos; while (*pStart != '\0') { pStart++; } while (pStart > pPos) { pStart--; if (*pStart == ' ' || *pStart == '\t') { *pStart = '\0'; } else { break; } } // Ignore comments or blank lines if (*pPos == '#' || *pPos == '\0') { continue; } // First ":" is separator between token and value char* pSep = strchr(pPos, ':'); if (pSep == NULL) { // Must have a : FGString details = GetLineString(); throw FGException(FGException::kConfigMissingColon, details); } *pSep = '\0'; pSep++; while (*pSep == ' ' || *pSep == '\t') { pSep++; } if (!strcmp(pPos, "RemoteDir")) { if (!remDir.IsEmpty()) { FGString details("RemoteDir, "); details += GetLineString(); throw FGException(FGException::kDuplicateToken, details); } remDir = pSep; componentsToFind--; } else if (!strcmp(pPos, "LocalDir")) { if (!localDir.IsEmpty()) { FGString details("LocalDir, "); details += GetLineString(); throw FGException(FGException::kDuplicateToken, details); } localDir = pSep; componentsToFind--; } else if (!strcmp(pPos, "Name")) { if (!ruleName.IsEmpty()) { FGString details("Name, "); details += GetLineString(); throw FGException(FGException::kDuplicateToken, details); } ruleName = pSep; componentsToFind--; } else if (!strcmp(pPos, "Host")) { if (!host.IsEmpty()) { FGString details("Host, "); details += GetLineString(); throw FGException(FGException::kDuplicateToken, details); } host = pSep; componentsToFind--; } else if (!strcmp(pPos, "File")) { if (!file.IsEmpty()) { FGString details("File, "); details += GetLineString(); throw FGException(FGException::kDuplicateToken, details); } file = pSep; componentsToFind--; } else { // Unknown tag FGString details(pPos); details += ", "; details += GetLineString(); } if (!componentsToFind) { // Oooh ready to make a FileGrab FGFileGrab theGrab(ruleName, host, remDir, localDir, file); mpInternals->mFiles.push_back(theGrab); componentsToFind = 5; ruleName.MakeEmpty(); host.MakeEmpty(); file.MakeEmpty(); localDir.MakeEmpty(); remDir.MakeEmpty(); } } // end: while(more lines) // Ended halfway through a rule? if (componentsToFind != 5) { throw FGException(FGException::kUnexpectedEndOfConfig); } } // end try() block catch (FGException&) { fclose(pFile); throw; } fclose(pFile); } // This function spawns lots of threads to attempt to complete several // rules simultaneously. The semantics are this: This call does not // complete until all spawned threads complete. At any given time the // number of threads active is limited to a maximum value void FGFileList::GetAllFiles(void) const { assert(msThreadsActive == 0); // Grab the mutex for child exit notification int lockRet = pthread_mutex_trylock(&Internal_FGFileList::msChildExitMutex); assert(lockRet == 0); std::vector::const_iterator iFiles; for (iFiles = mpInternals->mFiles.begin(); iFiles != mpInternals->mFiles.end(); iFiles++) { // If we are operating with max. children, must wait for some to finish assert(msThreadsActive >=0 && msThreadsActive <= FGGlob::gMaxThreads); if (msThreadsActive == FGGlob::gMaxThreads) { // Wait for a single thread to complete pthread_cond_wait(&Internal_FGFileList::msChildExitCondition, &Internal_FGFileList::msChildExitMutex); msThreadsActive--; pthread_mutex_unlock(&Internal_FGFileList::msBroadcastSerializer); } // OK so we are ready to launch a thread // We never sync on a thread's exit so make them detached pthread_attr_t threadAttr; pthread_attr_init(&threadAttr); int detachState = PTHREAD_CREATE_DETACHED; pthread_attr_setdetachstate(&threadAttr, detachState); pthread_t newThread; void* pArg = (void*)(&(*iFiles)); // And they're off.. msThreadsActive++; pthread_create(&newThread, &threadAttr, &ThreadStartPoint, pArg); pthread_attr_destroy(&threadAttr); } // OK so all grabs are scheduled. We might have some still active // child threads (in fact that's almost certain). Hence we must // wait for them to finish while (msThreadsActive > 0) { pthread_cond_wait(&Internal_FGFileList::msChildExitCondition, &Internal_FGFileList::msChildExitMutex); msThreadsActive--; pthread_mutex_unlock(&Internal_FGFileList::msBroadcastSerializer); } } // ALERT: When this function executes, it is in the context of the // child thread. void* FGFileList::ThreadStartPoint(void* pArg) { // Cast the arg, which is actually a pointer to the FGFileGrab object FGFileGrab* pGrab = (FGFileGrab*)pArg; FGString vmsg("About to run rule \""); vmsg += pGrab->GetRuleName(); vmsg += '"'; FGLogger::GetLogger().LogMsg(vmsg, FGLogger::kFGLLVerbose); try { pGrab->GrabIt(); } catch (FGException& e) { FGString msg("Failed to complete rule \""); msg += pGrab->GetRuleName(); msg += "\", reason follows"; // To guarantee reason is line _immediately_ after "reason follows" FGLogger::Lock(); FGLogger::GetLogger().LogMsg(msg, FGLogger::kFGLLWarn, true); FGLogger::GetLogger().LogException(e, true); FGLogger::Unlock(); } // Prevent other threads broadcasting before parent has seen // the broadcast pthread_mutex_lock(&FGFileList::Internal_FGFileList::msBroadcastSerializer); // Thread about to die; signal master parent thread // Only signal when parent is waiting for it (parent unlocks mutex when // it starts to wait for children to exit) pthread_mutex_lock(&FGFileList::Internal_FGFileList::msChildExitMutex); pthread_cond_broadcast(&FGFileList::Internal_FGFileList::msChildExitCondition); // Parent is trying to lock the mutex now - so it is blocked until we // unlock the mutex pthread_mutex_unlock(&FGFileList::Internal_FGFileList::msChildExitMutex); // However! There is a race condition. When we unlock, other children // are competing with the parent for the processor. If another child // runs first it will broadcast the condition before the parent has // acknowledged it and a broadcast will be lost // Hence the broadcast serializing mutex return 0; } ftpgrab-0.1.5/README.options0000640003136300116100000000057106707460732014337 0ustar cevansengHere is the list of command line options supported by "ftpgrab". -h Display usage -n Don't log to an output file -l Change log file name from default to -r Change config file name from default to -t Set maximum download threads to -v Output verbose progress updates to stdout ftpgrab-0.1.5/Makefile0000640003136300116100000000110207101744064013404 0ustar cevansengCC = g++ CFLAGS = -O2 -Wall #LIBS = -lefence LIBS = -lpthread LINK = -Wl,-s OBJS = main.o fgstring.o fgexc.o fgftpcon.o fgfileinfo.o \ fgfilegrab.o fgalist.o fgfshelp.o fgpickall.o \ fgdlist.o fgdlaction.o fgactioni.o fgfilelist.o \ fgpickallsync.o fgdelaction.o fgfpicki.o fgpickbest.o \ fgscomp.o fgicomp.o fgbdfname.o fgmrank.o fglogger.o \ fgdyymmddcomp.o fgfncomp.o fgpickregexp.o \ fgpickregexpsync.o fgconi.o fgcharcomp.o .cc.o: $(CC) -c $*.cc $(CFLAGS) ftpgrab: $(OBJS) $(CC) -o ftpgrab $(OBJS) $(LINK) $(LIBS) clean: rm -f *.o ftpgrab fglog.out ftpgrab-0.1.5/ftpgrabrc0000640003136300116100000000463207101760771013657 0ustar cevanseng# Example ftpgrabrc file # # Host can be prefixed by ftp:// or http:// otherwise ftp is assumed # n.b. http isn't implemented yet though # This rule keeps local copies of the 3 most recent versions of filenames # that look like "file1.txt", "file10.txt" etc. "Most recent" is not # defined by date, but by the version string embedded in the filename #Name: TestRule #Host: localhost #RemoteDir: /pub #LocalDir: /tmp/files #File: [3]file.txt # This rule downloads all new files in /pub directory #Name: TestRule2 #Host: localhost #RemoteDir: /pub #LocalDir: /tmp/fgdest #File: * # This rule downloads all new files in /pub directory AND deletes # all files in /tmp/fgdest NOT found remotely. So beware #Name: TestRule3 #Host: localhost #RemoteDir: /pub #LocalDir: /tmp/fgdest #File: * # This rule keeps the most recent linux kernel tarball in /home/kernel :) #Name: Kernel1 #Host: ftp.uk.kernel.org #RemoteDir: /pub/linux/kernel/v2.2 #LocalDir: /home/kernel #File: linux-2.2..tar.gz # This rule keeps the most recent version of Wine in /home/wine # The yymmdd format date is parsed out of the filename. Yes it works after # Y2K :-) #Name: WineMain #Host: tsx-11.mit.edu #RemoteDir: /pub/linux/ALPHA/Wine/development #LocalDir: /home/wine #File: Wine-.tar.gz # This rule downloads all files ending in ".rpm" # Note the use of a regular expression; arbitrary regular expressions # may be used # If the first character of the File field is "/", this denotes a # regular expression #Name: Crypto RPMs #Host: ftp.replay.com #RemoteDir: /pub/crypto/redhat/i386 #LocalDir: /home/ftp/crypto #File: /.*\.rpm # This rule downloades all files ending in ".rpm" # Additionally local ".rpm" files not found on the remote site are deleted # The chacter "!" appearing after the "/" (denoting regexp) causes the # deletion behaviour #Name: Crypto RPMs #Host: ftp.replay.com #RemoteDir: /pub/crypto/redhat/i386 #LocalDir: /home/ftp/crypto #File: /!.*\.rpm # This small yet often useful feature, namely , can be useful for # stuff like samba which likes subversions 'a', 'b', etc. An example is # probably best. New in v0.1.1 # Note that the character is _optional_, so 2.0.5 is a match, but 2.0.5a # is a better match and 2.0.5b better still. If you want the character # non-optional, use #Name: Samba (UK mirror) #Host: uk.samba.org #RemoteDir: /packages/samba #LocalDir: /home/ftp/samba #File: file-2.0..tar.gz ftpgrab-0.1.5/fgmrank.h0000640003136300116100000000161006715647107013560 0ustar cevanseng#ifndef _FGMRANK_H #define _FGMRANK_H // fgmrank.h // // FGMatchRanking // // Class encapsulating "rank" given to a filename. Typically, the newer // the version specified within the filename, the higher the rank class FGMatchRanking { public: // Default constructor FGMatchRanking(); // Need these for the mm ~FGMatchRanking(); FGMatchRanking(const FGMatchRanking& other); FGMatchRanking& operator=(const FGMatchRanking& other); // Aside from quality of match, was this a match at all? bool IsMatch(void) const; // Compare two rankings bool operator<(const FGMatchRanking& other) const; // Stuff another value on the stack void AddValue(int value); // Set the match flag void SetMatch(void); private: // That fabled Cheshire Cat to hide implementation details again struct Internal_FGMatchRanking; Internal_FGMatchRanking* mpInternals; }; #endif // _FGMRANK_H ftpgrab-0.1.5/fglogger.h0000640003136300116100000000336507022701226013721 0ustar cevanseng#ifndef _FGLOGGER_H #define _FGLOGGER_H // fglogger.h // Simple class encapsulating the event logger #ifndef _FGSTRING_H #include "fgstring.h" #endif // Fun and games!! struct _IO_FILE; typedef struct _IO_FILE FILE; class FGException; class FGLogger { public: // Enum of log severities enum EFGLogSeverity { kFGLLDebug = 1, kFGLLVerbose, kFGLLInfo, kFGLLWarn, kFGLLErr, kFGLLBadNews }; // To shut up some warnings FGLogger(); // The all important methods to log things void LogMsg(const FGString& msg, EFGLogSeverity severity, bool locked = false); void LogException(const FGException& e, bool locked = false); // Method to access the logger object - singleton pattern static FGLogger& GetLogger(void); // To set the log file name static void SetLogName(const FGString& name); // Special lock methods for client activity that needs to guarantee // sequentially logged messages appear uninterrupted by other thread // logging activity static void Lock(void); static void Unlock(void); private: // Banned! FGLogger(const FGLogger& other); FGLogger& operator=(const FGLogger& other); // Construct the logger FGLogger(const FGString& fileName); // The actual log instance static FGLogger* mspLogger; // The filename to log to - must be set before first call to // GetLogger() static FGString msFileName; // Internal methods FGString GetDateStamp(void) const; // Stream we are logging to FILE* mpFile; // Helper classes // Urgh. Can't forward declare pthread_mutex_t => use of void* class MutexAutoUnlocker { public: MutexAutoUnlocker(); void Prime(void* pMutex); ~MutexAutoUnlocker(); private: void* mpMutex; }; }; #endif // _FGLOGGER_H ftpgrab-0.1.5/BUILD0000640003136300116100000000045311271655735012547 0ustar cevansengTo build type "make". If it doesn't build drop me a note. For the record I have tested it on RH5.1 machine with egcs as the C++ compiler. It also works on my home machine (RH5.9 beta, egcs-1.1.2) Modernized: builds with "make" on my Ubuntu-8.04 based machine now. Modernized #2: also Ubuntu-9.04 ftpgrab-0.1.5/fgfshelp.h0000640003136300116100000000124506674312073013730 0ustar cevanseng#ifndef _FGFSHELP_H #define _FGFSHELP_H // fgfshelp.h // // Helper class for local filesystem operations; static members only class FGDirListing; class FGString; class FGFileSystemHelper { public: // Get a _local_ dir listing static FGDirListing GetDirListing(const FGString& dir); // Transfer data from one (remote) fd to another static void TransferRemoteToLocal(int fromFD, int toFD, int bytes); // Create local file to put data in static int CreateLocalFile(const FGString& file); // Delete a local file static void DeleteFile(const FGString& file); private: // Constants static const int msTransferBufSize = 1024 * 64; }; #endif // _FGFSHELP_H ftpgrab-0.1.5/fgdyymmddcomp.h0000640003136300116100000000126506702375015014774 0ustar cevanseng#ifndef _FGDYYMMDD_H #define _FGDYYMMDD_H // fgdyyddmm.h // // FGDateYYMMDDComponent // // Class to handle matching and ranking of date components // e.g. Wine-990328.tar.gz etc. #ifndef _FGFNCOMP_H #include "fgfncomp.h" #endif class FGDateYYMMDDComponent : public FGFileNameComponent { public: // Default constructor FGDateYYMMDDComponent(); virtual bool MatchAndRankComponent(FGString& fnameRemainder, int* pMatchVal) const; virtual ~FGDateYYMMDDComponent(); private: // Banned! FGDateYYMMDDComponent(const FGDateYYMMDDComponent& other); FGDateYYMMDDComponent& operator=(const FGDateYYMMDDComponent& other); }; #endif // _FGDYYMMDD_H ftpgrab-0.1.5/fgpickall.h0000640003136300116100000000124106702232167014057 0ustar cevanseng#ifndef _FGPICKALL_H #define _FGPICKALL_H // fgpickall.h // // Implementation of a file picker that selects all remote files for // which we do not have local copies #ifndef _FGFPICKI_H #include "fgfpicki.h" #endif class FGConnectionInterface; class FGPickAll : public FGFilePickerInterface { public: // Must be constructed with connection interface FGPickAll(FGConnectionInterface* pConnIf); virtual FGActionList DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir); private: // Banned! FGPickAll(const FGPickAll& other); FGPickAll& operator=(const FGPickAll& other); }; #endif // _FGPICKALL_H ftpgrab-0.1.5/fgftpcon.cc0000640003136300116100000002553211267436570014106 0ustar cevanseng// fgftpcon.cc #include "fgftpcon.h" #include "fgexc.h" #include "fgstring.h" #include "fgdlist.h" #ifndef _FGGLOB_H #include "fgglob.h" #endif #include #include #include #include #include // For IPTOS_THROUGHPUT #include #include #include #include #include #include #include #include const int FGFTPCon::msFTPPort = IPPORT_FTP; const int FGFTPCon::msRespPassRequired = 331; const int FGFTPCon::msRespBadLogin = 530; const int FGFTPCon::msRespLoginOK = 230; const int FGFTPCon::msRespNotFound = 550; const int FGFTPCon::msRespTransferDone = 226; const int FGFTPCon::msRespChangeDirOK = 250; const int FGFTPCon::msRespCommandOK = 200; const int FGFTPCon::msRespGoodbye = 221; const int FGFTPCon::msRespPleaseLogin = 220; const int FGFTPCon::msRespPassiveOK = 227; const int FGFTPCon::msRespTransferOpen = 150; FGFTPCon::FGFTPCon() : mConnected(false), mCommandFD(-1), mpCommandStream(NULL), mDataFD(-1), mRemotePort(-1), mpDirStream(NULL), mRemoteAddr((unsigned long)-1) { } FGFTPCon::~FGFTPCon() { assert(mConnected == false); assert(mDataFD == -1); } void FGFTPCon::Connect(const FGString& host) { // Preconditions assert(mConnected == false); struct hostent resBuf; int thread_h_errno; struct hostent* pHost; char randomBuf[1024]; // Look up the host (void) gethostbyname_r((const char*)host, &resBuf, randomBuf, sizeof(randomBuf), &pHost, &thread_h_errno); if (pHost == NULL) { if (thread_h_errno == TRY_AGAIN) { throw FGException(FGException::kHostResolveRetryableFail, host); } else { throw FGException(FGException::kHostResolveFail, host); } } // Ok make a socket and connect it to the host's FTP port InternalConnect(&mCommandFD, msFTPPort, pHost->h_addr); mpCommandStream = fdopen(mCommandFD, "r+"); mConnected = true; // Parse the FTP server's greeting. Throw exception if we're not // let in cleanly. try { do { GetResponse(); switch (mResponse) { case msRespLoginOK: break; case msRespBadLogin: throw FGException(FGException::kAccessDenied); case msRespPleaseLogin: SendCommand("user anonymous"); break; case msRespPassRequired: { FGString pword("pass ftpgrab@"); // Fix: if the hostname has come back not fully qualified then // don't append it to "ftpgrab@". This is because many FTP sites // reject a hostname without a "." in it. For example the password // "ftpgrab@localhost" often fails. if (FGGlob::gHostName.Contains('.')) { pword += FGGlob::gHostName; } SendCommand(pword); break; } default: throw FGException(FGException::kResponseMalformed); } } while (mResponse != msRespLoginOK); // Set type to binary SendCommand("type i"); GetResponse(); // XXX - check response? // Bargain - we're connected } catch (FGException&) { mConnected = false; fclose(mpCommandStream); mpCommandStream = NULL; throw; } return; } void FGFTPCon::GetResponse(void) { // Preconditions assert(mConnected == true); assert(mCommandFD >= 0); char msgType; do { // Throws exception if overly long line encountered GetLine(); int len = strlen(mBuf); // Need at least 4 chars - 3 digit number and carrige return if (len < 4 || !isdigit(mBuf[0]) || !isdigit(mBuf[1]) || !isdigit(mBuf[2])) { // Damn. ProFTPd appears to emit greeting lines not prefixed by // a numeric response code. So we must assume the line is OK //throw FGException(FGException::kResponseMalformed, "Bad numeric response"); msgType = '-'; continue; } // Expect space or "-" for more lines msgType = mBuf[3]; char miniBuf[4]; miniBuf[3] = '\0'; strncpy(miniBuf, mBuf, 3); mResponse = atoi(miniBuf); } while (msgType == '-'); } void FGFTPCon::GetLine(void) { // Preconditions assert(mConnected == true); assert(mpCommandStream != NULL); char* pRet = fgets(mBuf, sizeof(mBuf), mpCommandStream); if (pRet == 0) { // Oh dear throw FGException(FGException::kConnectLost); } if (strchr(mBuf, '\n') == NULL) { throw FGException(FGException::kResponseTooLong); } } void FGFTPCon::SendCommand(const FGString& cmd) const { // Preconditions assert(mConnected == true); assert(mpCommandStream != NULL); char cmdbuf[2048]; snprintf(cmdbuf, sizeof(cmdbuf), "%s\r\n", (const char*)cmd); int numWrite = strlen(cmdbuf); int written = fwrite(cmdbuf, 1, numWrite, mpCommandStream); if (written != numWrite) { throw FGException(FGException::kConnectLost); } } void FGFTPCon::ChangeDir(const FGString& dir) { // Preconditions assert(mConnected == true); FGString command("cwd "); command += dir; SendCommand(command); GetResponse(); if (mResponse == msRespNotFound) { throw FGException(FGException::kNoSuchDir, dir); } else if (mResponse != msRespChangeDirOK) { throw FGException(FGException::kResponseUnexpected); } } void FGFTPCon::SetupPassivePort(void) { // Preconditions assert(mRemotePort == -1); SendCommand("pasv"); GetResponse(); if (mResponse != msRespPassiveOK) { throw FGException(FGException::kResponseUnexpected); } // We have to parse a line which for example may look like // "227 Entering Passive Mode (127,0,0,1,4,133)" int a, b, c, d, e, f; char* pBrace = strchr(mBuf, '('); if (pBrace == NULL) { throw FGException(FGException::kResponseMalformed, "Missing brace in PASV reply"); } int found = sscanf(pBrace, "(%d,%d,%d,%d,%d,%d)", &a, &b, &c, &d, &e, &f); if (found != 6) { throw FGException(FGException::kResponseMalformed, "Failed to parse 6 args in PASV reply"); } mRemotePort = (e << 8) | f; char buf[32]; snprintf(buf, sizeof(buf), "%d.%d.%d.%d", a, b, c, d); // Obsolete but never mind.. :) mRemoteAddr = inet_addr(buf); // Connect to indicated remote port and IP InternalConnect(&mDataFD, mRemotePort, &mRemoteAddr); // It's a data stream (dir listing or file) so set IP TOS // to maximize throughput int tosVal = IPTOS_THROUGHPUT; setsockopt(mDataFD, IPPROTO_IP, IP_TOS, &tosVal, sizeof(tosVal)); } FGDirListing FGFTPCon::GetDirListing(void) { // Preconditions assert(mConnected == true); assert(mpCommandStream != NULL); assert(mDataFD == -1); assert(mpDirStream == NULL); FGDirListing ret("NOTSET"); // Use passive mode due to laziness - it's marginally easier to code // as a first effort SetupPassivePort(); // Get remote end to set up data transfer // NLST just lists filename without extra details // So we don't use that we use LIST instead SendCommand("list"); GetResponse(); if (mResponse != msRespTransferOpen) { throw FGException(FGException::kResponseUnexpected); } // Make a stream from connection mpDirStream = fdopen(mDataFD, "r"); // Read off connection - 1 line, 1 file while (fgets(mBuf, sizeof(mBuf), mpDirStream) != NULL) { // Safety if (strlen(mBuf) == 0) { continue; } // Don't forget to strip any trailing \r or \n characters char* pEndNul = mBuf + strlen(mBuf); bool replaced; do { replaced = false; pEndNul--; char theChar = *pEndNul; if (theChar == '\r' || theChar == '\n') { *pEndNul = '\0'; replaced = true; } } while (replaced && pEndNul > mBuf); int fSize; int duff; int day; char fileBuf[256]; char duffBuf[32]; char perms[20]; char mon[10]; char time[10]; int numFound = sscanf(mBuf, "%19s %d %31s %31s %d %9s %d %9s %255s", perms, &duff, duffBuf, duffBuf, &fSize, mon, &day, time, fileBuf); if (numFound == 9) { FGString theFileName(fileBuf); bool isDir = (perms[0] == 'd'); bool isFile = (perms[0] == '-'); FGFileInfo theFileInfo(theFileName, fSize, isDir, isFile); ret.push_back(theFileInfo); } } // Deallocate resources fclose(mpDirStream); mpDirStream = NULL; mDataFD = -1; mRemotePort = -1; mRemoteAddr = (unsigned long)-1; // XXX check return for "transfer complete" GetResponse(); return ret; } int FGFTPCon::GetFile(const FGString& file) { // Preconditions assert(mConnected == true); assert(mpCommandStream != NULL); assert(mDataFD == -1); assert(mpDirStream == NULL); // Use passive mode due to laziness - it's marginally easier to code // as a first effort SetupPassivePort(); // Get remote end to set up data transfer // NLST just lists filename without extra details - date etc. FGString cmd("retr "); cmd += file; try { SendCommand(cmd); GetResponse(); if (mResponse != msRespTransferOpen) { throw FGException(FGException::kResponseUnexpected); } } catch (FGException&) { PostFileTransfer(); throw; } // Try and set low water mark on socket for efficency // (less syscalls per transfer) // Bah! Linux 2.2 doesn't support this #ifdef SO_RCVLOWAT int val = 1024 * 64; (void) setsockopt(mDataFD, SOL_SOCKET, SO_RCVLOWAT, &val, sizeof(val)); #endif return mDataFD; } void FGFTPCon::Disconnect(void) { assert(mConnected == true); assert(mDataFD == -1); fclose(mpCommandStream); mpCommandStream = NULL; } void FGFTPCon::InternalConnect(int* pDestFD, int port, void* pInetAddr) { // Connect to indicated remote port and IP // Hardcoded constant should get got from "getprotobyname(tcp)" *pDestFD = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); if (*pDestFD == -1) { throw FGException(FGException::kConnectResourceAllocFail, strerror(errno)); } #ifdef SO_KEEPALIVE // Get the OS to check the remote end doesn't silently drop the connection int one = 1; setsockopt(*pDestFD, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(int)); #endif struct sockaddr_in addr; addr.sin_family = AF_INET; addr.sin_port = htons(port); memcpy(&addr.sin_addr, pInetAddr, sizeof(addr.sin_addr)); int conRet = connect(*pDestFD, (struct sockaddr*)&addr, sizeof(addr)); if (conRet == -1) { switch (errno) { case ECONNREFUSED: throw FGException(FGException::kConnectRefused); case ETIMEDOUT: case ENETUNREACH: throw FGException(FGException::kConnectHostUncontactable, strerror(errno)); default: throw FGException(FGException::kConnectMiscFail, strerror(errno)); } } } void FGFTPCon::PostFileTransfer(void) { // Preconditions assert(mConnected == true); assert(mDataFD != -1); // Close stuff before checking response otherwise an exception may // be thrown => resource leak close(mDataFD); mDataFD = -1; mRemotePort = -1; mRemoteAddr = (unsigned long)-1; // Eat up the (hopefully!) "transfer OK" message GetResponse(); // XXX verify response } ftpgrab-0.1.5/README.gcc.2.70000640003136300116100000000143211267247221013673 0ustar cevansengI have a report that g++ version 2.7 fails to compile ftpgrab. AFAIK, exception support in gcc 2.7 was shall we say "suboptimal" :-) The error given is In file included from main.cc:10: fglogger.h:58: warning: `class FGLogger' only defines private constructors and has no friends main.cc: In function `int main()': main.cc:46: warning: `catch', `throw', and `try' are all C++ reserved words main.cc:46: exception handling disabled, use -fhandle-exceptions to enable. main.cc:62: `e' undeclared (first use this function) main.cc:62: (Each undeclared identifier is reported only once main.cc:62: for each function it appears in.) There is currently some confusion as to whether adding "-fhandle-exceptions" actually works. Thanks to Urivan Saaib for reporting this problem! ftpgrab-0.1.5/fgpickbest.cc0000640003136300116100000001141511267246505014411 0ustar cevanseng// fgpickbest.cc #include "fgpickbest.h" #ifndef _FGALIST_H #include "fgalist.h" #endif #ifndef _FGBDFNAME_H #include "fgbdfname.h" #endif #ifndef _FGDLIST_H #include "fgdlist.h" #endif #ifndef _FGMRANK_H #include "fgmrank.h" #endif #ifndef _FGDLACTION_H #include "fgdlaction.h" #endif #ifndef _FGDELACTION_H #include "fgdelaction.h" #endif #ifndef __SGI_STL_MAP_H #include #endif #include #undef DEBUG_PICKER struct FGPickBest::Internal_FGPickBest { Internal_FGPickBest(const FGString& fname); int mRevisionsCount; FGBrokenDownFileName mFileBits; }; FGPickBest::Internal_FGPickBest::Internal_FGPickBest(const FGString& fname) : mFileBits(fname) { } FGPickBest::FGPickBest(FGConnectionInterface* pConnIf, const FGString& match, int count) : FGFilePickerInterface(pConnIf), mpInternals(0) { assert(count > 0); mpInternals = new Internal_FGPickBest(match); mpInternals->mRevisionsCount = count; } FGPickBest::~FGPickBest() { if (mpInternals) { delete mpInternals; mpInternals = 0; } } FGActionList FGPickBest::DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir) { FGActionList ret; // Special structures used in this algorithm typedef enum _floc { kLocal = 0, kRemote, kBoth } EFGFileLocation; // n.b. 2nd arg, int, isn't allowed (by ANSI) to be of type // EFGFileLocation typedef std::pair FGPickMapInfo; typedef std::map FGPickMap; typedef FGPickMap::iterator FGPickMapIterator; typedef FGPickMap::const_iterator FGPickMapCIterator; // The sacred map itself FGPickMap mainMap; // Inject all local matches into map std::vector::const_iterator iFiles; for (iFiles = localDir.begin(); iFiles != localDir.end(); iFiles++) { // Rule based matching only considers files if (!iFiles->IsRegularFile()) { continue; } FGString fileName = iFiles->GetFileName(); FGMatchRanking thisRank = mpInternals->mFileBits.GetRanking(fileName); if (thisRank.IsMatch()) { #ifdef DEBUG_PICKER printf("Local file found\n"); #endif FGPickMapInfo newPair; newPair.first = *iFiles; newPair.second = kLocal; FGPickMap::value_type theValue(thisRank, newPair); mainMap.insert(theValue); } } // Now inject all remote files into the map // Any duplicates get their location flag set to "both" for (iFiles = remoteDir.begin(); iFiles != remoteDir.end(); iFiles++) { FGString fileName = iFiles->GetFileName(); // Rule based matching only considers files if (!iFiles->IsRegularFile()) { continue; } FGMatchRanking thisRank = mpInternals->mFileBits.GetRanking(fileName); // If we have less matches than we want, or the current match is // better than our worst match yet, take it if (thisRank.IsMatch()) { #ifdef DEBUG_PICKER printf("Remote file found\n"); #endif // Two cases - rank value already in map, or not FGPickMapIterator iMap = mainMap.find(thisRank); if (iMap == mainMap.end()) { // Not there yet FGPickMapInfo newPair; newPair.first = *iFiles; newPair.second = kRemote; FGPickMap::value_type theValue(thisRank, newPair); mainMap.insert(theValue); } else { // Yes, already there iMap->second.second = kBoth; } } } // Use our map to decide actions // Empty map? Just get out.. if (mainMap.size() == 0) { return ret; } FGPickMapCIterator iAllMap = mainMap.end(); // end() points off end => back iterator off one iAllMap--; int wanted = mpInternals->mRevisionsCount; do { // Case 1: we are in the wanted region if (wanted > 0) { // Within the wanted region, make download actions // for all "remote only" files if (iAllMap->second.second == kRemote) { ret.push_back(new FGDownloadAction(iAllMap->second.first.GetFileName(), mpConnIf, localDir.GetDirName(), iAllMap->second.first.GetSize())); } } else { // Case 2: we are in the ditch region // Within the ditch region, make delete actions for // all "local only" or "local and remote" files EFGFileLocation fl = (EFGFileLocation)iAllMap->second.second; if (fl == kLocal || fl == kBoth) { ret.push_back(new FGDeleteAction(iAllMap->second.first.GetFileName(), localDir.GetDirName())); } } // Make sure we only try and keep the top N matches on local disk wanted--; // Decrement the iterator (move towards worse matches) } while (iAllMap-- != mainMap.begin()); return ret; } ftpgrab-0.1.5/fgscomp.h0000640003136300116100000000126106702374774013576 0ustar cevanseng#ifndef _FGSCOMP_H #define _FGSCOMP_H // fgscomp.h // // FGStringComponent: handles matching of plain string bits of a filename #include "fgfncomp.h" #ifndef _FGSTRING_H #include "fgstring.h" #endif class FGStringComponent : public FGFileNameComponent { public: // Construct from string fragment FGStringComponent(const FGString& str); virtual bool MatchAndRankComponent(FGString& fnameRemainder, int* pMatchVal) const; virtual ~FGStringComponent(); private: // Banned! FGStringComponent(const FGStringComponent& other); FGStringComponent& operator=(const FGStringComponent& other); FGString mStringBit; }; #endif // _FGSCOMP_H ftpgrab-0.1.5/fgfpicki.cc0000640003136300116100000000021406674515012014043 0ustar cevanseng// fgfpicki.cc #include "fgfpicki.h" FGFilePickerInterface::FGFilePickerInterface(FGConnectionInterface* pConnIf) : mpConnIf(pConnIf) { } ftpgrab-0.1.5/fgicomp.h0000640003136300116100000000126706702374743013566 0ustar cevanseng#ifndef _FGICOMP_H #define _FGICOMP_H // fgicomp.h // // FGIntegerComponent: handles matching of numeric bits of filename. // Probably the most used ranking primitive; e.g. linux-2.2..tar.gz // and 2.2.3 is newer than 2.2.2 :-) #include "fgfncomp.h" class FGIntegerComponent : public FGFileNameComponent { public: // Default constructor FGIntegerComponent(); virtual bool MatchAndRankComponent(FGString& fnameRemainder, int* pMatchVal) const; virtual ~FGIntegerComponent(); private: // Banned! FGIntegerComponent(const FGIntegerComponent& other); FGIntegerComponent& operator=(const FGIntegerComponent& other); }; #endif // _FGICOMP_H ftpgrab-0.1.5/fgdelaction.cc0000640003136300116100000000175106717336470014556 0ustar cevanseng// fgdelaction.cc #include "fgdelaction.h" #ifndef _FGFSHELP_H #include "fgfshelp.h" #endif #ifndef _FGSTRING_H #include "fgstring.h" #endif #ifndef _FGEXC_H #include "fgexc.h" #endif FGDeleteAction::FGDeleteAction(const FGString& fname, const FGString& localDir) : mFileName(fname), mLocalDir(localDir) { } FGDeleteAction::~FGDeleteAction() { } void FGDeleteAction::VirtualDo(void) const { // Just nuke the file FGString file = mLocalDir; file += '/'; file += mFileName; // If we get a "not found" exception, ignore it // This is because if something else deletes it, we don't care. We // only care that the file is _gone_ :) // I introduced this because two threads can race such that two of // them notice the same file needs wasting try { FGFileSystemHelper::DeleteFile(file); } catch (FGException& e) { if (e.GetReason() != FGException::kLocalNoSuchFile) { throw; } } } void FGDeleteAction::Abort(void) const { } ftpgrab-0.1.5/fgdlaction.h0000640003136300116100000000175207101737707014250 0ustar cevanseng#ifndef _FGDLACTION_H #define _FGDLACTION_H // fgdlaction.h // // Implementation class of the download action #include "fgactioni.h" #ifndef _FGSTRING_H #include "fgstring.h" #endif class FGConnectionInterface; class FGDownloadAction : public FGActionInterface { public: // Construct with filename to download FGDownloadAction(const FGString& fname, FGConnectionInterface* pConnIf, const FGString& localDir, int size = -1); ~FGDownloadAction(); // Overridden virtual Do method virtual void VirtualDo(void) const; // And Abort() virtual void Abort(void) const; private: // Banned!! FGDownloadAction(const FGDownloadAction& other); FGDownloadAction& operator=(const FGDownloadAction& other); FGConnectionInterface* mpConnIf; FGString mFileName; FGString mLocalDir; // File descriptor pointing to the local file copy we are downloading int mDestFD; // Expected size of the file we are downloading int mFileSize; }; #endif // _FGDLACTION_H ftpgrab-0.1.5/COPYING0000640003136300116100000000007606706650245013017 0ustar cevansengLicense is GPL. TODO: Replace this file with GPL licence :-) ftpgrab-0.1.5/fgglob.h0000640003136300116100000000042506707461071013371 0ustar cevanseng#ifndef _FGGLOB_H #define _FGGLOB_H class FGGlob { public: static FGString gHostName; static bool gLogDisabled; // Max. num threads on the go at once static int gMaxThreads; // Are we outputting progress to stdout static bool gVerbose; }; #endif // _FGGLOB_H ftpgrab-0.1.5/fgalist.h0000640003136300116100000000063011267245601013554 0ustar cevanseng#ifndef _FGALIST_H #define _FGALIST_H // fgalist.h // // FGActionList - details actions to be taken (if any) to satisfy // a given grab rule e.g. fetch file A, delete file B #ifndef __SGI_STL_VECTOR_H #include #endif class FGActionInterface; class FGActionList : public std::vector { public: void DoActions(void) const; void FreeResources(void); }; #endif // _FGALIST_H ftpgrab-0.1.5/fgpickall.cc0000640003136300116100000000235411267245673014233 0ustar cevanseng// fgpickall.cc #include "fgpickall.h" // Dir listings #ifndef _FGDLIST_H #include "fgdlist.h" #endif // Action lists #ifndef _FGALIST_H #include "fgalist.h" #endif // The download file action #ifndef _FGDLACTION_H #include "fgdlaction.h" #endif FGPickAll::FGPickAll(FGConnectionInterface* pConnIf) : FGFilePickerInterface(pConnIf) { } FGActionList FGPickAll::DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir) { // The picker for the "normal" wildcard operation (non recursive), // leaves all local files intact // Very simple rule: grab any file in the remoteDir that is // not in the localDir FGActionList ret; // Get list of files in remote but not local FGDirListing newFiles = FGDirListing::GetRHSOnlyFiles(localDir, remoteDir); // Make a "download" action for all the above files std::vector::const_iterator iFiles; for (iFiles = newFiles.begin(); iFiles != newFiles.end(); iFiles++) { FGActionInterface* pNewAction; pNewAction = new FGDownloadAction(iFiles->GetFileName(), mpConnIf, localDir.GetDirName(), iFiles->GetSize()); ret.push_back(pNewAction); } return ret; } ftpgrab-0.1.5/fgfshelp.cc0000640003136300116100000001022111271650516014054 0ustar cevanseng// fgfshelp.cc #include "fgfshelp.h" #ifndef _FGDLIST_H #include "fgdlist.h" #endif #ifndef _FGSTRING_H #include "fgstring.h" #endif #ifndef _FGEXC_H #include "fgexc.h" #endif #include #include #include #include #include #include #include #include #include #include FGDirListing FGFileSystemHelper::GetDirListing(const FGString& dir) { FGDirListing ret(dir); DIR* pDir = opendir(dir); if (pDir == NULL) { // Oh dear throw FGException(FGException::kLocalNoSuchDir, dir); } struct dirent* pDirEnt; while ((pDirEnt = readdir(pDir)) != NULL) { // Only report the file if it's not partially downloaded // (this is indicated by permissions ---------T) struct stat statBuf; FGString fname(dir); fname += '/'; fname += pDirEnt->d_name; if (stat(fname, &statBuf) != -1) { if (statBuf.st_mode != (S_ISVTX | S_IWUSR)) { FGFileInfo fileInfo(pDirEnt->d_name, statBuf.st_size, S_ISDIR(statBuf.st_mode), S_ISREG(statBuf.st_mode)); ret.push_back(fileInfo); } } } closedir(pDir); return ret; } void FGFileSystemHelper::TransferRemoteToLocal(int fromFD, int toFD, int bytes) { // Preconditions assert(fromFD >= 0); assert(toFD >= 0); // Transfer buffer char buf[msTransferBufSize]; // Accounting int bytesWritten = 0; bool hitEOF = false; // Monitoring fd_set remoteFDset; FD_ZERO(&remoteFDset); FD_SET(fromFD, &remoteFDset); try { while ((bytes == -1 || bytesWritten < bytes) && !hitEOF) { // Timeout on remote read // For now we will use 5 minutes struct timeval tv; tv.tv_usec = 0; tv.tv_sec = 60 * 5; int selectRet = select(fromFD+1, &remoteFDset, NULL, NULL, &tv); // Did we timeout if (selectRet == 0) { // How irritating throw FGException(FGException::kDownloadTimeout); } int numRead = read(fromFD, buf, sizeof(buf)); if (!numRead) { hitEOF = true; // Throw exception if expecting specific number of bytes if (bytes != -1 && bytesWritten < bytes) { throw FGException(FGException::kFileTooShort); } } else if (numRead == -1) { throw FGException(FGException::kNetworkReadError, strerror(errno)); } else { int writeRet = write(toFD, buf, numRead); if (writeRet == -1) { throw FGException(FGException::kWriteFailed, strerror(errno)); } bytesWritten += numRead; } } // end: while (want more bytes) } catch (FGException& e) { close(toFD); throw e; } // If we completed the transfer fully, update file permissions to // reflect this if (bytes == -1 || bytes == bytesWritten) { fchmod(toFD, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); } // We are no longer writing to the file - so lose the lock flock(toFD, LOCK_UN); // And close it close(toFD); } int FGFileSystemHelper::CreateLocalFile(const FGString& file) { // Create with permissions "1400" == "---w-----T" // This flags file as "in transit" int theFD = open(file, O_CREAT | O_WRONLY, S_ISVTX | S_IWUSR); if (theFD == -1) { FGString details(file); details += ": "; details += strerror(errno); throw FGException(FGException::kFileCreateFailed, details); } // Before truncating, lock the file - in case another thread / process // is currently downloading the file int lockRet = flock(theFD, LOCK_EX | LOCK_NB); if (lockRet == -1) { close(theFD); throw FGException(FGException::kConcurrentGrabs); } int ret = ftruncate(theFD, 0); (void) ret; return theFD; } void FGFileSystemHelper::DeleteFile(const FGString& file) { int delRet = unlink(file); if (delRet == -1) { switch (errno) { case EACCES: case EPERM: case EROFS: throw FGException(FGException::kLocalPermissionDenied); case ENOENT: throw FGException(FGException::kLocalNoSuchFile); default: throw FGException(FGException::kUnlinkFailed); } } } ftpgrab-0.1.5/fgpickregexpsync.cc0000640003136300116100000000560311267246613015645 0ustar cevanseng// fgpickregexpsync.cc #include "fgpickregexpsync.h" // Dir listings #ifndef _FGDLIST_H #include "fgdlist.h" #endif // Action lists #ifndef _FGALIST_H #include "fgalist.h" #endif // The download file action #ifndef _FGDLACTION_H #include "fgdlaction.h" #endif // The delete file action #ifndef _FGDELACTION_H #include "fgdelaction.h" #endif #ifndef _FGEXC_H #include "fgexc.h" #endif #include FGPickRegexpSync::FGPickRegexpSync(FGConnectionInterface* pConnIf, const FGString& regexp) : FGFilePickerInterface(pConnIf), mpRegExp(0) { mpRegExp = new regex_t; try { int ret = regcomp(mpRegExp, regexp, REG_EXTENDED | REG_NOSUB); if (ret != 0) { throw FGException(FGException::kInvalidRegexp); } } catch (FGException&) { delete mpRegExp; throw; } } FGPickRegexpSync::~FGPickRegexpSync() { if (mpRegExp != 0) { regfree(mpRegExp); delete mpRegExp; } } FGActionList FGPickRegexpSync::DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir) { // The picker for the "normal" operation (non recursive), // leaves all local files intact // Very simple rule: grab any file in the remoteDir that is // not in the localDir and matches regexp FGActionList ret; // Get list of files in local but not remote FGDirListing staleFiles = FGDirListing::GetRHSOnlyFiles(remoteDir, localDir); std::vector::const_iterator iFiles; // Pare down list to only regexp matches FGDirListing finalWasteList("NOTUSED"); for (iFiles = staleFiles.begin(); iFiles != staleFiles.end(); iFiles++) { int match = regexec(mpRegExp, iFiles->GetFileName(), 0, 0, 0); if (match == 0) { finalWasteList.push_back(*iFiles); } } // For each stale file, make an "erase" action for (iFiles = finalWasteList.begin(); iFiles != finalWasteList.end(); iFiles++) { FGActionInterface* pNewAction; pNewAction = new FGDeleteAction(iFiles->GetFileName(), localDir.GetDirName()); ret.push_back(pNewAction); } // Get list of files in remote but not local FGDirListing newFiles = FGDirListing::GetRHSOnlyFiles(localDir, remoteDir); // Pare down listing to only regexp matches FGDirListing finalList("NOTUSED"); for (iFiles = newFiles.begin(); iFiles != newFiles.end(); iFiles++) { int match = regexec(mpRegExp, iFiles->GetFileName(), 0, 0, 0); if (match == 0) { finalList.push_back(*iFiles); } } // Make a "download" action for all the above files for (iFiles = finalList.begin(); iFiles != finalList.end(); iFiles++) { FGActionInterface* pNewAction; pNewAction = new FGDownloadAction(iFiles->GetFileName(), mpConnIf, localDir.GetDirName(), iFiles->GetSize()); ret.push_back(pNewAction); } return ret; } ftpgrab-0.1.5/README0000640003136300116100000000423011271655707012641 0ustar cevansengThis is ftpgrab v0.1.4 Author: Chris Evans, , who is at pains to point out that this 10 year old code is not indicative of the quality of my more recent work :) ftpgrab is distributed under the GPLv2 license. What does ftpgrab do ==================== ftpgrab is a utility for maintaining FTP mirrors. In fact not unlike the "Mirror" perl program. However ftpgrab is oriented towards the smaller site which doesn't have the resources to mirror entire version trees of software. The primary "plus point" of ftpgrab is that it can base download decisions by parsing version numbers out of filenames. For example, ftpgrab will recognize that the file "linux-2.2.2.tar.gz" is newer than "linux-2.2.1.tar.gz" based on the version string. It will then download the new version and delete the old one when it is done, thus saving you mirroring 10's of kernel versions all at >10Mb each. Usage ===== To use ftpgrab, see the "ftpgrabrc" example file that comes with this tarball. Customize the file as needed. When done simply run the ftpgrab program whilst in the directory containing the "ftpgrabrc" file you want parsed. ftpgrab will parse the file, iterate across all download rules once, then exit. Hence if you want to run ftpgrab in a daemon like mode, do something like while true; do ./ftpgrab; sleep 12h; done In a future version this will not be neccessary (ftpgrab will gain a daemon-like mode). Problems? ========= If you have any problems it is likely to be my fault for lack of documentation. However, check out "fglog.out" to see if any useful messages crop up in there. If you are still stuck drop me a note (my e-mail is at the top of this README), and I will try and help you. There is every possibility you will hit bugs. Limitations =========== - HTTP protocol not yet supported. Will be in a future release. - Needs recursive fetch options - Needs version parsing in directory components too - See TODO - Only tested against a small amount of FTP servers To those who say "why bother" ============================= Yeah I know... Mirror does a very similar thing. But I felt like writing my own tool that did exactly what we needed ;-) Cheers Chris ftpgrab-0.1.5/fgpickallrecurse.h0000640003136300116100000000000006674313740015445 0ustar cevansengftpgrab-0.1.5/fgstring.h0000640003136300116100000000221211271650252013741 0ustar cevanseng#ifndef _FGSTRING_H #define _FGSTRING_H // fgstring.h // // My own C++ string class // Because: the only one I can find on my system is woefully inadequate class FGString { public: // Constructors // Default constructor FGString(); // Construct from const char* FGString(const char* pStr); // Copy constructor FGString(const FGString& other); // Assignment op. FGString& operator=(const FGString& other); // Destructor ~FGString(); // Comparison operators bool operator==(const FGString& other) const; bool operator!=(const FGString& other) const; // Casts // Cast to a const char* operator const char*() const; // Concatenation void operator+=(const FGString& other); void operator+=(char other); // Get methods unsigned int GetLength(void) const; char operator[](int pos) const; // Substring methods FGString Left(unsigned int num) const; FGString Right(unsigned int num) const; bool Contains(char c) const; // Conversions int AToI(void) const; bool IsEmpty(void) const; void MakeEmpty(void); private: // Internals char* mpString; unsigned int mLength; }; #endif // _FGSTRING_H ftpgrab-0.1.5/fgexc.cc0000640003136300116100000000327507101750445013364 0ustar cevanseng// fgexc.cc #include "fgexc.h" // String table of exception messages const char* const exceptionTexts[] = {"Hostname lookup failed", "Hostname lookup failed", "Couldn't allocate resources to connect", "Connection refused at remote site", "No response from remote site", "Failed to connect to remote site", "Lost connection to remote site", "FTP response too large!", "FTP response malformed!", "FTP response code unexpected!", "FTP server refuses connection", "FTP server is full", "Remote directory not found", "Remote file not found", "Local directory not found", "Local file not found", "Permission denied (local)", "Timeout downloading file", "Cannot open config file", "Missing `:' in config file", "Duplicate token in config file", "Config file ended in middle of rule", "Failed to create local file", "Detected concurrent download of same file", "Failed to unlink local file", "I/O error writing local file", "Could not open/create log file", "Unexpected end of string parsing filename rule", "Unexpected character parsing filename rule", "Unrecognized component token", "Invalid regular expression", "Error reading from network", "Unexpected file EOF (remote died?)" }; FGException::FGException(EFGExceptions reason, const FGString& details) : mWhat(reason), mDetails(details) { } FGException::FGException(EFGExceptions reason) : mWhat(reason) { } FGException::EFGExceptions FGException::GetReason(void) const { return mWhat; } FGString FGException::GetFullDetails(void) const { FGString ret(exceptionTexts[(int)mWhat]); if (!mDetails.IsEmpty()) { ret += ": "; ret += mDetails; } return ret; } ftpgrab-0.1.5/fgftpcon.h0000640003136300116100000000356507101737162013743 0ustar cevanseng#ifndef _FGFTPCON_H #define _FGFTPCON_H // fgftpcon.h // // FTPGrabFTPConnection // // Class implementing getting of files over FTP protocol #ifndef _FGCONI_H #include "fgconi.h" #endif // Fun and games!! struct _IO_FILE; typedef struct _IO_FILE FILE; class FGFTPCon : public FGConnectionInterface { public: // Default constructor FGFTPCon(); // Destructor ~FGFTPCon(); // Interface methods from base class virtual void Connect(const FGString& host); virtual void ChangeDir(const FGString& dir); virtual FGDirListing GetDirListing(void); virtual int GetFile(const FGString& file); virtual void PostFileTransfer(void); virtual void Disconnect(void); private: // Banned! FGFTPCon(const FGFTPCon& other); FGFTPCon& operator=(const FGFTPCon& other); // Implementation details bool mConnected; // File descriptor of connected command stream int mCommandFD; // Stream version of the above FILE* mpCommandStream; // Buffer for storing/parsing the FTP server replies char mBuf[1024]; // Value of FTP server's reply int mResponse; // Details for remote data connection int mDataFD; int mRemotePort; FILE* mpDirStream; unsigned long mRemoteAddr; // Internal constants static const int msFTPPort; static const int msRespPassRequired; static const int msRespBadLogin; static const int msRespLoginOK; static const int msRespNotFound; static const int msRespTransferDone; static const int msRespChangeDirOK; static const int msRespCommandOK; static const int msRespGoodbye; static const int msRespPleaseLogin; static const int msRespPassiveOK; static const int msRespTransferOpen; // Internal functions void GetResponse(void); void GetLine(void); void SendCommand(const FGString& cmd) const; void SetupPassivePort(void); void InternalConnect(int* pDestFD, int port, void* pInetAddr); }; #endif // _FGFTPCON_H ftpgrab-0.1.5/fgicomp.cc0000640003136300116100000000172407101740260013703 0ustar cevanseng// fgicomp.cc #include "fgicomp.h" #ifndef _FGSTRING_H #include "fgstring.h" #endif #include FGIntegerComponent::FGIntegerComponent() { } FGIntegerComponent::~FGIntegerComponent() { } bool FGIntegerComponent::MatchAndRankComponent(FGString& fnameRemainder, int* pMatchVal) const { // Initialize to be safe *pMatchVal = -1; // Sanity checks if (fnameRemainder.GetLength() < 1 || !isdigit(fnameRemainder[0])) { return false; } unsigned int digits = 0; // Build up the number here FGString theInt; do { theInt += fnameRemainder[digits]; digits++; } while (digits < fnameRemainder.GetLength() && isdigit(fnameRemainder[digits])); // Set the match value *pMatchVal = theInt.AToI(); // Chop off the digits we consumed int charsLeft = fnameRemainder.GetLength() - digits; FGString bitLeft = fnameRemainder.Right(charsLeft); fnameRemainder = bitLeft; return true; } ftpgrab-0.1.5/fgconi.cc0000640003136300116100000000011106707432340013521 0ustar cevanseng#include "fgconi.h" FGConnectionInterface::~FGConnectionInterface() { } ftpgrab-0.1.5/fgscomp.cc0000640003136300116100000000155606702375241013731 0ustar cevanseng// fgscomp.cc #include "fgscomp.h" FGStringComponent::FGStringComponent(const FGString& str) : mStringBit(str) { } FGStringComponent::~FGStringComponent() { } bool FGStringComponent::MatchAndRankComponent(FGString& fnameRemainder, int* pMatchVal) const { // Match of a string fragment will always be valueless *pMatchVal = -1; // No match if our fragment is bigger than what's left int fragLen = mStringBit.GetLength(); int compLen = fnameRemainder.GetLength(); if (fragLen > compLen) { return false; } // Do comparison FGString leftBit = fnameRemainder.Left(fragLen); if (mStringBit != leftBit) { return false; } // OK so its a match // Truncate the filename remainder accordingly FGString fnameLeft = fnameRemainder.Right(compLen - fragLen); fnameRemainder = fnameLeft; return true; } ftpgrab-0.1.5/fghttpcon.h0000640003136300116100000000001706672051733014123 0ustar cevanseng// fghttpcon.h ftpgrab-0.1.5/fgpickregexp.cc0000640003136300116100000000401711267246575014755 0ustar cevanseng// fgpickregexp.cc #include "fgpickregexp.h" // Dir listings #ifndef _FGDLIST_H #include "fgdlist.h" #endif // Action lists #ifndef _FGALIST_H #include "fgalist.h" #endif // The download file action #ifndef _FGDLACTION_H #include "fgdlaction.h" #endif #ifndef _FGEXC_H #include "fgexc.h" #endif #include FGPickRegexp::FGPickRegexp(FGConnectionInterface* pConnIf, const FGString& regexp) : FGFilePickerInterface(pConnIf), mpRegExp(0) { mpRegExp = new regex_t; try { int ret = regcomp(mpRegExp, regexp, REG_EXTENDED | REG_NOSUB); if (ret != 0) { throw FGException(FGException::kInvalidRegexp); } } catch (FGException&) { delete mpRegExp; throw; } } FGPickRegexp::~FGPickRegexp() { if (mpRegExp != 0) { regfree(mpRegExp); delete mpRegExp; } } FGActionList FGPickRegexp::DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir) { // The picker for the "normal" operation (non recursive), // leaves all local files intact // Very simple rule: grab any file in the remoteDir that is // not in the localDir and matches regexp FGActionList ret; // Get list of files in remote but not local FGDirListing newFiles = FGDirListing::GetRHSOnlyFiles(localDir, remoteDir); // Pare down listing to only regexp matches FGDirListing finalList("NOTUSED"); std::vector::const_iterator iFiles; for (iFiles = newFiles.begin(); iFiles != newFiles.end(); iFiles++) { int match = regexec(mpRegExp, iFiles->GetFileName(), 0, 0, 0); if (match == 0) { finalList.push_back(*iFiles); } } // Make a "download" action for all the above files for (iFiles = finalList.begin(); iFiles != finalList.end(); iFiles++) { FGActionInterface* pNewAction; pNewAction = new FGDownloadAction(iFiles->GetFileName(), mpConnIf, localDir.GetDirName(), iFiles->GetSize()); ret.push_back(pNewAction); } return ret; } ftpgrab-0.1.5/fgfileinfo.h0000640003136300116100000000133307101737616014240 0ustar cevanseng#ifndef _FGFILEINFO_H #define _FGFILEINFO_H // fgfileinfo.h #ifndef _FGSTRING_H #include "fgstring.h" #endif class FGFileInfo { public: // Construct from filename FGFileInfo(const FGString& filename); // Construct from filename, size, isDir, isFile FGFileInfo(const FGString& filename, int size, bool isDir, bool isFile); // Pesky STL needs this FGFileInfo(); // Do two files have identical attributes? bool operator==(const FGFileInfo& other) const; // Get methods const FGString& GetFileName(void) const; bool IsRegularFile(void) const; int GetSize(void) const; private: FGString mFileName; int mSize; bool mIsDir; bool mIsRegularFile; // TODO: Date modified }; #endif // _FGFILEINFO_H ftpgrab-0.1.5/TODO0000640003136300116100000000116207115533222012437 0ustar cevansengFor ? - Timeout on FTP conversation [done?] - HTTP protocol - recursive fetching - reget of partially transferred files - retry of temporary failures - download based on file size change only - download based on date stamp change only - Ability to parse directory names as well as filenames - Merge pick and picksync classes into one which takes boolean flag - Desist from the grotty "exceptions in constructors" scenario - Better use constructors/destructors to allocate/deallocate resources Probably won't do ================= - multiple alternative sites in one "rule" - reuse connections to identical remote hostnames ftpgrab-0.1.5/fgfileinfo.cc0000640003136300116100000000143306702424205014370 0ustar cevanseng// fgfileinfo.cc #include "fgfileinfo.h" FGFileInfo::FGFileInfo(const FGString& filename) : mFileName(filename), mSize(-1), mIsDir(false), mIsRegularFile(true) { } FGFileInfo::FGFileInfo(const FGString& filename, int size, bool isDir, bool isFile) : mFileName(filename), mSize(size), mIsDir(isDir), mIsRegularFile(isFile) { } FGFileInfo::FGFileInfo() : mSize(-1), mIsDir(false), mIsRegularFile(true) { } bool FGFileInfo::operator==(const FGFileInfo& other) const { return (mFileName == other.mFileName && mIsRegularFile == other.mIsRegularFile); } const FGString& FGFileInfo::GetFileName(void) const { return mFileName; } bool FGFileInfo::IsRegularFile(void) const { return mIsRegularFile; } int FGFileInfo::GetSize(void) const { return mSize; } ftpgrab-0.1.5/fgpickallrecurse.cc0000640003136300116100000000000006674313741015604 0ustar cevansengftpgrab-0.1.5/fgactioni.cc0000640003136300116100000000056606717072576014251 0ustar cevanseng// fgactioni.cc #include "fgactioni.h" #ifndef _FGEXC_H #include "fgexc.h" #endif FGActionInterface::~FGActionInterface() { } void FGActionInterface::Do(void) const { try { VirtualDo(); } // Any other type of exception is _real_ serious catch (FGException& e) { if (e.GetReason() != FGException::kConcurrentGrabs) { Abort(); } throw; } } ftpgrab-0.1.5/fgconi.h0000640003136300116100000000236706707431035013402 0ustar cevanseng#ifndef _FGCONI_H #define _FGCONI_H // fgconi.h // // FTPGrabConnectionInterface // // Interface class detailing requirements of each FTPGrab protocol // we might be fetching from e.g. FTP, HTTP etc. class FGString; class FGDirListing; class FGConnectionInterface { public: // Need virtual destructor - we delete by the base class virtual ~FGConnectionInterface(); // Connect method // INPUTS: Hostname of host to connect to // RETURNS: Nothing // THROWS: Various connect failed exceptions virtual void Connect(const FGString& host) = 0; // Change directory method // INPUTS: Directory name // RETURNS: Nothing // THROWS: Directory not found exception virtual void ChangeDir(const FGString& dir) = 0; // Get directory listing method // INPUTS: None // RETURNS: Vector of directory entries virtual FGDirListing GetDirListing(void) = 0; // Retrieve file method // INPUTS: Filename // RETURNS: fd to source of data // THROWS: File not found, etc. virtual int GetFile(const FGString& file) = 0; // Post retrive file method // Some protocols need to perform cleanup/verification after // a transfer virtual void PostFileTransfer(void) = 0; // Disconnect virtual void Disconnect(void) = 0; }; #endif // _FGCONI_H ftpgrab-0.1.5/fgdlist.h0000640003136300116100000000131011267245627013563 0ustar cevanseng#ifndef _FGDLIST_H #define _FGDLIST_H // fgdlist.h #ifndef _FGFILEINFO_H #include "fgfileinfo.h" #endif #ifndef __SGI_STL_VECTOR_H #include #endif #ifndef _FGSTRING_H #define "fgstring.h" #endif class FGDirListing : public std::vector { public: // Constructor must specify the name of the dir FGDirListing(const FGString& dirName); // Some static helper methods for comparing file lists etc. static FGDirListing GetRHSOnlyFiles(const FGDirListing& lhs, const FGDirListing& rhs); const FGString& GetDirName(void) const; private: // Directory name (full path) this listing is actually for FGString mDirName; }; #endif // _FGDLIST_H ftpgrab-0.1.5/fgbdfname.h0000640003136300116100000000143506676032612014044 0ustar cevanseng#ifndef _FGBDFNAME_H #define _FGBDFNAME_H // fgbdfname.h // // FGBrokenDownFileName // // Class representing a filename broken down into components which // typically consist of strings and version numbers class FGString; class FGMatchRanking; class FGBrokenDownFileName { public: // Constructor - takes in filename to parse into components FGBrokenDownFileName(const FGString& fname); FGMatchRanking GetRanking(const FGString& cmp) const; // Destructor ~FGBrokenDownFileName(); private: // Banned!! FGBrokenDownFileName(const FGBrokenDownFileName& other); FGBrokenDownFileName& operator=(const FGBrokenDownFileName& other); // Cheshire Cat to hide internals struct Internal_FGBrokenDownFileName; Internal_FGBrokenDownFileName* mpInternals; }; #endif // _FGBDFNAME_H ftpgrab-0.1.5/Changelog0000640003136300116100000000747011275171255013577 0ustar cevanseng0.0.3a - first public release - New "DESIGN" file - Bugfix: check return code from write() - Bugfix: match filename properly even if it doesn't contain any rankables - New "COPYING" file - FEATURE: regular expression support, hurrah! - Bugfix: Cope with ProFTPd responses better (highlighted by www.freiburg.linux.de) - Bugfix: Send FULL hostname as e-mail address (fixes ftp.apache.org) Released 0.0.4a - Fix compile warnings in the FGLogger - Add "-n" flag to disable log output to file - Add "-l" flag to specify different log file name to default - Add "-r" flag to specify different rcfile name to default - FEATURE: Multi-threaded! Default 5 threads download simultaneously - Add "-h" flag to display usage - Add "-t" flag to control max. number of active threads - Bugfix: fix horrible race in threading logic - Bugfix: use re-entrant version of gethostbyname() - Bugfix: nail horrendous file descriptor leak - Updated README - Updated TODO - New README.options file Packaged 0.0.5a but not released - Bugfix: Another (should be the last) thread deadlock nailed - ISSUE: Multithreading stuff oopses (nastily) 2.0.36pre kernels?? - Add "-v" option to output verbose progress to stdout - Add Debian diffs. Thanks to Christian T. Steigies - Nail last compile warning Released 0.0.6a - Fix missing "break" in parsing -t option - Add note on gcc-2.7 - Fix stupid match value bug for multiple component matches - Rewrite fgpickbest.cc with algorithm that actually works :-( - Add "probably won't do" section to TODO :-) - Don't accept downloaded file unless its the same length we expect - Throw exception on read error from network Tentatively packaged 0.0.7a (to become 0.1.0 if stable) - Make debian/rules executable for Christian - Fix crash bug if current download detected - Fix a few similar bugs to the above found "by inspection" - Fix so concurrent download correctly reported - Fix shameful off-by-one error in new file pick "best" logic - Fix delete of file if we abort due to concurrent download - Don't show spurious "rule failed" if we can't delete an old file version (we may have been raced by another thread) - NOTE: 2.0.37pre12 is out, _allegedly_ it fixes ftpgrab oopsing 2.0.x. Thanks to Alan Cox as usual :) - NOTE: Damn, 0.0.7a was a little buggy - Add Christian's latest Debian diff, including a manpage update. Thanks Christian! Tentatively packaged 0.1.0 - Fix so gcc 2.95 can build it (operator += needed to be declared as returing void) - Spew out version number in "-h" option - FEATURE: New character filename component, Useful for stuff like samba-2.0.5a.tar.gz. Note the "a" - Fix verbose mode so rule failure reasons get dumped to stdout - Fix above character feature :-) - Add ability for character filename component to be optional - Fix: ignore whitespace at end of rule line too Package/release 0.1.1 - "make clean" now removes fglog.out so I don't ship it by mistake ;-) - Build with -Wall, nail all warnings - Oops! Missing destructor in FGPickBest (memory leak) - Build with "-O2" optimisation by default - Strip executable by default - Report "connection lost" if remote site boots us out straight away (was "response too large") - Check for read errors on command stream! Should avoid 100% CPU spin bug - Use SO_KEEPALIVE. Should stop permanently hung ftpgrab threads - Don't send password with hostnames without "." in (e.g. ftpgrab@localhost) - Apply Debian Alpha patch - Update Debian diffs Package/release 0.1.2 - Make it build with modern tool chains. This is 10+ years old code. I dare not look at it in detail for fear of discovering how bad a coder I used to be. - Fix send of FTP comments to include CR as well as LF. Package/release 0.1.3 - Make it build with Ubuntu 9.04 too. Package/release 0.1.4 - Another build error: get rid of useless referece to sa_restorer. Package/release 0.1.5 ftpgrab-0.1.5/fgstring.cc0000640003136300116100000000575711271650302014114 0ustar cevanseng// fgstring.cc #include "fgstring.h" #include #include #include FGString::FGString() : mpString(0), mLength(0) { } FGString::FGString(const FGString& other) { mLength = other.GetLength(); if (mLength) { mpString = new char[mLength+1]; strcpy(mpString, other); } else { mpString = 0; } } FGString& FGString::operator=(const FGString& other) { if (this == &other) { return *this; } if (mpString) { delete [] mpString; } mLength = other.GetLength(); if (mLength) { mpString = new char[mLength+1]; strcpy(mpString, other); } else { mpString = 0; } return *this; } FGString::~FGString() { if (mpString) { delete [] mpString; } } FGString::FGString(const char* pStr) { mLength = strlen(pStr); mpString = new char[mLength+1]; strcpy(mpString, pStr); } FGString::operator const char*() const { return mpString; } unsigned int FGString::GetLength(void) const { return mLength; } void FGString::operator+=(const FGString& other) { mLength = mLength + other.GetLength(); char* pTmp = new char[mLength+1]; if (mpString) { strcpy(pTmp, mpString); strcat(pTmp, other); delete [] mpString; } else { strcpy(pTmp, other); } mpString = pTmp; } void FGString::operator+=(char other) { mLength++; char* pTmp = new char[mLength+1]; if (mpString) { strcpy(pTmp, mpString); pTmp[mLength-1] = other; pTmp[mLength] = '\0'; delete [] mpString; } else { pTmp[0] = other; pTmp[1] = '\0'; } mpString = pTmp; } bool FGString::operator==(const FGString& other) const { if (!mpString || !other.mpString) { return false; } return (strcmp(mpString, other.mpString) == 0); } bool FGString::operator!=(const FGString& other) const { return !(*this == other); } bool FGString::IsEmpty(void) const { if (!mpString || !mLength) { return true; } return false; } void FGString::MakeEmpty(void) { if (mpString) { delete [] mpString; mpString = 0; mLength = 0; } } char FGString::operator[](int pos) const { assert(pos >= 0 && (unsigned int) pos <= mLength); return mpString[pos]; } FGString FGString::Left(unsigned int num) const { if (num == 0) { FGString ret; return ret; } if (num >= mLength) { return *this; } // Oh, we have to do some work FGString ret; ret.mLength = num; ret.mpString = new char[num+1]; strncpy(ret.mpString, mpString, num); ret.mpString[num] = '\0'; return ret; } FGString FGString::Right(unsigned int num) const { if (num == 0) { FGString ret; return ret; } if (num >= mLength) { return *this; } // Oh, we have to do some work FGString ret; ret.mLength = num; ret.mpString = new char[num+1]; strncpy(ret.mpString, mpString + mLength - num, num); ret.mpString[num] = '\0'; return ret; } int FGString::AToI(void) const { return atoi(mpString); } bool FGString::Contains(char c) const { if (strchr(mpString, c)) { return true; } return false; } ftpgrab-0.1.5/fgbdfname.cc0000640003136300116100000001042211267246547014204 0ustar cevanseng// fgbdfname.cc #include "fgbdfname.h" #ifndef _FGSTRING_H #include "fgstring.h" #endif #ifndef _FGMRANK_H #include "fgmrank.h" #endif #ifndef _FGSCOMP_H #include "fgscomp.h" #endif #ifndef _FGICOMP_H #include "fgicomp.h" #endif #ifndef _FGDYYMMDDCOMP_H #include "fgdyymmddcomp.h" #endif #ifndef _FGCHARCOMP_H #include "fgcharcomp.h" #endif #ifndef _FGEXC_H #include "fgexc.h" #endif #ifndef __SGI_STL_VECTOR_H #include #endif struct FGBrokenDownFileName::Internal_FGBrokenDownFileName { std::vector mComponents; }; FGBrokenDownFileName::FGBrokenDownFileName(const FGString& fname) { mpInternals = new Internal_FGBrokenDownFileName; // Need to catch exceptions to clean up above resources try { // We must now parse the supplied filename into components unsigned int index = 0; // Parse state bool esc = false; bool inTok = false; FGString bit; while (index <= fname.GetLength()) { char c = fname[index++]; if (c == '\0') { // End of string if (inTok) { // Error! throw FGException(FGException::kParseEndOfString); } if (bit.GetLength() > 0) { FGFileNameComponent* pNew; pNew = new FGStringComponent(bit); mpInternals->mComponents.push_back(pNew); } } else if (esc || (c != '\\' && c != '<' && c != '>')) { bit += c; esc = false; } else if (c == '\\') { esc = true; } else if (c == '<') { if (inTok) { throw FGException(FGException::kParseUnexpectedChar, "<"); } inTok = true; if (bit.GetLength() > 0) { FGFileNameComponent* pNew; pNew = new FGStringComponent(bit); mpInternals->mComponents.push_back(pNew); } bit.MakeEmpty(); } else { // c == '>' if (!inTok) { throw FGException(FGException::kParseUnexpectedChar, ">"); } FGString strInt("int"); FGString strYYMMDD("YYMMDD"); FGString strChar("char"); FGString strOptChar("optchar"); if (bit == strInt) { FGFileNameComponent* pNew; pNew = new FGIntegerComponent; mpInternals->mComponents.push_back(pNew); } else if (bit == strYYMMDD) { FGFileNameComponent* pNew; pNew = new FGDateYYMMDDComponent; mpInternals->mComponents.push_back(pNew); } else if (bit == strChar) { FGFileNameComponent* pNew; pNew = new FGCharacterComponent(false); mpInternals->mComponents.push_back(pNew); } else if (bit == strOptChar) { FGFileNameComponent* pNew; pNew = new FGCharacterComponent(true); mpInternals->mComponents.push_back(pNew); } else { throw FGException(FGException::kUnrecognizedComponent, bit); } inTok = false; bit.MakeEmpty(); } } // end while (more chars to parse) } // end try block catch (FGException&) { delete this; throw; } } FGBrokenDownFileName::~FGBrokenDownFileName() { if (mpInternals) { // Free the vector of pointers std::vector::iterator iComps; for (iComps = mpInternals->mComponents.begin(); iComps != mpInternals->mComponents.end(); iComps++) { delete *iComps; } delete mpInternals; } } FGMatchRanking FGBrokenDownFileName::GetRanking(const FGString& cmp) const { FGMatchRanking ret; FGString mangleCompare(cmp); // Iterate through components std::vector::const_iterator iComps; for (iComps = mpInternals->mComponents.begin(); iComps != mpInternals->mComponents.end(); iComps++) { int result = -1; bool matched = (*iComps)->MatchAndRankComponent(mangleCompare, &result); if (matched == false) { // Oh dear FGMatchRanking duffRet; return duffRet; } if (result != -1) { ret.AddValue(result); } } // Not a match if any unmatched string remains if (mangleCompare.GetLength() > 0) { FGMatchRanking duffRet; return duffRet; } // It's a match.. this return value indicates how good a match // i.e. linux-2.2.1.tar.gz is better that linux-2.2.0.tar.gz ret.SetMatch(); return ret; } ftpgrab-0.1.5/fgfilegrab.cc0000640003136300116100000001141011271650342014344 0ustar cevanseng// fgfilegrab.cc #include "fgfilegrab.h" // Possible protocol interface derived classes #ifndef _FGFTPCON_H #include "fgftpcon.h" #endif // Possible picker interface derived classes #ifndef _FGPICKALL_H #include "fgpickall.h" #endif #ifndef _FGPICKALLSYNC_H #include "fgpickallsync.h" #endif #ifndef _FGPICKBEST_H #include "fgpickbest.h" #endif #ifndef _FGPICKREGEXP_H #include "fgpickregexp.h" #endif #ifndef _FGPICKREGEXPSYNC_H #include "fgpickregexpsync.h" #endif // Directory listings #ifndef _FGDLIST_H #include "fgdlist.h" #endif // Local filesystem helper #ifndef _FGFSHELP_H #include "fgfshelp.h" #endif // Action lists #ifndef _FGALIST_H #include "fgalist.h" #endif #ifndef _FGLOGGER_H #include "fglogger.h" #endif #ifndef _FGEXC_H #include "fgexc.h" #endif #include #include FGFileGrab::FGFileGrab(const FGString& ruleName, const FGString& host, const FGString& remDir, const FGString& localDir, const FGString& file) : mpConInterface(0), mpPickInterface(0) { mRemoteHost = host; mRemoteDir = remDir; mLocalDir = localDir; mRuleName = ruleName; // XXX :-) mpConInterface = new FGFTPCon; FGString cmp("*"); FGString cmp2("*!"); int nameLen = file.GetLength(); if (file == cmp) { mpPickInterface = new FGPickAll(mpConInterface); } else if (file == cmp2) { mpPickInterface = new FGPickAllSync(mpConInterface); } else if (nameLen > 1 && file[0] == '/' && file[1] == '!') { // /! sequence means match a regular expression, and delete local // files matching if not found remotely, too. FGString regExp(file.Right(nameLen - 2)); // Alert alert leak mpPickInterface = new FGPickRegexpSync(mpConInterface, regExp); } else if (nameLen > 0 && file[0] == '/') { // / character means regular expression follows FGString regExp(file.Right(nameLen - 1)); // Alert throws exception - leaks mpConInterface mpPickInterface = new FGPickRegexp(mpConInterface, regExp); } else { // Extract number of revisions if neccessary int nrevs = 1; FGString newFile(file); if (file.GetLength() > 1 && file[0] == '[') { unsigned int index = 1; FGString strNum; while (index < file.GetLength() && file[index] != ']') { strNum += file[index]; index++; } // If we ended by hitting "]" then extract revisions count // and lop [] enclosed bit off string if (index < file.GetLength()) { int newRevs = atoi(strNum); if (newRevs > 1) { nrevs = newRevs; } int charsLeft = file.GetLength() - (index + 1); newFile = file.Right(charsLeft); } } // Alert alert this might throw an exception // If this class (FGFileGrab) allocates any resources in the // constructor they will need to be taken care of mpPickInterface = new FGPickBest(mpConInterface, newFile, nrevs); } } const FGString& FGFileGrab::GetRuleName(void) const { return mRuleName; } void FGFileGrab::FreeResources(void) { if (mpConInterface) { delete mpConInterface; } if (mpPickInterface) { delete mpPickInterface; } } void FGFileGrab::GrabIt(void) const { // Pre-conditions assert(mpConInterface != 0); assert(mpPickInterface != 0); FGLogger& theLog = FGLogger::GetLogger(); // Connect to remote FGString logMsg; logMsg = "Trying to connect to host: "; logMsg += mRemoteHost; theLog.LogMsg(logMsg, FGLogger::kFGLLVerbose); mpConInterface->Connect(mRemoteHost); theLog.LogMsg("Connect successful", FGLogger::kFGLLVerbose); // Once we are connected we must guarantee to call Disconnect() at // some time try { // Change to required directory logMsg = "Changing to remote directory: "; logMsg += mRemoteDir; theLog.LogMsg(logMsg, FGLogger::kFGLLVerbose); mpConInterface->ChangeDir(mRemoteDir); // Get list of files in said dir theLog.LogMsg("Getting local and remote file lists", FGLogger::kFGLLVerbose); FGDirListing remoteList = mpConInterface->GetDirListing(); // Get list of files in local dir of interest FGDirListing localList = FGFileSystemHelper::GetDirListing(mLocalDir); // Ask the picker interface what we propose to do given the two file lists FGActionList actions = mpPickInterface->DecideActions(localList, remoteList); // Let it rip... try { actions.DoActions(); } catch (FGException&) { actions.FreeResources(); throw; } // Free some resources actions.FreeResources(); } catch (FGException&) { theLog.LogMsg("Disconnecting (error condition)", FGLogger::kFGLLVerbose); mpConInterface->Disconnect(); throw; } // Clean up theLog.LogMsg("Disconnecting", FGLogger::kFGLLVerbose); mpConInterface->Disconnect(); } ftpgrab-0.1.5/fgcharcomp.h0000640003136300116100000000135207022702611014227 0ustar cevanseng#ifndef _FGCHARCOMP_H #define _FGCHARCOMP_H // fgcharcomp.h // // FGCharacterComponent: handles ranked matching of a single character, // with disregard to case. Useful when stuff is released with a name like // e.g. samba-2.0.5a.tar.gz #include "fgfncomp.h" class FGCharacterComponent : public FGFileNameComponent { public: // Constructor FGCharacterComponent(bool isOptional); virtual bool MatchAndRankComponent(FGString& fnameRemainder, int* pMatchVal) const; virtual ~FGCharacterComponent(); private: // Banned! FGCharacterComponent(const FGCharacterComponent& other); FGCharacterComponent& operator=(const FGCharacterComponent& other); bool mIsOptional; }; #endif // _FGCHARCOMP_H ftpgrab-0.1.5/fgdelaction.h0000640003136300116100000000133606702416317014410 0ustar cevanseng#ifndef _FGDELACTION_H #define _FGDELACTION_H // fgdelaction.h // // Implementation class of the download action #include "fgactioni.h" #ifndef _FGSTRING_H #include "fgstring.h" #endif class FGConnectionInterface; class FGDeleteAction : public FGActionInterface { public: // Construct with filename to download FGDeleteAction(const FGString& fname, const FGString& localDir); ~FGDeleteAction(); // Overridden virtual Do method virtual void VirtualDo(void) const; // And Abort() virtual void Abort(void) const; private: // Banned! FGDeleteAction(const FGDeleteAction& other); FGDeleteAction& operator=(const FGDeleteAction& other); FGString mFileName; FGString mLocalDir; }; #endif // _FGDELACTION_H ftpgrab-0.1.5/fgexc.h0000640003136300116100000000373107101750344013221 0ustar cevanseng#ifndef _FGEXC_H #define _FGEXC_H // fgexc.h // // Header file for exception class #ifndef _FGSTRING_H #include "fgstring.h" #endif class FGException { public: // Enumerated exception types enum EFGExceptions { // Connection failures kHostResolveFail = 0, kHostResolveRetryableFail, kConnectResourceAllocFail, kConnectRefused, kConnectHostUncontactable, kConnectMiscFail, kConnectLost, // FTP protocol parse failures kResponseTooLong, kResponseMalformed, kResponseUnexpected, // FTP server doesn't like us failures kAccessDenied, kServerFull, // When we can't find stuff we expect kNoSuchDir, kNoSuchFile, // Local file handling issues kLocalNoSuchDir, kLocalNoSuchFile, kLocalPermissionDenied, // Timeout d/loading file kDownloadTimeout, // Config file parse failures kConfigFileNotFound, kConfigMissingColon, kDuplicateToken, kUnexpectedEndOfConfig, // Misc. filesystem stuff kFileCreateFailed, kConcurrentGrabs, kUnlinkFailed, kWriteFailed, // Log file failures kLogFileOpenFailed, // Failures parsing filenames/components kParseEndOfString, kParseUnexpectedChar, kUnrecognizedComponent, // Duff user supplied regular expression kInvalidRegexp, // Problems reading file from network kNetworkReadError, kFileTooShort, }; // Constructor - pass in reason for throwing exception // plus additional details message FGException(EFGExceptions reason, const FGString& details); // Constructor where no additional details available FGException(EFGExceptions reason); EFGExceptions GetReason(void) const; // Methods to analyze severity of exception/what went wrong etc. bool IsConditionRetryable(void) const; // Severity is 0-10 int GetSeverity(void) const; FGString GetFullDetails(void) const; private: // Internal storage details EFGExceptions mWhat; FGString mDetails; }; #endif // _FGEXC_H ftpgrab-0.1.5/fgpickallsync.cc0000640003136300116100000000342711267246413015123 0ustar cevanseng// fgpickallsync.cc #include "fgpickallsync.h" // Dir listings #ifndef _FGDLIST_H #include "fgdlist.h" #endif // Action lists #ifndef _FGALIST_H #include "fgalist.h" #endif // The download file action #ifndef _FGDLACTION_H #include "fgdlaction.h" #endif // The local erase file action #ifndef _FGDELACTION_H #include "fgdelaction.h" #endif FGPickAllSync::FGPickAllSync(FGConnectionInterface* pConnIf) : FGFilePickerInterface(pConnIf) { } FGActionList FGPickAllSync::DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir) { // The picker for the "normal" wildcard operation (non recursive), // leaves all local files intact // Very simple rule: grab any file in the remoteDir that is // not in the localDir FGActionList ret; // Get list of files in local but not remote FGDirListing staleFiles = FGDirListing::GetRHSOnlyFiles(remoteDir, localDir); // For each stale file, make an "erase" action std::vector::const_iterator iFiles; for (iFiles = staleFiles.begin(); iFiles != staleFiles.end(); iFiles++) { FGActionInterface* pNewAction; pNewAction = new FGDeleteAction(iFiles->GetFileName(), localDir.GetDirName()); ret.push_back(pNewAction); } // Get list of files in remote but not local FGDirListing newFiles = FGDirListing::GetRHSOnlyFiles(localDir, remoteDir); // Make a "download" action for all the above files for (iFiles = newFiles.begin(); iFiles != newFiles.end(); iFiles++) { FGActionInterface* pNewAction; pNewAction = new FGDownloadAction(iFiles->GetFileName(), mpConnIf, localDir.GetDirName(), iFiles->GetSize()); ret.push_back(pNewAction); } return ret; } ftpgrab-0.1.5/DESIGN0000640003136300116100000000123306704750744012657 0ustar cevansengThe C++ design is supposed to give a loosely coupled, highly extensible code base. I'm interested in C++ design patterns, so any comments (even flames) about the standard of my code/design are welcome. n.b. I know about the following issues. - throwing of exceptions in constructors, tsk tsk. - "suicidial" constructors (delete this) - Poor, non constructor/destructor automated memory management in places For the curious, look at the base classes FGConnectionInterface, FGPickerInterface and FGActionInterface. These interface definitions are at the core of the programs extensibility. One day I might copy my UML diagram into an art package and include it :-) ftpgrab-0.1.5/fgfilelist.h0000640003136300116100000000164611267246351014266 0ustar cevanseng#ifndef _FGFILELIST_H #define _FGFILELIST_H // fgfilelist.h // Internal call which is the starting point for // child threads void* ThreadStartPoint(void* pArg); class FGString; class FGFileList { public: // Constructor, desctructor FGFileList(); ~FGFileList(); // Loads file list from config rc file void LoadConfig(const FGString& configFile); // The all import "grab them all" method :-) void GetAllFiles(void) const; private: // Banned! FGFileList(const FGFileList& other); FGFileList& operator=(const FGFileList& other); // Internal helpers FGString GetLineString(void) const; // Using the Cheshire Cat to hide implementation details struct Internal_FGFileList; Internal_FGFileList* mpInternals; int mLine; // Counter of active child rule threads // (shared across all threads) static int msThreadsActive; static void* ThreadStartPoint(void* pArg); }; #endif // _FGFILELIST_H ftpgrab-0.1.5/fgfpicki.h0000640003136300116100000000144706674514722013725 0ustar cevanseng#ifndef _FGFPICKI_H #define _FGFPICKI_H // fgfpicki.h // // FGFilePickerInterface // // Defines interface to deciding what files we want to download from // remote and erase locally. We decide based upon a directory listing // of the remote directory and a directory listing of the local // directory. class FGActionList; class FGDirListing; class FGConnectionInterface; class FGFilePickerInterface { public: // Construct taking connection on which to do downloads FGFilePickerInterface(FGConnectionInterface* pConnIf); virtual FGActionList DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir) = 0; protected: // Some actions must be performed in the context of a connection FGConnectionInterface* mpConnIf; }; #endif // _FGFPICKI_H ftpgrab-0.1.5/fgdlist.cc0000640003136300116100000000240211267245727013725 0ustar cevanseng// fgdlist.cc #include "fgdlist.h" #ifndef _FGSTING_H #include "fgstring.h" #endif FGDirListing::FGDirListing(const FGString& dirName) : mDirName(dirName) { } FGDirListing FGDirListing::GetRHSOnlyFiles(const FGDirListing& lhs, const FGDirListing& rhs) { FGDirListing ret("NODIR"); // Could be dealing with 100's or 1000's of files. Should really use // map (tree based) not hopeless vector iteration. Heard of O(n^2)? :-) std::vector::const_iterator iRHSFiles; for (iRHSFiles = rhs.begin(); iRHSFiles != rhs.end(); iRHSFiles++) { if (iRHSFiles->IsRegularFile()) { ret.push_back(*iRHSFiles); } } std::vector::const_iterator iLHSFiles; for (iLHSFiles = lhs.begin(); iLHSFiles != lhs.end(); iLHSFiles++) { // For each LHS files scan through current list for match // Ugh slow slow need a tree based thing see above comment std::vector::iterator iCurFiles; for (iCurFiles = ret.begin(); iCurFiles != ret.end(); iCurFiles++) { if (*iLHSFiles == *iCurFiles) { // Match so waste it from return list ret.erase(iCurFiles); break; } } } return ret; } const FGString& FGDirListing::GetDirName(void) const { return mDirName; } ftpgrab-0.1.5/fgdyymmddcomp.cc0000640003136300116100000000304606702375272015136 0ustar cevanseng// fgdyymmdd.cc #include "fgdyymmddcomp.h" #ifndef _FGSTRING_H #include "fgstring.h" #endif #include #include #include FGDateYYMMDDComponent::FGDateYYMMDDComponent() { } FGDateYYMMDDComponent::~FGDateYYMMDDComponent() { } bool FGDateYYMMDDComponent::MatchAndRankComponent(FGString& fnameRemainder, int* pMatchVal) const { // Initialize to be safe *pMatchVal = -1; // Need 6 characters! if (fnameRemainder.GetLength() < 6) { return false; } FGString dateBit = fnameRemainder.Left(6); // Must be all digits if (!isdigit(dateBit[0]) || !isdigit(dateBit[1]) || !isdigit(dateBit[2]) || !isdigit(dateBit[3]) || !isdigit(dateBit[4]) || !isdigit(dateBit[5])) { return false; } // OK so its all digits // From here it will always be a match, we don't validate the month/day // values are valid (e.g. 17th Month of the year?) FGString yearStr = dateBit.Left(2); int year = atoi(yearStr); // Year 00 (2000) must have greater value than, e.g., 99 (1999) if (year < 50) { year += 50; } else { year -= 50; } // Build the final value string char yearBuf[3]; sprintf(yearBuf, "%d", year); FGString theVal(yearBuf); theVal += dateBit[2]; theVal += dateBit[3]; theVal += dateBit[4]; theVal += dateBit[5]; *pMatchVal = atoi(theVal); // Consume the 6 digits we matched int charsLeft = fnameRemainder.GetLength() - 6; FGString bitLeft = fnameRemainder.Right(charsLeft); fnameRemainder = bitLeft; return true; } ftpgrab-0.1.5/fgdlaction.cc0000640003136300116100000000373107101740022014365 0ustar cevanseng// fgdlaction.cc #include "fgdlaction.h" // Downloading involves interfacing with remote connection #ifndef _FGCONI_H #include "fgconi.h" #endif #ifndef _FGFSHELP_H #include "fgfshelp.h" #endif #ifndef _FGSTRING_H #include "fgstring.h" #endif #ifndef _FGEXC_H #include "fgexc.h" #endif #ifndef _FGLOGGER_H #include "fglogger.h" #endif FGDownloadAction::FGDownloadAction(const FGString& fname, FGConnectionInterface* pConnIf, const FGString& localDir, int size) : mpConnIf(pConnIf), mFileName(fname), mLocalDir(localDir), mDestFD(-1), mFileSize(size) { } FGDownloadAction::~FGDownloadAction() { } void FGDownloadAction::VirtualDo(void) const { FGString msg; msg = "Starting download of file `"; msg += mFileName; msg += '\''; FGLogger::GetLogger().LogMsg(msg, FGLogger::kFGLLVerbose); // Interface already has directory state ready for direct grab // Also, we're connected at this point int dataFD = mpConnIf->GetFile(mFileName); // Create the destination file for writing FGString destName(mLocalDir); destName += '/'; destName += mFileName; try { int destFD = FGFileSystemHelper::CreateLocalFile(destName); FGFileSystemHelper::TransferRemoteToLocal(dataFD, destFD, mFileSize); } catch (FGException& e) { mpConnIf->PostFileTransfer(); throw e; } // Success - cool better log message msg = "Successful transfer of file `"; msg += mFileName; msg += '\''; FGLogger::GetLogger().LogMsg(msg, FGLogger::kFGLLInfo); mpConnIf->PostFileTransfer(); } void FGDownloadAction::Abort(void) const { // Something has gone wrong during the download // Delete the file we started to download // Ignore errors in case we didn't get as far as creating the file FGString delFile(mLocalDir); delFile += '/'; delFile += mFileName; try { FGFileSystemHelper::DeleteFile(delFile); } catch (FGException&) { // Do nothing } } ftpgrab-0.1.5/fgfncomp.cc0000640003136300116100000000012706702375460014066 0ustar cevanseng// fgfncomp.cc #include "fgfncomp.h" FGFileNameComponent::~FGFileNameComponent() { } ftpgrab-0.1.5/fgpickbest.h0000640003136300116100000000201407101741705014240 0ustar cevanseng#ifndef _FGPICKBEST_H #define _FGPICKBEST_H // fgpickbest.h // // Implementation of a file picker that selects the most "recent" // version(s) #ifndef _FGFPICKI_H #include "fgfpicki.h" #endif class FGString; class FGConnectionInterface; class FGPickBest : public FGFilePickerInterface { public: // Must be constructed with connection interface // and a count of how many revisions of a file to // retain locally, e.g. 2 => best match and previous // best match // Also should be constructed with match string FGPickBest(FGConnectionInterface* pConnIf, const FGString& match, int count); virtual ~FGPickBest(); virtual FGActionList DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir); private: // Banned! FGPickBest(const FGPickBest& other); FGPickBest& operator=(const FGPickBest& other); // Implementation details hidden by Cheshire cat struct Internal_FGPickBest; Internal_FGPickBest* mpInternals; }; #endif // _FGPICKBEST_H ftpgrab-0.1.5/fgmrank.cc0000640003136300116100000000426711267246644013730 0ustar cevanseng// fgmrank.cc #include "fgmrank.h" #ifndef __SGI_STL_VECTOR_H #include #endif struct FGMatchRanking::Internal_FGMatchRanking { bool mIsMatch; std::vector mValues; }; FGMatchRanking::FGMatchRanking() { mpInternals = new Internal_FGMatchRanking; mpInternals->mIsMatch = false; } FGMatchRanking::~FGMatchRanking() { delete mpInternals; } FGMatchRanking::FGMatchRanking(const FGMatchRanking& other) { mpInternals = new Internal_FGMatchRanking; *mpInternals = *other.mpInternals; } FGMatchRanking& FGMatchRanking::operator=(const FGMatchRanking& other) { if (this == &other) { return *this; } // Free existing stuff delete mpInternals; mpInternals = new Internal_FGMatchRanking; *mpInternals = *other.mpInternals; return *this; } void FGMatchRanking::AddValue(int value) { mpInternals->mValues.push_back(value); } void FGMatchRanking::SetMatch(void) { mpInternals->mIsMatch = true; } bool FGMatchRanking::IsMatch(void) const { return mpInternals->mIsMatch; } bool FGMatchRanking::operator<(const FGMatchRanking& other) const { // Always less than if we're not a match // n.b. nothing should be comparing against non-match anyway if (mpInternals->mIsMatch == false) { return true; } // Never less than, if comparing against non-match if (other.mpInternals->mIsMatch == false) { return false; } // Otherwise iterate through components one by one int thisSize = mpInternals->mValues.size(); int otherSize = other.mpInternals->mValues.size(); int min = thisSize; if (otherSize < min) { min = otherSize; } int index; for (index = 0; index < min; index++) { if (mpInternals->mValues[index] < other.mpInternals->mValues[index]) { // Yes we are an older version return true; } else if (mpInternals->mValues[index] > other.mpInternals->mValues[index]) { // Newer version.. so not older version return false; } // else.. equal so progress to next component and compare } // Oh we ran out of comparisions // If we have more bits of version we are greater (not less than) if (thisSize > otherSize) { return false; } // Oh we are equal // That's not less than return false; } ftpgrab-0.1.5/fgfncomp.h0000640003136300116100000000127706702375427013742 0ustar cevanseng#ifndef _FGFNCOMP_H #define _FGFNCOMP_H // fgfncomp.h // // FGFileNameComponent: base class for "bits" of a filename which can // be compared, matched, ranked etc. to determine how "recent" a file is class FGString; class FGFileNameComponent { public: // In: string with remainder of filename to be matched/ranked // Out: int representing rank value (-1 for valueless match) // Out: String is modified; matched bit is chopped off // Out: bool, did we match or not? virtual bool MatchAndRankComponent(FGString& fnameRemainder, int* pMatchVal) const = 0; // Need virtual destructor for the mm virtual ~FGFileNameComponent(); }; #endif // _FGFNCOMP_H ftpgrab-0.1.5/README.URGENT.KERNEL2.00000640003136300116100000000162006717340756015047 0ustar cevanseng[NOTE: Allegedly, kernel 2.0.37pre12 includes the fix] Hi! Thanks for reading this. You'll be pleased you did, if you have a version 2.0 Linux kernel! Up to and including kernel 2.0.37pre10 seem to have a bug with multithreaded programs under certain conditions. This is important because as of ftpgrab-0.0.5a, ftpgrab uses the pthreads threading library to achieve concurrent downloads. The kernel bug manifests itself with an "oops" and often messages saying "VFS: Close: file count is 0". Sometimes processes other than ftpgrab itself may be caused to "oops". This is unfortunate. Kernel series 2.2 does not appear affected by this nasty bug. As a workaround for 2.0, either use ftpgrab-0.0.4a which is pretty functional and does not use threads, or perhaps try limiting ftpgrab to a single child thread with "-t 1" as a command line parameter. I'd be surprised if the latter measure helper much though! ftpgrab-0.1.5/fgpickallsync.h0000640003136300116100000000131006702232330014741 0ustar cevanseng#ifndef _FGPICKALLSYNC_H #define _FGPICKALLSYNC_H // fgpickallsync.h // // Implementation of a file picker that selects all remote files for // which we do not have local copies #ifndef _FGFPICKI_H #include "fgfpicki.h" #endif class FGConnectionInterface; class FGPickAllSync : public FGFilePickerInterface { public: // Must be constructed with connection interface FGPickAllSync(FGConnectionInterface* pConnIf); virtual FGActionList DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir); private: // Banned! FGPickAllSync(const FGPickAllSync& other); FGPickAllSync& operator=(const FGPickAllSync& other); }; #endif // _FGPICKALLSYNC_H ftpgrab-0.1.5/fgalist.cc0000640003136300116100000000074711267245661013731 0ustar cevanseng// fgalist.cc #include "fgalist.h" // Action interface #ifndef _FGACTIONI_H #include "fgactioni.h" #endif void FGActionList::DoActions(void) const { std::vector::const_iterator iActions; for (iActions = begin(); iActions != end(); iActions++) { (*iActions)->Do(); } } void FGActionList::FreeResources(void) { std::vector::iterator iActions; for (iActions = begin(); iActions != end(); iActions++) { delete *iActions; } } ftpgrab-0.1.5/fgpickregexp.h0000640003136300116100000000177507101741133014605 0ustar cevanseng#ifndef _FGPICKREGEXP_H #define _FGPICKREGEXP_H // fgpickregexp.h // // Implementation of a file picker that selects all remote files matching // a given regular expression for which we do not have remote copies #ifndef _FGFPICKI_H #include "fgfpicki.h" #endif class FGConnectionInterface; class FGString; // Ah the joys of c++ forward declarations struct re_pattern_buffer; typedef struct re_pattern_buffer regex_t; class FGPickRegexp : public FGFilePickerInterface { public: // Must be constructed with connection interface and regexp match string FGPickRegexp(FGConnectionInterface* pConnIf, const FGString& regexp); virtual ~FGPickRegexp(); virtual FGActionList DecideActions(const FGDirListing& localDir, const FGDirListing& remoteDir); private: // Banned! FGPickRegexp(const FGPickRegexp& other); FGPickRegexp& operator=(const FGPickRegexp& other); // Internal representation of a regexp regex_t* mpRegExp; }; #endif // _FGPICKREGEXP_H