BatchJobs/0000755000176000001440000000000012502327541012142 5ustar ripleyusersBatchJobs/inst/0000755000176000001440000000000012502317043013113 5ustar ripleyusersBatchJobs/inst/CITATION0000644000176000001440000000177312502317043014260 0ustar ripleyusers## -*- mode: r -*- citHeader("To cite BatchJobs or BatchExperiments in publications use:") citEntry(entry = "Article", title = "{BatchJobs} and {BatchExperiments}: Abstraction Mechanisms for Using {R} in Batch Environments", author = personList(as.person("Bernd Bischl"), as.person("Michel Lang"), as.person("Olaf Mersmann"), as.person("J{\\\"o}rg Rahnenf{\\\"u}hrer"), as.person("Claus Weihs")), journal = "Journal of Statistical Software", year = "2015", volume = "64", number = "11", pages = "1--25", url = "http://www.jstatsoft.org/v64/i11/", textVersion = paste("Bernd Bischl, Michel Lang, Olaf Mersmann, Joerg Rahnenfuehrer, Claus Weihs (2015).", "BatchJobs and BatchExperiments: Abstraction Mechanisms for Using R in Batch Environments.", "Journal of Statistical Software, 64(11), 1-25.", "URL http://www.jstatsoft.org/v64/i11/.") ) BatchJobs/inst/bin/0000755000176000001440000000000012415510734013670 5ustar ripleyusersBatchJobs/inst/bin/linux-helper0000755000176000001440000000746712415510734016250 0ustar ripleyusers#!/bin/bash ## linux-helper: Helper for the multicore and SSH cluster functions of the BatchJobs R ## package. ## ## Requires the following Unix command line utilities: ## ## * grep, wc, ps, kill, uptime, echo, cat, possibly setsid ## ## The following commands are implemented. First argument is always the command name. ## For other arguments see below. Each command returns a character vector. ## ## number-of-cpus ## Return the number of PEs on worker. ## ## start-job RHOME NICE ROPTIONS RFILE OUTFILE ## Start an R CMD BATCH process running $RFILE and log ## the output in $OUTFILE. ## RHOME is the path to the R installation. ## Returns: PID of sh process which spawned R. We use that as batch.job.id. ## ## kill-job PID ## Kill the R job with PPID $PID. The PPID is the PID of ## the sh process returned by start-job. ## First a TERM is sent, then 1 sec delay, then KILL. ## ## status FILEDIR ## Return 4 numbers: ## - load average of last 1 min, as given by e.g. uptime ## - number of R processes by _all_ users ## - number of R processes by _all_ users which have a load of >= 50% ## - number of R processes by current user ## which match $FILEDIR/jobs in the cmd call of R ## ## list-jobs FILEDIR ## Return the PPIDs of running R jobs operating on $FILEDIR/jobs. ### efficient [ -n `which [prog]` ] command_exists () { hash "$1" 2> /dev/null; } CMD="$1"; shift export LC_ALL=C ### Avoid any localization issues. shopt -s nocasematch ### Case insensitive regular expressions case $CMD in number-of-cpus) if [[ `uname` =~ "Linux" ]]; then cat /proc/cpuinfo | grep '^processor' | wc -l else sysctl -n hw.ncpu ## darwin fi ;; start-job) RCMD="R" if [ -n "$1" ]; then RCMD="${1}/bin/R" fi # only use setsid on linux, not on darwin (not available there) if [[ `uname` =~ "Linux" ]] && command_exists setsid; then RCMD="setsid $RCMD" fi if [ -n "$2" ] && command_exists nice; then RCMD="nice -n ${2} $RCMD" fi shift 2 # nothing should be on all 3 streams except maybe a segfault. throw away. # see comment also in cfLocal # we use setsid to start the process in a different session. other ctrl+C in the master # session leads to killing of child processes in multicore mode as the SIGINT is passed down # $@ ${RCMD} CMD BATCH $@ \ > /dev/null 2> /dev/null < /dev/null & echo $! ;; kill-job) RPPID=$1 # find R process whose ppid is the batch.job.id, i.e. the one of the sh process # we must list all processes (without terminal) for SSH RPID=$(ps -e -o pid= -o ppid= | awk -v j=$RPPID '$2 == j {print $1}') kill -TERM $RPID > /dev/null 2> /dev/null sleep 1 kill -KILL $RPID > /dev/null 2> /dev/null exit 0 ;; status) # remove everyting till load average(s): # then delete commas LOAD=$(uptime | awk '{gsub(/.*:/,""); {gsub(/,/,"")}; print $1}') JOBDIR="$1/jobs" # print 3 columns for all processes # use ww for unlimited width in ps for command output # we count all R procs, all R50, and all where JOBDIR was in the call args ps -e -ww -o pcpu= -o ucomm= -o command= | \ awk -v j=$JOBDIR -v sysload=$LOAD ' BEGIN {rprocs=0;rprocs_50=0;njobs=0} $2 != "R" {next} {rprocs++} $1 > 50.0 {rprocs_50++} $0 ~ j {njobs++} END {print sysload " " rprocs " " rprocs_50 " " njobs}' ;; list-jobs) JOBDIR="$1/jobs" ps -e -ww -o ppid= -o ucomm= -o command= | \ awk -v j=$JOBDIR '$2 == "R" && $0 ~ j { print $1 }' ;; *) esac BatchJobs/inst/etc/0000755000176000001440000000000012415510734013673 5ustar ripleyusersBatchJobs/inst/etc/BatchJobs_global_config.R0000644000176000001440000000025012415510734020517 0ustar ripleyuserscluster.functions = makeClusterFunctionsInteractive() mail.start = "none" mail.done = "none" mail.error = "none" db.driver = "SQLite" db.options = list() debug = FALSE BatchJobs/tests/0000755000176000001440000000000012415510734013305 5ustar ripleyusersBatchJobs/tests/run-all.R0000644000176000001440000000005212415510734014777 0ustar ripleyuserslibrary(testthat) test_check("BatchJobs") BatchJobs/tests/testthat/0000755000176000001440000000000012502316642015144 5ustar ripleyusersBatchJobs/tests/testthat/test_sweepRegistry.R0000644000176000001440000000134112415510734021202 0ustar ripleyuserscontext("sweepRegistry") test_that("sweepRegistry", { countFiles = function(reg, type) { fun = switch(type, scripts = BatchJobs:::getRScriptFilePath, logs = BatchJobs:::getLogFilePath ) fs = sapply(getJobIds(reg), fun, reg = reg) sum(sapply(fs, file.exists)) } reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:3) submitJobs(reg) waitForJobs(reg) expect_equal(countFiles(reg, "scripts"), 3L) expect_equal(countFiles(reg, "logs"), 3L) sweepRegistry(reg, "scripts") expect_equal(countFiles(reg, "scripts"), 0L) expect_equal(countFiles(reg, "logs"), 3L) sweepRegistry(reg, "logs") expect_equal(countFiles(reg, "scripts"), 0L) expect_equal(countFiles(reg, "logs"), 0L) }) BatchJobs/tests/testthat/test_export.R0000644000176000001440000000427712465170730017665 0ustar ripleyuserscontext("exports") test_that("exports", { reg = makeTestRegistry() p = file.path(reg$file.dir, "exports") save2(exported_x = 1:3, file = file.path(p, "x.RData")) loadExports(reg) expect_true(exists("exported_x", where=.GlobalEnv)) expect_equal(exported_x, 1:3) batchMap(reg, function(i) exported_x[i], i = 1:3) submitJobs(reg) waitForJobs(reg) res = loadResults(reg, simplify=TRUE, use.names="none") expect_equal(res, 1:3) }) test_that("exports with batchExport and batchUnexport", { reg = makeTestRegistry() expect_error(batchExport(reg, a = 1, li = list(a = 2)), "more than once") expect_equal(batchExport(reg, a = 1, b = 99), c("a", "b")) batchMap(reg, function(x) x + a + b, 1) submitJobs(reg) waitForJobs(reg) expect_equal(loadResult(reg, 1), 101) if (interactive()) { expect_equal(testJob(reg, 1), 101) expect_equal(testJob(reg, 1, external = FALSE), 101) expect_equal(testJob(reg, 1, external = TRUE), 101) } expect_equal(batchUnexport(reg, "a"), "a") suppressWarnings(rm(list = "a", envir = .GlobalEnv)) submitJobs(reg, 1) waitForJobs(reg) expect_equal(length(findErrors(reg)), 1) expect_true(grepl("not found", getErrorMessages(reg, 1), fixed = TRUE)) }) test_that("export: load defined files with loadExports", { reg = makeTestRegistry() batchExport(reg, x.1 = 4, x.2 = 3, x.3 = 2, x.4 = 1) suppressMessages(loadExports(reg, paste0("x.", seq(2, 6, 2)))) expect_false(exists("x.1", where = .GlobalEnv)) expect_true(exists("x.2", where = .GlobalEnv)) expect_false(exists("x.3", where = .GlobalEnv)) expect_true(exists("x.4", where = .GlobalEnv)) expect_false(exists("x.6", where = .GlobalEnv)) }) #FIXME: # I currently do not know how to run this test so it does not break R CMD Check # I get # > cannot open file 'startup.Rs': No such file or directory # 'make test' would work # --> run it only in interactive tests for now if (interactive()) { test_that("exports work with external testJob", { reg = makeTestRegistry() p = file.path(reg$file.dir, "exports") save2(exported_x = 1:3, file = file.path(p, "x.RData")) batchMap(reg, function(i) exported_x[i], i = 1:2) res = testJob(reg, 2L, external=TRUE) expect_equal(res, 2L) }) } BatchJobs/tests/testthat/helpers.R0000644000176000001440000000232312415510734016732 0ustar ripleyuserslibrary(BBmisc) options(BBmisc.ProgressBar.style = "off") options(BatchJobs.verbose = FALSE) conf = getConfig() conf$default.resources$walltime = 120 conf$default.resources$memory = 256 conf$mail.start = conf$mail.done = conf$mail.error = "none" conf$raise.warnings = FALSE conf$max.concurrent.jobs = Inf setConfig(conf = conf) rm(conf) getWorkDir = function() { work.dir = "unittests-files" # if env var is set we do the tests in the current wd, # if not, we do everything in a subdir of tempdir() # required for debians autopkgtest if (isExpensiveExampleOk()) file.path(getwd(), work.dir) else file.path(tempdir(), work.dir) } getTempDir = function() { file.path(getWorkDir(), basename(tempfile("tmp"))) } makeTestRegistry = function(packages = character(0L), work.dir = getWorkDir(), file.dir = getTempDir(), ...) { dir.create(file.dir, recursive = TRUE, showWarning = FALSE) makeRegistry( id = "unittests", seed = 1L, packages = packages, file.dir = file.dir, work.dir = work.dir, ... ) } # cleanup stray files cleanup = function() { unittests.dir = getWorkDir() if (file.exists(unittests.dir)) return(unlink(unittests.dir, recursive = TRUE) == 0L) return(TRUE) } BatchJobs/tests/testthat/test_reduceResults.R0000644000176000001440000001471312415510734021166 0ustar ripleyuserscontext("reduceResults") test_that("reduceResults", { reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:3) submitJobs(reg) waitForJobs(reg) y = reduceResults(reg, fun=function(aggr, job, res) aggr+res, init=0L) expect_equal(y, sum((1:3)^2)) }) test_that("reduceResults works with empty results", { reg = makeTestRegistry() batchMap(reg, function(x) x, 1) expect_equal(reduceResults(reg, fun=function(aggr, job, res) 1), NULL) expect_equal(reduceResults(reg, fun=function(aggr, job, res) 1, init=0L), 0L) submitJobs(reg) waitForJobs(reg) expect_equal(reduceResults(reg, fun=function(aggr, job, res) 1, ids=integer(0L)), NULL) expect_equal(reduceResults(reg, fun=function(aggr, job, res) 1, ids=integer(0L), init=0L), 0L) }) test_that("reduceResults works with imputations", { reg = makeTestRegistry() batchMap(reg, identity, 1:5) submitJobs(reg, c(1:2, 4:5)) waitForJobs(reg) expect_equal(reduceResults(reg, ids=1:5, fun=function(aggr, job, res) aggr+res, impute.val=3), 15) expect_equal(reduceResults(reg, ids=1:5, fun=function(aggr, job, res) sum(aggr, res, na.rm=TRUE), impute.val=NA), 12) }) test_that("reduceResults with multiple.result.files", { reg = makeTestRegistry(multiple.result.files=TRUE) batchMap(reg, function(x) list(foo = x, bar = 1), 1:10) submitJobs(reg) waitForJobs(reg) # quick check for named list as return value of loadResult expect_equal(loadResult(reg, 2)$foo, 2) expect_equal(loadResult(reg, 2, part="foo"), list(foo=2)) expect_equal(loadResult(reg, 2, part="bar"), list(bar=1)) expect_equal(loadResult(reg, 2, part=c("bar", "foo")), list(bar=1, foo=2)) # no part = all parts expect_equal(reduceResults(reg, fun=function(aggr, job, res) c(aggr, res$foo), init = c()), 1:10) expect_equal(reduceResults(reg, fun=function(aggr, job, res) c(aggr, res$foo), init = c(), part = c("foo", "bar")), 1:10) # wrong part expect_equal(reduceResults(reg, fun=function(aggr, job, res) c(aggr, res$foo), init = c(), part="bar"), NULL) # right part expect_equal(reduceResults(reg, fun=function(aggr, job, res) c(aggr, res$foo), init = c(), part="foo"), 1:10) }) test_that("reduceResultsReturnValue", { xs = 1:3 reg = makeTestRegistry() batchMap(reg, function(x) x^2, xs) submitJobs(reg) waitForJobs(reg) z1 = (1:3)^2 expect_equal(reduceResultsVector(reg, use.names="none"), z1) expect_equal(reduceResultsList(reg, use.names="none"), as.list(z1)) expect_equal(reduceResultsMatrix(reg, use.names="none"), matrix(z1, ncol=1)) # data.frame w/o colnames is nothing we want to test ... # y = data.frame(z1); colnames(y) = NULL y = data.frame(z1); colnames(y) = "X"; rownames(y) = as.character(rownames(y)) expect_equal(reduceResultsDataFrame(reg), y) z2 = z1 names(z2) = xs names(z2) = xs expect_equal(reduceResultsVector(reg, use.names="ids"), z2) expect_equal(reduceResultsList(reg, use.names="ids"), as.list(z2)) y = matrix(z2, ncol=1); rownames(y)=xs; colnames(y) = NULL expect_equal(reduceResultsMatrix(reg, use.names="ids"), y) reg = makeTestRegistry() batchMap(reg, function(x) list(a=x, b=x^2), xs) submitJobs(reg) waitForJobs(reg) expect_equal(reduceResultsVector(reg, fun=function(job, res) res$a, use.names="none"), xs) expect_equal(reduceResultsVector(reg, fun=function(job, res) res$b, use.names="none"), xs^2) expect_equal(reduceResultsList(reg, fun=function(job, res) res$a, use.names="none"), as.list(xs)) expect_equal(reduceResultsList(reg, fun=function(job, res) res$b, use.names="none"), as.list((xs)^2)) y = cbind(xs, z1); dimnames(y) = NULL expect_equal(reduceResultsMatrix(reg, use.names="none"), y) rownames(y) = xs; colnames(y) = c("a", "b") expect_equal(reduceResultsMatrix(reg, use.names="ids"), y) dimnames(y) = NULL; y = t(y) expect_equal(reduceResultsMatrix(reg, use.names="none", rows=FALSE), y) colnames(y) = xs; rownames(y) = c("a", "b") expect_equal(reduceResultsMatrix(reg, use.names="ids", rows=FALSE), y) y = data.frame(xs, z1); rownames(y) = as.character(xs); colnames(y) = c("a", "b") expect_equal(reduceResultsDataFrame(reg), y) reg = makeTestRegistry() batchMap(reg, function(i) list(a="a", b="b"), 1:2) submitJobs(reg) waitForJobs(reg) y = data.frame(a=rep("a", 2), b=rep("b", 2), stringsAsFactors=FALSE) rownames(y) = as.character(rownames(y)) expect_equal(reduceResultsDataFrame(reg, strings.as.factors=FALSE), y) y = data.frame(a=rep("a", 2), b=rep("b", 2), stringsAsFactors=TRUE) rownames(y) = as.character(rownames(y)) expect_equal(reduceResultsDataFrame(reg, strings.as.factors=TRUE), y) }) test_that("reduceResultsReturnValue works with empty results", { xs = integer(0) reg = makeTestRegistry() batchMap(reg, function(x) identity, xs) submitJobs(reg) waitForJobs(reg) expect_equal(reduceResultsVector(reg, use.names="none"), c()) expect_equal(reduceResultsList(reg, use.names="none"), list()) expect_equal(reduceResultsMatrix(reg, use.names="none"), matrix(0, nrow=0,ncol=0)) expect_equal(reduceResultsDataFrame(reg, use.names="none"), data.frame()) }) test_that("reduceResultsReturnValue works with imputations", { reg = makeTestRegistry() ids = batchMap(reg, identity, 1:5) submitJobs(reg, 1:4) waitForJobs(reg) res = c(1:4, NA_real_) expect_equal(reduceResultsVector(reg, ids, impute.val=NA_real_), setNames(res, 1:5)) expect_equal(reduceResultsList(reg, ids, impute.val=NA_real_), setNames(as.list(res), 1:5)) expect_equal(reduceResultsMatrix(reg, ids, impute.val=NA_real_), setRowNames(matrix(res), 1:5)) expect_equal(reduceResultsDataFrame(reg, ids, impute.val=NA_real_, use.names="none"), data.frame(X = res)) }) test_that("reduceResultsReturnValue works with other args", { reg = makeTestRegistry() xs = 1:3 batchMap(reg, identity, xs) submitJobs(reg) waitForJobs(reg) z = reduceResultsMatrix(reg, fun = function(job, res, y) (res-y)^2, y = 1, use.names = "none") expect_equal(z[,1], (xs-1)^2) }) test_that("reduceResultsReturnValue works with job names", { reg = makeTestRegistry() ns = letters[1:3] xs = setNames(1:3, ns) batchMap(reg, identity, xs, use.names=TRUE) submitJobs(reg) waitForJobs(reg) expect_equal(names(reduceResultsList(reg, use.names="names")), ns) expect_equal(names(reduceResultsVector(reg, use.names="names")), ns) expect_equal(rownames(reduceResultsDataFrame(reg, use.names="names")), ns) expect_equal(rownames(reduceResultsMatrix(reg, use.names="names")), ns) expect_equal(colnames(reduceResultsMatrix(reg, use.names="names", rows=FALSE)), ns) }) BatchJobs/tests/testthat/test_getJobInfo.R0000644000176000001440000000351312415510734020357 0ustar ripleyuserscontext("getJobInfo") test_that("getJobInfo", { reg = makeTestRegistry() batchMap(reg, function(x, i) x^i, 1:3, i = rep(2, 3)) mycheck = function(tab) { expect_true(is.data.frame(tab)) expect_equal(tab$id, 1:3) expect_true(nrow(tab) == 3) expect_true(is(tab$time.submitted, "POSIXt")) expect_true(is(tab$time.started, "POSIXt")) expect_true(is(tab$time.done, "POSIXt")) expect_true(is.numeric(tab$time.running)) expect_true(is.numeric(tab$memory)) expect_true(is.numeric(tab$time.queued)) expect_true(all(is.na(tab$error.msg))) expect_true(is.integer(tab$r.pid)) expect_true(is.integer(tab$seed)) } tab = getJobInfo(reg) mycheck(tab) submitJobs(reg) waitForJobs(reg) tab = getJobInfo(reg) mycheck(tab) tab = getJobInfo(reg, ids = integer(0)) expect_true(is.data.frame(tab)) expect_true(nrow(tab) == 0L) tab = getJobInfo(reg, pars=TRUE) expect_equal(tab$X, 1:3) expect_equal(tab$i, rep(2, 3)) tab = getJobInfo(reg, select = "time.running") expect_true(ncol(tab) == 2) # id always selected expect_true(names(tab)[1] == "id") tab = getJobInfo(reg, select = c("id", "time.running")) expect_true(ncol(tab) == 2) expect_error(getJobInfo(reg, select = "fooo"), "subset") }) # FIXME: for some reason this test produced mem = NA, but only in R CMD check # test_that("getJobInfo: memory correctly reported", { # if (!isWindows() && isExpensiveExampleOk()) { # reg = makeTestRegistry() # conf = getConfig() # on.exit(setConfig(conf = conf)) # setConfig(cluster.functions = makeClusterFunctionsLocal()) # ids = batchMap(reg, function(n) { x = 1:(10^n); x + rev(x) }, n = c(1, 6)) # submitJobs(reg, ids) # waitForJobs(reg, ids) # mem = getJobInfo(reg, select = "memory")$memory # expect_true(testNumeric(mem, any.missing = FALSE)) # } # }) BatchJobs/tests/testthat/test_sanitizePath.R0000644000176000001440000000277412415510734021004 0ustar ripleyuserscontext("sanitizePath") test_that("sanitizePath", { cS = function(x) gsub("\\", "/", x, fixed = TRUE) if (isLinux()) { setwd(tempdir()) paths = c(tempfile(), "..", "~") expect_identical(sanitizePath(paths, make.absolute = TRUE), c(paths[1], dirname(tempdir()), path.expand("~"))) expect_equal(sanitizePath(paths, make.absolute = TRUE) == paths, c(TRUE, FALSE, FALSE)) } else if (isWindows()) { setwd(tempdir()) paths = c(tempfile(), "..") expect_identical(sanitizePath(paths), cS(c(paths[1], dirname(tempdir())))) expect_equal(sanitizePath(paths, normalize = TRUE) == cS(paths), c(TRUE, FALSE)) } else if (isDarwin()) { # mac os often has a symlinked tempdir setwd(tempdir()) paths = c(tempfile(), "~") expect_identical(sanitizePath(paths, make.absolute = TRUE), cS(c(paths[1], path.expand("~")))) expect_equal(sanitizePath(paths, normalize = TRUE) == cS(paths), c(TRUE, FALSE)) } path = "c:\\temp" expect_false(sanitizePath(path, make.absolute = FALSE) == path) expect_identical(sanitizePath(path, make.absolute = FALSE), cS(path)) expect_identical(sanitizePath(path, make.absolute = TRUE), cS(path)) path = file.path(tempdir(), "..", fsep = "/") expect_false(sanitizePath(path, make.absolute = TRUE, normalize.absolute = TRUE) == cS(path)) expect_identical(sanitizePath(path, make.absolute = FALSE), cS(path)) if (!isDarwin()) expect_identical(sanitizePath(path, make.absolute = TRUE, normalize.absolute = TRUE), cS(dirname(dirname(path)))) }) BatchJobs/tests/testthat/test_testJob.R0000644000176000001440000000144412415510734017744 0ustar ripleyuserscontext("testJob") test_that("testJob", { reg = makeTestRegistry() id = 1L batchMap(reg, identity, 1) res = suppressAll(testJob(reg, id, external=FALSE)) expect_equal(res, 1) ids = findNotDone(reg) expect_equal(ids, id) expect_output({ st = showStatus(reg) }, "Status for 1 jobs") expect_equal(st$submitted, 0) expect_equal(st$started, 0) expect_equal(st$done, 0) expect_equal(st$error, 0) reg = makeTestRegistry() f = function(i) {library(xxxxx);1} batchMap(reg, f, 1) expect_error(testJob(reg, id, external=FALSE), "xxxxx") ids = findNotDone(reg) expect_equal(ids, id) expect_output({ df = showStatus(reg) }, "Status for 1 jobs") expect_equal(st$submitted, 0) expect_equal(st$started, 0) expect_equal(st$done, 0) expect_equal(st$error, 0) }) BatchJobs/tests/testthat/test_filterResults.R0000644000176000001440000000106012415510734021173 0ustar ripleyuserscontext("filterResults") test_that("filterResults", { reg = makeTestRegistry() batchMap(reg, function(x) x, 1:5) submitJobs(reg) waitForJobs(reg) ids = filterResults(reg, fun=function(job, res) res > 3) expect_equal(ids, c(4, 5)) ids = filterResults(reg, getJobIds(reg)[2:4], fun=function(job, res) res > 3) expect_equal(ids, c(4)) }) test_that("filterResults works with empty results", { reg = makeTestRegistry() batchMap(reg, function(x) x, 1) ids = filterResults(reg, fun=function(job, res) TRUE) expect_equal(ids, integer(0)) }) BatchJobs/tests/testthat/test_getJobParamDf.R0000644000176000001440000000037212415510734020776 0ustar ripleyuserscontext("getJobParamDf") test_that("getJobParamDf", { reg = makeTestRegistry() grid = batchExpandGrid(reg, function(x,y) x*y, x = 1:3, y = 5) expect_equal(getJobParamDf(reg), grid) expect_equal(getJobParamDf(reg, ids = 2:3), grid[2:3,]) }) BatchJobs/tests/testthat/test_batchMapQuick.R0000644000176000001440000000261312415510734021045 0ustar ripleyuserscontext("batchMapQuick") test_that("batchMapQuick", { reg = batchMapQuick(function(x) x^2, 1:3, file.dir = getTempDir()) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, (1:3)^2, check.attributes = FALSE) reg = batchMapQuick(function(x,y,z) (x+y)*z, 1:3, 4:6, more.args = list(z = 2), file.dir = getTempDir()) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, ((1:3) + (4:6))*2, check.attributes = FALSE) reg = batchMapQuick(identity, 1:3, packages = "MASS", chunk.size = 2, file.dir = getTempDir()) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, 1:3, check.attributes = FALSE) reg = batchMapQuick(identity, 1:3, inds = c(1,3), file.dir = getTempDir()) waitForJobs(reg) expect_equal(findDone(reg), c(1,3)) y = sapply(c(1,3), function(id) loadResult(reg, id)) expect_equal(y, c(1,3), check.attributes = FALSE) reg = batchMapQuick(identity, file.dir = getTempDir()) waitForJobs(reg) expect_equal(getJobNr(reg), 0L) }) test_that("batchMapQuick chunks properly", { prev = getOption("BatchJobs.verbose", TRUE) options(BatchJobs.verbose = TRUE) expect_message({ reg = batchMapQuick(identity, 1:4, chunk.size = 2, file.dir = getTempDir()) }, "Submitting 2 chunks / 4 jobs") options(BatchJobs.verbose = prev) waitForJobs(reg) }) BatchJobs/tests/testthat/test_loadResults.R0000644000176000001440000000244512415510734020635 0ustar ripleyuserscontext("loadResults") test_that("loadResults", { reg = makeTestRegistry() ids = 1:2 batchMap(reg, identity, ids) submitJobs(reg) waitForJobs(reg) ys1 = 1:2 ys2 = loadResults(reg, simplify=TRUE, use.names="none") expect_equal(ys1, ys2) ys1 = as.list(ys1) ys2 = loadResults(reg, simplify=FALSE, use.names="none") expect_equal(ys1, ys2) names(ys1) = ids ys2 = loadResults(reg, simplify=FALSE, use.names="ids") expect_equal(ys1, ys2) ys1 = unlist(ys1) ys2 = loadResults(reg, simplify=TRUE, use.names="ids") expect_equal(ys1, ys2) ys2 = loadResults(reg, 2) expect_equal(list("2"=2), ys2) nl = list() names(nl) = character(0L) expect_equal(loadResults(reg, ids=integer(0), simplify=TRUE, use.names="ids"), nl) expect_equal(loadResults(reg, ids=integer(0), simplify=FALSE, use.names="ids"), nl) expect_equal(loadResults(reg, ids=integer(0), simplify=TRUE, use.names="none"), list()) expect_equal(loadResults(reg, ids=integer(0), simplify=FALSE, use.names="none"), list()) # test names of loadResults reg = makeTestRegistry() batchMap(reg, identity, letters, use.names=TRUE) submitJobs(reg) waitForJobs(reg) expect_equal(names(loadResults(reg, use.names = "ids")), as.character(1:26)) expect_equal(names(loadResults(reg, use.names = "names")), letters) }) BatchJobs/tests/testthat/test_submitJobs.R0000644000176000001440000000311512415510734020450 0ustar ripleyuserscontext("submitJobs") test_that("submitJobs", { reg = makeTestRegistry() batchMap(reg, identity, 123) submitJobs(reg) waitForJobs(reg) y = loadResult(reg, 1L) expect_equal(y, 123) # check conversion reg = makeTestRegistry() batchMap(reg, identity, 1:2) submitJobs(reg) waitForJobs(reg) submitJobs(reg, 1:2) waitForJobs(reg) submitJobs(reg, c(1,2)) waitForJobs(reg) }) test_that("submitJobs works with empty id vector", { reg = makeTestRegistry() batchMap(reg, identity, 1:10) submitJobs(reg, ids = integer(0L)) waitForJobs(reg) expect_equal(findSubmitted(reg), integer(0L)) }) test_that("submitJobs works with multiple result files", { reg = makeTestRegistry(multiple.result.files=TRUE) # no list returned batchMap(reg, identity, 1) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 1L) reg = makeTestRegistry(multiple.result.files=TRUE) f = function(x) list(a=x, b=2*x) ids = 1:2 batchMap(reg, f, ids) submitJobs(reg) waitForJobs(reg) expect_equal(findDone(reg), ids) ys = loadResults(reg, ids) expect_equal(ys, list("1"=list(a=1, b=2), "2"=list(a=2, b=4))) ys = loadResults(reg, 2, part="b") expect_equal(ys, list("2"=list(b=4))) }) test_that("submitJobs works with chunking", { reg = makeTestRegistry() batchMap(reg, identity, 1:5) ch = chunk(getJobIds(reg), chunk.size=2) submitJobs(reg, ids=ch[1:2]) waitForJobs(reg) expect_equal(findDone(reg), 1:4) submitJobs(reg, ids=ch) waitForJobs(reg) expect_equal(findDone(reg), 1:5) expect_equal(loadResults(reg, simplify=TRUE, use.names="none"), 1:5) }) BatchJobs/tests/testthat/test_resources.R0000644000176000001440000000243712424457704020356 0ustar ripleyuserscontext("resources") test_that("resources", { # FIXME this is still broken for all major cluster system # because the resources are stupid! # use resources on slave reg = makeTestRegistry() batchMap(reg, function(i) getResources(), 1:2) res = list(walltime=5*60, memory=500) submitJobs(reg, resources=res) waitForJobs(reg) expect_equal(loadResult(reg, 1)[names(res)], res) # see FIXME in testJob.R # expect_equal(testJob(reg, 1, resources=res, external=FALSE)[names(res)], res) # query on master res1 = getJobResources(reg, 1)[[1]] res2 = getJobResources(reg, 2)[[1]] expect_equal(res1, res2) expect_equal(res1[names(res)], res) #submit only a few jobs resetJobs(reg, ids=1:2, force=TRUE) submitJobs(reg, ids=1, resources=res) waitForJobs(reg) expect_equal(loadResult(reg, 1)[names(res)], res) expect_error(getJobResources(reg, 1:2), "not been submitted") res1 = getJobResources(reg) res2 = getJobResources(reg, 1) expect_equal(res1, res2) # defaults in conf # conf = BatchJobs:::getBatchJobsConf() # conf$default.resources = list(walltime=1, memory=2, xxx=3) # reg = makeTestRegistry() # batchMap(reg, function(i) getResources(), 1) # expect_error(submitJobs(reg, resources=list(memory=200)), "Illegal resources used") # waitForJobs(reg) }) BatchJobs/tests/testthat/test_resetJobs.R0000644000176000001440000000071612415510734020273 0ustar ripleyuserscontext("resetJobs") test_that("resetJobs", { reg = makeTestRegistry() batchMap(reg, identity, 1:3) ids = getJobIds(reg) submitJobs(reg) waitForJobs(reg) done = findDone(reg) # test that nothing happens on empty id vector resetJobs(reg) resetJobs(reg, ids=integer(0L), force=TRUE) expect_equal(done, findDone(reg)) # now really reset the first job resetJobs(reg, done[1], force=TRUE) expect_equal(setdiff(done, findDone(reg)), 1) }) BatchJobs/tests/testthat/test_getLogFiles.R0000644000176000001440000000067112415510734020537 0ustar ripleyuserscontext("getLogFiles") test_that("getLogFiles", { reg = makeTestRegistry() batchMap(reg, identity, 1:10) ids = getJobIds(reg) ch = chunk(ids[1:8], chunk.size=4) submitJobs(reg, ch) submitJobs(reg, ids[9:10]) waitForJobs(reg) fs = getLogFiles(reg, ids) expect_equal(length(unique(fs)), 4) expect_true(grepl("9", fs[9])) expect_true(grepl("10", fs[10])) expect_equal(getLogFiles(reg, integer(0L)), character(0L)) }) BatchJobs/tests/testthat/test_waitForJobs.R0000644000176000001440000000103612415510734020560 0ustar ripleyuserscontext("waitForJobs") test_that("waitForJobs", { f = function(x) { if (x == 5) stop("x == 5") else x } reg = makeTestRegistry() batchMap(reg, f, 1:5) submitJobs(reg) waitForJobs(reg) expect_equal(waitForJobs(reg, 1:5, stop.on.error = FALSE), FALSE) expect_warning(waitForJobs(reg, 1:5, stop.on.error = TRUE)) expect_equal(suppressWarnings(waitForJobs(reg, 1:5, stop.on.error = TRUE)), FALSE) expect_equal(waitForJobs(reg, 1:4, stop.on.error=FALSE), TRUE) expect_equal(waitForJobs(reg, 1:4, stop.on.error=TRUE), TRUE) }) BatchJobs/tests/testthat/test_batchMap.R0000644000176000001440000000231212415510734020044 0ustar ripleyuserscontext("batchMap") test_that("batchMap", { reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:3) submitJobs(reg) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, (1:3)^2, check.attributes = FALSE) reg = makeTestRegistry() batchMap(reg, function(x,y,z) (x+y)*z, 1:3, 4:6, more.args = list(z = 2)) submitJobs(reg) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, ((1:3) + (4:6))*2, check.attributes = FALSE) reg = makeTestRegistry() expect_equal(batchMap(reg, function(...) 1), integer(0L)) expect_equal(batchMap(reg, function(...) 1, i = integer(0L)), integer(0L)) reg = makeTestRegistry() expect_equal(batchMap(reg, function(...) 1, a = 1:3, b = 1:3, use.names = TRUE), 1:3) # factors reg = makeTestRegistry() ids = batchMap(reg, identity, factor(letters[1:5])) submitJobs(reg) waitForJobs(reg) expect_equal(loadResults(reg, simplify = TRUE), setNames(factor(letters[1:5]), 1:5)) # primitive functions reg = makeTestRegistry() ids = batchMap(reg, sin, 1:2) submitJobs(reg) waitForJobs(reg) expect_equal(as.numeric(loadResults(reg, simplify = TRUE)), sin(1:2)) }) BatchJobs/tests/testthat/test_setJobFunction.R0000644000176000001440000000065712415510734021273 0ustar ripleyuserscontext("setJobFunction") test_that("setJobFunction", { f = function(x) if(x == 2) stop(2) else x reg = makeTestRegistry() batchMap(reg, f, x = 1:3) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 2L) expect_equal(findDone(reg), c(1L, 3L)) f = function(x) x setJobFunction(reg, fun=f, ids=2L, reset=TRUE, force=TRUE) submitJobs(reg, 2L) waitForJobs(reg) expect_equal(findDone(reg), 1:3) }) BatchJobs/tests/testthat/test_findJobs.R0000644000176000001440000000227712415510734020075 0ustar ripleyuserscontext("findJobs") test_that("findJobs", { reg = makeTestRegistry() batchMap(reg, function(x) x, 10:13) ids = findJobs(reg, pars=.arg1 > 10) expect_equal(ids, 2:4) reg = makeTestRegistry() batchMap(reg, function(x) x, x=10:13) ids = findJobs(reg, pars=x > 10) expect_equal(ids, 2:4) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) x*y, x=1:2, y=10:11) ids = findJobs(reg, pars=x == 2) expect_true(length(ids) == 2) ids = findJobs(reg, pars=x == 1 && y == 11) expect_true(length(ids) == 1) reg = makeTestRegistry() xi = 3 batchExpandGrid(reg, function(x, y) x+y, x=1:4, y=1:4) expect_equal(findJobs(reg, pars = (x == y)), c(1, 6, 11, 16)) expect_equal(findJobs(reg, pars = (x == xi)), c(3, 7, 11, 15)) }) test_that("findJobs with names", { reg = makeTestRegistry() batchMap(reg, identity, letters[1:5], use.names=TRUE) ids = findJobs(reg, jobnames = c("b", "c", "d")) expect_equal(ids, 2:4) submitJobs(reg, ids) waitForJobs(reg) expect_equal(names(loadResults(reg, ids, use.names = "names")), c("b", "c", "d")) setJobNames(reg, 1:5, sprintf("x%i", 1:5)) expect_equal(names(loadResults(reg, ids, use.names = "names")), c("x2", "x3", "x4")) }) BatchJobs/tests/testthat/test_showStatus.R0000644000176000001440000000153712415510734020521 0ustar ripleyuserscontext("showStatus") test_that("showStatus", { reg = makeTestRegistry() f = function(x) { if (x < 3) stop() else { Sys.sleep(2) x } } batchMap(reg, f, 1:3) submitJobs(reg) waitForJobs(reg) expect_output({ showStatus(reg) }, "Status for 3 jobs") expect_output({ showStatus(reg, getJobIds(reg)[2:3]) }, "Status for 2 jobs") }) test_that("showStatus only shows some errors", { reg = makeTestRegistry() batchMap(reg, function(x) stop(), 1:3) submitJobs(reg) waitForJobs(reg) expect_output({ showStatus(reg, errors=1) }, "Showing first 1 errors:") }) test_that("showStatus works with empty id vector", { reg = makeTestRegistry() batchMap(reg, function(x) stop(), 1:3) submitJobs(reg) waitForJobs(reg) expect_output({ showStatus(reg, integer(0L)) }, "Status for 0 jobs") }) BatchJobs/tests/testthat/test_batchReduce.R0000644000176000001440000000134312415510734020541 0ustar ripleyuserscontext("batchReduce") test_that("batchReduce", { reg = makeTestRegistry() ids = batchReduce(reg, function(aggr, x) aggr+x, 1:20, init=0, block.size=10) submitJobs(reg) waitForJobs(reg) y = reduceResults(reg, fun=function(aggr, job, res) aggr+res, init=0) expect_equal(y, sum(1:20)) expect_equal(ids, 1:2) reg = makeTestRegistry() ids = batchReduce(reg, function(aggr, x, y) aggr+x+y, 1:20, init=0, block.size=10, more.args=list(y=1)) submitJobs(reg) waitForJobs(reg) y = reduceResults(reg, fun=function(aggr, job, res) aggr+res, init=0) expect_equal(y, sum(1:20)+20) reg = makeTestRegistry() expect_equal(batchReduce(reg, function(aggr, x) aggr+x, integer(0L), init=0, block.size=10), integer(0L)) }) BatchJobs/tests/testthat/test_findStatus.R0000644000176000001440000000063112415510734020453 0ustar ripleyuserscontext("findStatus") test_that("findStatus", { reg = makeTestRegistry() f = function(i) { if (i == 2) stop(123) } batchMap(reg, f, 1:3) ids = getJobIds(reg) submitJobs(reg) waitForJobs(reg) j = findDone(reg) expect_equal(j, ids[-2]) j = findErrors(reg) expect_equal(j, ids[2]) j = findNotDone(reg) expect_equal(j, ids[2]) j = findStarted(reg) expect_equal(j, 1:3) }) BatchJobs/tests/testthat/test_batchMapResults.R0000644000176000001440000000163212415510734021432 0ustar ripleyuserscontext("batchMapResults") test_that("batchMapResults", { reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:3) submitJobs(reg) waitForJobs(reg) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir = td) batchMapResults(reg, reg2, fun=function(job, res, y, z) (res+y)*z, 4:6, more.args=list(z=2)) submitJobs(reg2) waitForJobs(reg2) expect_equal(loadResults(reg2, use.names="none", simplify=TRUE), ((1:3)^2 + (4:6))*2) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir = td) ids = getJobIds(reg) batchMapResults(reg, reg2, function(job, res, y, z) (res+y)*z, 5:6, ids=ids[2:3], more.args=list(z=2)) submitJobs(reg2) waitForJobs(reg2) expect_equal(loadResults(reg2, use.names="none", simplify=TRUE), ((2:3)^2 + (5:6))*2) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir = td) batchMapResults(reg, reg2, function(job, res, ...) res, ids = integer(0L)) }) BatchJobs/tests/testthat/test_batchReduceResults.R0000644000176000001440000000220012415510734022114 0ustar ripleyuserscontext("batchReduceResults") test_that("batchReduceResults", { reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:5) submitJobs(reg) waitForJobs(reg) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir=td) batchReduceResults(reg, reg2, block.size=3, init=c(), fun=function(aggr, job, res) c(aggr, res)) submitJobs(reg2) waitForJobs(reg2) expect_equal(loadResult(reg2, getJobIds(reg)[1]), (1:3)^2, check.names=FALSE) expect_equal(loadResult(reg2, getJobIds(reg)[2]), (4:5)^2, check.names=FALSE) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir=td) ids = getJobIds(reg) batchReduceResults(reg, reg2, ids=ids[2:4], block.size=2, init=c(), fun=function(aggr, job, res) c(aggr, res)) submitJobs(reg2) waitForJobs(reg2) expect_equal(loadResult(reg2, getJobIds(reg)[1]), (2:3)^2, check.names=FALSE) expect_equal(loadResult(reg2, getJobIds(reg)[2]), (4)^2, check.names=FALSE) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir=td) expect_equal(batchReduceResults(reg, reg2, ids = integer(0L), block.size=2, init=c(), fun=function(aggr, job, res) c(aggr, res)), integer(0L)) }) BatchJobs/tests/testthat/test_sourceRegistryFiles.R0000644000176000001440000000672712415510734022357 0ustar ripleyuserscontext("source registry files") test_that("source registry files", { mywd = file.path(getWorkDir(), "myworkdir") dir.create(mywd, recursive = TRUE, showWarnings = FALSE) setwd(mywd) src.dir.subdir = c(file.path(mywd, "unittest-sources-subdir"), "unittest-sources-subdir-relative") x = lapply(src.dir.subdir, dir.create, recursive = TRUE, showWarnings = FALSE) src.dir.parent = c(file.path(dirname(mywd), "unittest-sources-parent"), "../unittest-sources-parent-relative") x = lapply(src.dir.parent, dir.create, recursive = TRUE, showWarnings = FALSE) for (src.dir in c(src.dir.subdir, src.dir.parent)) { cat("xxx = 123", file = file.path(src.dir, "test.R")) reg = makeTestRegistry(src.dir = src.dir, work.dir = mywd) expect_true(exists("xxx", envir = .GlobalEnv)) rm(list = "xxx", envir = .GlobalEnv) loadRegistry(reg$file.dir) expect_true(exists("xxx", envir = .GlobalEnv)) rm(list = "xxx", envir = .GlobalEnv) batchMap(reg, function(i) i + xxx, i = 1:3) submitJobs(reg) waitForJobs(reg) res = loadResults(reg, simplify = TRUE, use.names = "none") expect_equal(res, 123 + 1:3) } src.files.subdir = file.path(src.dir.subdir, "test.R") src.files.parent = file.path(src.dir.parent, "test.R") for (src.files in c(src.files.subdir, src.files.parent)) { reg = makeTestRegistry(src.files = src.files, work.dir = mywd) expect_true(exists("xxx", envir = .GlobalEnv)) rm(list = "xxx", envir = .GlobalEnv) loadRegistry(reg$file.dir) expect_true(exists("xxx", envir = .GlobalEnv)) rm(list = "xxx", envir = .GlobalEnv) batchMap(reg, function(i) i + xxx, i = 1:3) submitJobs(reg) waitForJobs(reg) res = loadResults(reg, simplify = TRUE, use.names = "none") expect_equal(res, 123 + 1:3) } }) test_that("source registry mutators work", { mywd = file.path(getWorkDir(), "myworkdir") dir.create(mywd, recursive = TRUE, showWarnings = FALSE) setwd(mywd) src.dir.subdir = c(file.path(mywd, "unittest-sources-subdir"), "unittest-sources-subdir-relative") x = lapply(src.dir.subdir, dir.create, recursive = TRUE, showWarnings = FALSE) src.dir.parent = c(file.path(dirname(mywd), "unittest-sources-parent"), "../unittest-sources-parent-relative") x = lapply(src.dir.parent, dir.create, recursive = TRUE, showWarnings = FALSE) src.files.subdir = file.path(src.dir.subdir, "test.R") src.files.parent = file.path(src.dir.parent, "test.R") reg = makeTestRegistry() expect_equal(reg$src.files, character(0L)) expect_equal(reg$src.dirs, character(0L)) reg = addRegistrySourceFiles(reg, src.files.subdir, src.now = FALSE) expect_equal(reg$src.files, sanitizePath(src.files.subdir, make.absolute = FALSE)) expect_equal(reg$src.dirs, character(0L)) reg = addRegistrySourceFiles(reg, src.files.subdir, src.now = FALSE) expect_equal(reg$src.files, sanitizePath(src.files.subdir, make.absolute = FALSE)) expect_equal(reg$src.dirs, character(0L)) reg = addRegistrySourceDirs(reg, src.dir.parent, src.now = FALSE) expect_equal(reg$src.files, sanitizePath(src.files.subdir, make.absolute = FALSE)) expect_equal(reg$src.dirs, sanitizePath(src.dir.parent, make.absolute = FALSE)) reg = removeRegistrySourceFiles(reg, reg$src.files) expect_equal(reg$src.files, character(0L)) expect_equal(reg$src.dirs, sanitizePath(src.dir.parent, make.absolute = FALSE)) reg = removeRegistrySourceDirs(reg, reg$src.dirs) expect_equal(reg$src.files, character(0L)) expect_equal(reg$src.dirs, character(0L)) }) BatchJobs/tests/testthat/test_zzz_cleanup.R0000644000176000001440000000012712415510734020673 0ustar ripleyuserscontext("cleanup") test_that("Removing unittest files", { expect_true(cleanup()) }) BatchJobs/tests/testthat/test_getJob.R0000644000176000001440000000165412415510734017547 0ustar ripleyuserscontext("getJob") test_that("getJob", { reg = makeTestRegistry() batchMap(reg, identity, 1:3) ids = getJobIds(reg) j = getJob(reg, ids[2]) expect_equal(j$id, ids[2]) expect_equal(j$pars, list(2)) expect_is(j, "Job") js = getJobs(reg, ids[2:3]) expect_is(js, "list") expect_equal(length(js), 2) expect_is(js[[1]], "Job") expect_is(js[[2]], "Job") expect_equal(js[[1]]$id, ids[2]) expect_equal(js[[2]]$id, ids[3]) expect_equal(js[[1]]$pars, list(2)) expect_equal(js[[2]]$pars, list(3)) # empty ids expect_equal(getJobs(reg, integer(0)), list()) # bad id expect_error(getJob(reg, "xxx"), "integer") expect_error(getJobs(reg, -1), "Ids not present in registry: -1") }) test_that("getJob returns jobs in same order as ids", { reg = makeTestRegistry() batchMap(reg, identity, 1:3) ids = getJobIds(reg) ids = rev(ids) js = getJobs(reg, ids) expect_equal(ids, extractSubList(js, "id")) }) BatchJobs/tests/testthat/test_database.R0000644000176000001440000000274212415510734020100 0ustar ripleyuserscontext("database") test_that("database", { err.msg = "error with 'quotes'" reg = makeTestRegistry() batchMap(reg, identity, 1) BatchJobs:::dbSendMessage(reg, BatchJobs:::dbMakeMessageError(reg, 1, err.msg=err.msg)) expect_equal(unname(getErrorMessages(reg)), "error with \"quotes\"") }) test_that("dbSelectWithIds works", { reg = makeTestRegistry() batchMap(reg, identity, 1:3) query = sprintf("SELECT job_id from %s_job_status", reg$id) jids = BatchJobs:::dbSelectWithIds(reg, query, where=TRUE, limit=2)$job_id expect_equal(jids, 1:2) submitJobs(reg, 1:2) waitForJobs(reg) query = sprintf("SELECT job_id from %s_job_status where submitted IS NOT NULL", reg$id) jids = BatchJobs:::dbSelectWithIds(reg, query, where=FALSE)$job_id expect_equal(jids, 1:2) jids = BatchJobs:::dbSelectWithIds(reg, query, ids=1:3, where=FALSE)$job_id expect_equal(jids, 1:2) jids = BatchJobs:::dbSelectWithIds(reg, query, where=FALSE, limit=1)$job_id expect_equal(jids, 1) jids = BatchJobs:::dbSelectWithIds(reg, query, ids=1:3, where=FALSE, limit=1)$job_id expect_equal(jids, 1) jids = BatchJobs:::dbSelectWithIds(reg, query, ids=2:3, where=FALSE)$job_id expect_equal(jids, 2) reg = makeTestRegistry() batchMap(reg, identity, 1:4) submitJobs(reg, 3:4) waitForJobs(reg) query = sprintf("SELECT job_id from %s_job_status where submitted IS NOT NULL", reg$id) jids = BatchJobs:::dbSelectWithIds(reg, query, ids=1:4, where=FALSE,)$job_id expect_equal(jids, 3:4) }) BatchJobs/tests/testthat/test_sqlquotes.R0000644000176000001440000000107212415510734020367 0ustar ripleyuserscontext("sqlquotes") test_that("sqlquotes", { reg = makeTestRegistry() f = function(x) stop("Quoting is 'on'") batchMap(reg, f, 1:2) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 1:2) reg = makeTestRegistry() f = function(x) stop('Quoting is "on"') batchMap(reg, f, 1:2) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 1:2) reg = makeTestRegistry() f = function(x) stop("Some special chars: \n{}\r\t(),;") batchMap(reg, f, 1:2) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 1:2) }) BatchJobs/tests/testthat/test_batchExpandGrid.R0000644000176000001440000000211412415510734021354 0ustar ripleyuserscontext("batchExpandGrid") test_that("batchExpandGrid", { reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) x*y, 1:3, 5) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(y, (1:3) * 5) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) x*y, 1:3, 2:3) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(sum(y), sum(outer(1:3, 2:3))) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y,z) x*y+z, 1:3, 5, more.args=list(z=10)) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(y, ((1:3) * 5) + 10) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) paste(x,y), 1, c("a", "b")) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(y, c("1 a", "1 b")) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) x^y, y=2, x=1:3) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(y, (1:3)^2) }) BatchJobs/tests/testthat/test_packages.R0000644000176000001440000000074112415510734020107 0ustar ripleyuserscontext("packages") test_that("packages", { reg = makeTestRegistry(packages = "MASS") expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "MASS"))) reg = addRegistryPackages(reg, "testthat") expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "MASS", "testthat"))) reg = removeRegistryPackages(reg, "MASS") expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "testthat"))) }) BatchJobs/tests/testthat/test_makeRegistry.R0000644000176000001440000000241612502316642020777 0ustar ripleyuserscontext("makeRegistry") test_that("makeRegistry", { reg = makeTestRegistry() expect_true(inherits(reg, "Registry")) expect_true(is.list(reg)) expect_true(file.exists(reg$file.dir)) expect_true(file.exists(file.path(reg$file.dir, "BatchJobs.db"))) expect_output(print(reg), "Job registry") df = BatchJobs:::dbGetJobStatusTable(reg) expect_true(is.data.frame(df) && nrow(df) == 0 && ncol(df) == 13) }) test_that("makeRegistry checks id", { expect_error(makeRegistry(id="runit-files"), "comply with") expect_error(makeRegistry(id="-files"), "comply with") }) test_that("makeRegistry checks packs", { makeTestRegistry(packages="base") expect_error(makeTestRegistry(packages="foo"), "Please install the following packages: foo") }) test_that("loadRegistry works", { reg1 = makeTestRegistry() reg2 = loadRegistry(reg1$file.dir) expect_is(reg2, "Registry") }) test_that("torturing makeRegistry/removeRegistry to create registries over and over works", { ## This fails (at least) on Windows if we set the working directory ## to be the current working directory, i.e. ".". This can be achieve ## using Sys.setenv("R_EXPENSIVE_EXAMPLE_OK"=TRUE). for (ii in 1:50) { reg = makeTestRegistry() expect_true(removeRegistry(reg, ask = "no")) } }) BatchJobs/tests/testthat/test_grepLogs.R0000644000176000001440000000070012415510734020106 0ustar ripleyusers# FIXME: can we find a way to generate log files in unit tests? context("grepLogs") test_that("grepLogs", { #.... no logs for unittests :( # reg = makeTestRegistry() # f = function(x) { # if (x %% 2 == 0) warning(x) else x # } # batchMap(reg, f, 1:10) # submitJobs(reg) # expect_equal(grepLogs(reg), 1:5*2L) # expect_equal(grepLogs(reg, ids=1:2), 2L) # expect_equal(grepLogs(reg, pattern = "xxx_nomatch_xxx"), integer(0)) }) BatchJobs/tests/testthat/test_doJob.R0000644000176000001440000000361412415510734017370 0ustar ripleyuserscontext("doJob") test_that("doJob", { reg = makeTestRegistry() id = 1L batchMap(reg, identity, 123) df = BatchJobs:::dbGetJobStatusTable(reg) expect_true(is.data.frame(df) && nrow(df) == 1 && ncol(df) == 13) ids = findNotDone(reg) expect_equal(ids, id) BatchJobs:::saveConf(reg) expect_output({ y = BatchJobs:::doJob(reg, id, multiple.result.files=FALSE, disable.mail=TRUE, last=id, array.id=NA_integer_) }, "BatchJobs job") waitForJobs(reg) expect_equal(y, TRUE) expect_equal(loadResult(reg, 1L), 123) df = BatchJobs:::dbGetJobStatusTable(reg) expect_true(!is.na(df$started) && !is.na(df$done) && is.na(df$error)) y = loadResult(reg, id) expect_equal(y, 123) ids = findNotDone(reg) expect_equal(length(ids), 0) # test working directory reg = makeTestRegistry() wd.now = getwd() wd.job = reg$work.dir bar = 123 save(bar=bar, file=file.path(wd.job, "foo.RData")) reg$work.dir = wd.job f = function(x) { load("foo.RData") bar+x } batchMap(reg, f, 1) BatchJobs:::saveConf(reg) expect_output({ y = BatchJobs:::doJob(reg, id, multiple.result.files=FALSE, disable.mail=TRUE, last=id, array.id=NA_integer_) }, "BatchJobs job") expect_equal(y, TRUE) expect_equal(loadResult(reg, 1L), bar + 1) expect_equal(getwd(), wd.now) unlink(file.path(wd.job, "foo.RData")) # test packages # be sneaky otherwise we get error here due to pack check reg = makeTestRegistry() reg$packages = list(foo="foo") BatchJobs:::saveRegistry(reg) batchMap(reg, identity, 1) expect_error(suppressAll(testJob(reg, 1)), "Please install the following packages: foo") expect_equal(findNotDone(reg), id) if (isExpensiveExampleOk()) { reg = makeTestRegistry(packages=c("randomForest")) f = function(i) randomForest(Species~., data=iris) batchMap(reg, f, 1) submitJobs(reg) waitForJobs(reg) expect_equal(length(findNotDone(reg)), 0) } }) BatchJobs/NAMESPACE0000644000176000001440000000630212471624612013366 0ustar ripleyusers# Generated by roxygen2 (4.1.0): do not edit by hand S3method(applyJobFunction,Registry) S3method(copyRequiredJobFiles,Registry) S3method(dbCreateJobDefTable,Registry) S3method(dbGetJobs,Registry) S3method(getJobInfo,Registry) S3method(getJobs,Registry) S3method(getWorkerNumberOfCPUs,WorkerLinux) S3method(getWorkerStatus,WorkerLinux) S3method(killWorkerJob,WorkerLinux) S3method(listWorkerJobs,WorkerLinux) S3method(print,ClusterFunctions) S3method(print,Config) S3method(print,Job) S3method(print,Registry) S3method(print,SubmitJobResult) S3method(setJobFunction,Registry) S3method(startWorkerJob,WorkerLinux) S3method(updateRegistry,Registry) export(addRegistryPackages) export(addRegistrySourceDirs) export(addRegistrySourceFiles) export(applyJobFunction) export(batchExpandGrid) export(batchExport) export(batchMap) export(batchMapQuick) export(batchMapResults) export(batchReduce) export(batchReduceResults) export(batchUnexport) export(callFunctionOnSSHWorkers) export(cfBrewTemplate) export(cfHandleUnknownSubmitError) export(cfKillBatchJob) export(cfReadBrewTemplate) export(copyRequiredJobFiles) export(dbCreateJobDefTable) export(dbGetJobs) export(debugMulticore) export(debugSSH) export(filterResults) export(findDisappeared) export(findDone) export(findErrors) export(findExpired) export(findJobs) export(findMissingResults) export(findNotDone) export(findNotErrors) export(findNotOnSystem) export(findNotRunning) export(findNotStarted) export(findNotSubmitted) export(findNotTerminated) export(findOnSystem) export(findRunning) export(findStarted) export(findSubmitted) export(findTerminated) export(getConfig) export(getErrorMessages) export(getJob) export(getJobIds) export(getJobInfo) export(getJobNr) export(getJobParamDf) export(getJobResources) export(getJobs) export(getLogFiles) export(getResources) export(getSSHWorkersInfo) export(getStatus) export(grepLogs) export(installPackagesOnSSHWorkers) export(killJobs) export(loadConfig) export(loadExports) export(loadRegistry) export(loadResult) export(loadResults) export(makeClusterFunctions) export(makeClusterFunctionsInteractive) export(makeClusterFunctionsLSF) export(makeClusterFunctionsLocal) export(makeClusterFunctionsMulticore) export(makeClusterFunctionsSGE) export(makeClusterFunctionsSLURM) export(makeClusterFunctionsSSH) export(makeClusterFunctionsTorque) export(makeJob) export(makeRegistry) export(makeSSHWorker) export(makeSubmitJobResult) export(reduceResults) export(reduceResultsDataFrame) export(reduceResultsList) export(reduceResultsMatrix) export(reduceResultsVector) export(removeRegistry) export(removeRegistryPackages) export(removeRegistrySourceDirs) export(removeRegistrySourceFiles) export(resetJobs) export(sanitizePath) export(setConfig) export(setJobFunction) export(setJobNames) export(showClusterStatus) export(showLog) export(showStatus) export(sourceRegistryFiles) export(submitJobs) export(sweepRegistry) export(testJob) export(updateRegistry) export(waitForJobs) import(BBmisc) import(DBI) import(RSQLite) import(checkmate) import(fail) import(methods) import(stats) import(utils) importFrom(brew,brew) importFrom(digest,digest) importFrom(parallel,detectCores) importFrom(sendmailR,sendmail) importFrom(stringr,str_extract) importFrom(stringr,str_trim) BatchJobs/NEWS0000644000176000001440000001420212502316642012640 0ustar ripleyusersBatchJobs_1.6: - Added functions: getStatus, removeRegistry, findDisappeared - All find* function now support a 'limit' argument (default unlimited) - We are less strict in arg checks of makeSSHWorker, so you can set a load higher than ncpus - loadExports: added argument "what" - batchApply now works with arrays - filehandling for the registry, especially on Windows, was improved (Thx Henrik Bengtsson) - You can set pragmas via db.options in your config file. - The SQLite pragma "busy_timeout" is now set per default with a value of 5000ms. - The default number of cores is now retrieved from the option "mc.cores" rather - New configuration option BatchJobs.load.config - Fixed some minor things. BatchJobs_1.5: - Updated to use RSQLite >=1.0.0 - Fixed a bug in waitForJobs(). - Fixed a regression parsing array job ids. - Argument 'progressbar' added to multiple functions to suppress displaying the progress bars. - Configuration option 'staged.queries' is now enabled per default. BatchJobs_1.4: - Added functions: batchExport, batchUnexport, loadExports - Added functions: addRegistryPackages, removeRegistryPackages - Added functions: addRegistrySourceFiles, addRegistrySourceDirs, removeRegistrySourceFiles, removeRegistrySourceDirs - Added functions: getJobParamDf - New option: "BatchJobs.clear.function.env". If TRUE the function environment will be erased prior to writing it to the file system. Otherwise a warning is issued if the environment exceeds 10Mb in size. - version 1.3 introduced a mini bug in the SGE interface (via new list.jobs.cmd option). Fixed. - Memory usage is now stored in the database and can be queried using getJobInfo(). BatchJobs_1.3: - Reduced memory consumption and disk i/o if 'more.args' is used. - added option in Registry for absolute paths src.files and src.dirs - Added option "BatchJobs.verbose" for report generators like knitr - added "file.dir" option to batchMapQuick, removed option 'temporary' as this is now more flexible. - package 'methods' is always loaded on slaved to avoid problems with S4 code - some smaller fixes - in makeClusterFunctions* it is now to configurable via what command jobs are listed on the system - default for BatchJobs.check.posix is now: OS != Windows BatchJobs_1.2: - Fixed a bug where too many running jobs were queried on some Torque systems - Fixed a bug where chunking did not work with batchMapQuick - Fixed mail subjects to be more precise on job IDs - Fixed output of running time in testJob() - More robust error handling in syncRegistry - waitForJobs is now more conservative in waiting for termination of jobs. - New argument "impute.val" for reduceResults-family of functions: This allows to conveniently reduce results while some are missing - Experimental framework to support sourcing of multiple files and directories - testJob can now run jobs interactively in the current R session if argument "external" is set to TRUE. BatchJobs_1.1-1135: - Increased minor version number - Added support for SLURM schedulers - Function getErrors renamed to getErrorMessages due to conflicts with RUnit. Interface change: argument "print" dropped - Redefined return value of function "waitForJobs" which should now be much easier to understand. - New option "BatchJobs.check.posix" to enable more relaxed checks on file and directory names - Helper functions to simplify integration into other packages: getConfig, loadConfig and setConfig. showConf was replaced with a print generic. - Improved stability and speed of killJobs - Added a workaround to fix file URI handling on windows - Many minor bug and documentation fixes - The registry now supports an "exports" directory of exported objects, which are always loaded on the slaves, e.g. data objects which are needed in every job This directory can be either managed manually or - more conveniently - via the "fail" package on CRAN. - The registry now supports a list of source directories. All R files in these subdirectories are sourced on the slaves (and also on the master) Nice if you have a lot of helper code in seperate R files which is needed in your jobs. - batchMap and all result functions where this is reasonable support naming results with the names of the elements that were mapped over (see use.names argument) - batchMap now supports mapping over any kind of object (not only lists and vectors) that supports the "length" and "[" / "[["- index method. BatchJobs_1.0-966: - Dropped support for R < 2.13.0 (.find.packages deprecated) - speedups in reduceResults[ReturnValue] - Output of showStatus now contains a time information - fixed a bug in findErrors (argument 'ids' was ignored) - New function sweepRegistry: Remove intermediate/temporary/obsolete files - Better error handling in case of I/O errors - New functions: findStarted() and findNotStarted() BatchJobs_1.0-915: - new option to raise warnings to errors on the slaves - option to set process priority via nice in clusterFunctionsMulticore and clusterFunctionsSSH - new helper functions: getErrors, getJobInfo, loadConfig, callFunctionOnSSHWorkers, getSSHWorkersInfo, installPackagesOnSSHWorkers, waitForJobs, several more find* functions - removed helper getJobTimes -> this is included in getJobInfo - experimental file-based caching mechanism to speed up the processing of queries generated on computational nodes - various smaller fixes - documentation fixes BatchJobs_1.0-606: - many dependencies are now imports - many fixes and improvements for stability in cluster functions - helper functions for your own cluster functions are exported and documented, they all start with cf* - interactive mode now has dummy functions for listing and killing jobs - interactive mode now generates log files - job resources (from submits) are stored and can be queried on the master and the slaves (see getResources and getJobResources) - defaults for job resources can be specified in config file, see config part on web page - multicore and SSH mode now supports r.options, i.e. options to start Rscript and R CMD BATCH with - function grepLogs to search log files for a pattern BatchJobs_1.0-527: - very minor bug fixes: for argument conversion and checks BatchJobs_1.0-485: - First submit to CRAN. BatchJobs/R/0000755000176000001440000000000012502316642012343 5ustar ripleyusersBatchJobs/R/getJobInfo.R0000644000176000001440000000747712415510734014534 0ustar ripleyusersgetJobInfoInternal = function(reg, ids, select, unit = "seconds", columns) { if (!missing(ids)) ids = checkIds(reg, ids) assertChoice(unit, c("seconds", "minutes", "hours", "days", "weeks")) select.db = c("submitted", "started", "done", "done - started AS time_running", "memory", "started - submitted AS time_queued", "error", "node", "batch_job_id", "r_pid", "seed") select.cns = c("time.submitted", "time.started", "time.done", "time.running", "memory", "time.queued", "error.msg", "nodename", "batch.id", "r.pid", "seed") columns = c(columns, setNames(select.db, select.cns)) if (!missing(select)) { assertSubset(select, c("id", select.cns)) columns = columns[names(columns) %in% c("id", select)] } tab = setNames(dbGetExpandedJobsTable(reg, ids, columns), names(columns)) if (nrow(tab) == 0L) return(tab) # convert times to POSIX if (!is.null(tab$time.submitted)) tab$time.submitted = dbConvertNumericToPOSIXct(tab$time.submitted) if (!is.null(tab$time.started)) tab$time.started = dbConvertNumericToPOSIXct(tab$time.started) if (!is.null(tab$time.done)) tab$time.done = dbConvertNumericToPOSIXct(tab$time.done) # shorten error messages if (!is.null(tab$error.msg)) tab$error.msg = vcapply(tab$error.msg, clipString, len = 30L) # convert time diffs div = setNames(c(1L, 60L, 3600L, 86400L, 604800L), c("seconds", "minutes", "hours", "days", "weeks"))[unit] if (!is.null(tab$time.running)) tab$time.running = as.numeric(tab$time.running) / div if (!is.null(tab$time.queued)) tab$time.queued = as.numeric(tab$time.queued) / div return(tab) } #' Get computational information of jobs. #' #' error messages (shortened, see \code{\link{showLog}} for detailed error messages), #' Returns time stamps (submitted, started, done), time running, approximate memory usage (in Mb, see note) #' time in queue, hostname of the host the job was executed, #' assigned batch ID, the R PID and the seed of the job. #' #' @note To estimate memory usage the sum of the last column of \code{\link[base]{gc}} is used. #' #' @template arg_reg #' @template arg_ids #' @param pars [\code{logical(1)}]\cr #' Include job parameters in the output? #' Default is \code{FALSE}. #' @param prefix.pars [\code{logical(1)}]\cr #' Should a prefix be added to job parameter names (column names) to avoid name clashes? #' Default is \code{FALSE}. #' @param select [\code{character}]\cr #' Select only a subset of columns. #' Usually this is not required and you can subset yourself, #' but in some rare cases it may be advantageous to not query all information. #' Note that the column \dQuote{id} (job id) is always selected. #' If not provided, all columns are queried and returned. #' @param unit [\code{character(1)}]\cr #' Unit to convert execution and queing times to. #' Possible values: \dQuote{seconds}, \dQuote{minutes}, \dQuote{hours}, #' \dQuote{days} and \dQuote{weeks}. #' Default is \dQuote{seconds}. #' @return [\code{data.frame}]. #' @family debug #' @export getJobInfo = function(reg, ids, pars = FALSE, prefix.pars = FALSE, select, unit = "seconds") { UseMethod("getJobInfo") } #' @method getJobInfo Registry #' @export getJobInfo.Registry = function(reg, ids, pars = FALSE, prefix.pars = FALSE, select, unit = "seconds") { syncRegistry(reg) assertFlag(pars) columns = c(id = "job_id") if (pars) columns = c(columns, c(pars = "pars")) tab = getJobInfoInternal(reg, ids, select, unit, columns) # unserialize parameters if (pars && !is.null(tab$pars)) { pars = convertListOfRowsToDataFrame(lapply(tab$pars, function(x) unserialize(charToRaw(x)))) if (prefix.pars) names(pars) = sprintf("job.par.%s", names(pars)) tab = cbind(subset(tab, select = -pars), pars) } return(tab) } BatchJobs/R/buffer.R0000644000176000001440000000163512415510734013745 0ustar ripleyusers# a simple preallocated stack. buffer = function(type = "list", capacity = 0L, value = TRUE, init = NULL, ...) { if (is.null(init)) { st = vector(type, capacity) n = 0L } else { st = init n = length(init) } rm(type) force(capacity) force(value) rm(init) get = function() { head(st, n) } push = function(x) { if (n == capacity) clear() n <<- n + 1L st[[n]] <<- x } pop = function() { n <<- min(0L, n - 1L) } top = function() { if (n == 0L) return(vector(type, 0L)) st[[n]] } clear = function() { if (is.function(value)) ret = value(get(), ...) else ret = value n <<- 0L ret } pos = function() { n } empty = function() { n == 0L } full = function() { n == capacity } list(get = get, push = push, pop = pop, top = top, clear = clear, pos = pos, empty = empty, full = full) } BatchJobs/R/clusterFunctionsInteractive.R0000644000176000001440000000416212415510734020242 0ustar ripleyusers#' Create cluster functions for sequential execution in same session. #' #' @description #' All jobs executed under these cluster functions are executed #' sequentially, in the same interactive R process that you currently are. #' That is, \code{submitJob} does not return until the #' job has finished. The main use of this \code{ClusterFunctions} #' implementation is to test and debug programs on a local computer. #' #' Listing jobs returns an empty vector (as no jobs can be running when you call this) #' and \code{killJob} returns at once (for the same reason). #' #' @param write.logs [\code{logical(1)}]\cr #' Sink the output to log files. Turning logging off can increase the speed of #' calculations but makes it next to impossible to debug. #' Default is \code{TRUE}. #' @return [\code{\link{ClusterFunctions}}]. #' @family clusterFunctions #' @export makeClusterFunctionsInteractive = function(write.logs = TRUE) { assertFlag(write.logs) submitJob = if(write.logs) { function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { # open log file for writing fn = file(log.file, open = "wt") sink(fn, type = "output") sink(fn, type = "message") on.exit({ sink(NULL, type = "output") sink(NULL, type = "message") close(fn) }) # sink both output and message streams try(sys.source(rscript, envir = new.env(), keep.source = FALSE)) # return job result (always successful) makeSubmitJobResult(status = 0L, batch.job.id = "cfInteractive", msg = "") } } else { function(conf, reg, job.name, rscript, log.file, job.dir, resources) { suppressAll(try(sys.source(rscript, envir = new.env(), keep.source = FALSE))) makeSubmitJobResult(status = 0L, batch.job.id = "cfInteractive", msg = "") } } killJob = function(conf, reg, batch.job.id) NULL listJobs = function(conf, reg) integer(0L) getArrayEnvirName = function() NA_character_ makeClusterFunctions(name = "Interactive", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/clusterFunctions.R0000644000176000001440000001213612415510734016044 0ustar ripleyusers#' Create a SubmitJobResult object. #' #' @description #' Use this function in your implementation of \code{\link{makeClusterFunctions}} #' to create a return value for the \code{submitJob} function. #' @param status [\code{integer(1)}]\cr #' Launch status of job. #' 0 means success, codes bewteen 1 and 100 are temporary #' errors and any error greater than 100 is a permanent failure. #' @param batch.job.id [\code{character(1)}]\cr #' Unique id of this job on batch system. Note that this is not the usual job id used in BatchJobs! #' Must be globally unique so that the job can be terminated #' using just this information. #' @param msg [\code{character(1)}]\cr #' Optional error message in case \code{status} is not equal to 0. #' Default is \dQuote{OK}, \dQuote{TEMPERR}, \dQuote{ERROR}, depending on \code{status}. #' @param ... [\code{any}]\cr #' Currently unused. #' @return [\code{\link{SubmitJobResult}}]. A list, containing #' \code{status}, \code{batch.job.id} and \code{msg}. #' @export #' @aliases SubmitJobResult makeSubmitJobResult = function(status, batch.job.id, msg, ...) { if (missing(msg)) { msg = if (status == 0L) "OK" else if (status <= 100L) "TEMPERR" else "ERROR" } setClasses(list(status = status, batch.job.id = batch.job.id, msg = msg), "SubmitJobResult") } #' @export #' @method print SubmitJobResult print.SubmitJobResult = function(x, ...) { cat("Job submission result:\n") catf(" ID : '%s'", x$batch.job.id) catf(" Status : %i", x$status) catf(" Msg : %s", x$msg) } #' Create a ClusterFuntions object. #' #' Use this funtion when you implement a backend for a batch system. #' You must define the functions specified in the arguments. #' @param name [\code{character(1)}]\cr #' Name of cluster functions. #' @param submitJob [\code{function(conf, reg, job.name, rscript, log.file, job.dir, resources, ...)}]\cr #' Function to submit a new job. #' Must return a \code{\link{SubmitJobResult}} object.\cr #' The arguments are:\cr #' conf [\code{environment}]: The user configuration.\cr #' reg [\code{\link{Registry}}]: The registry.\cr #' job.name [\code{character(1)}]: Name of job, used if the job is displayed on the batch system. This is just for display and not an id!\cr #' rscript [\code{character(1)}]: File path to R script that is used to execute the job.\cr #' log.file [\code{character(1)}]: File path where log file (.Rout) has to be placed.\cr #' job.dir [\code{character(1)}]: Directory where all files relating to this job are placed.\cr #' resources [\code{list}]: Freely definable list of required resources for this job, e.g. walltime or memory. #' @param killJob [\code{function(conf, reg, batch.job.id)}]\cr #' Function to kill a job on the batch system. #' Make sure that you definately kill the job! #' Return value is currently ignored.\cr #' The arguments are:\cr #' conf [\code{environment}]: The user configuration.\cr #' reg [\code{\link{Registry}}]: The registry.\cr #' batch.job.id [\code{character(1)}]: Batch job id, as produced by \code{submitJob}.\cr #' Set \code{killJob} to \code{NULL} if killing jobs cannot be supported. #' @param listJobs [\code{function(conf, reg)}]\cr #' List all jobs on the batch system for the current user / registry. #' This includes queued, running, held, idle, etc. jobs. #' Must return an integer vector of batch job ids, same format as they are produced by \code{submitJob}. #' It does not matter if you return a few job ids too many (e.g. all for the current user instead #' of all for the current registry), but you have to include all relevant ones. #' The arguments are:\cr #' conf [\code{environment}]: The user configuration.\cr #' reg [\code{\link{Registry}}]: The registry.\cr #' Set \code{listJobs} to \code{NULL} if listing jobs cannot be supported. #' @param getArrayEnvirName [\code{function()}]\cr #' Returns the name of the environment variable specifying the array ID. #' Should return \code{NA} if not supported. #' @param class [\code{character(1)}]\cr #' Optional class name for cluster functions object. #' Useful to provide a nice print method #' which might show additional information about the workers. #' Default is \code{NULL}. #' @param ... [\code{any}]\cr #' Currently ignored. #' @export #' @aliases ClusterFunctions #' @family clusterFunctions makeClusterFunctions = function(name, submitJob, killJob, listJobs, getArrayEnvirName, class = NULL, ...) { assertString(name) assertFunction(submitJob, c("conf", "reg", "job.name", "rscript", "log.file", "job.dir", "resources")) if (!is.null(killJob)) assertFunction(killJob, c("conf", "reg", "batch.job.id")) if (!is.null(listJobs)) assertFunction(listJobs, c("conf", "reg")) if (!is.null(getArrayEnvirName)) assertFunction(getArrayEnvirName, character(0L)) setClasses(list(name = name, submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName), c("ClusterFunctions", class)) } #' @export #' @method print ClusterFunctions print.ClusterFunctions = function(x, ...) { catf("%s cluster functions.", x$name) } BatchJobs/R/Registry.R0000644000176000001440000002366712502316642014314 0ustar ripleyusersmakeRegistryInternal = function(id, file.dir, sharding, work.dir, multiple.result.files, seed, packages, src.dirs, src.files) { checkIdValid(id, allow.minus = FALSE) assertString(file.dir) checkDir(file.dir, create = TRUE, check.empty = TRUE, check.posix = TRUE, msg = TRUE) file.dir = sanitizePath(file.dir, make.absolute = TRUE) if (missing(work.dir)) work.dir = getwd() else assertString(work.dir) checkDir(work.dir, check.posix = TRUE) work.dir = sanitizePath(work.dir, make.absolute = TRUE) assertFlag(sharding) assertFlag(multiple.result.files) seed = if (missing(seed)) getRandomSeed() else asInt(seed) assertCharacter(packages, any.missing = FALSE) packages = union(packages, "BatchJobs") requirePackages(packages, stop = TRUE, suppress.warnings = TRUE, default.method = "attach") assertCharacter(src.dirs, any.missing = FALSE) src.dirs = sanitizePath(src.dirs, make.absolute = FALSE) assertCharacter(src.files, any.missing = FALSE) src.files = sanitizePath(src.files, make.absolute = FALSE) # make paths absolute to be sure. otherwise cfSSH wont work for example # also check the dirs # file dir # job dir job.dir = getJobParentDir(file.dir) checkDir(job.dir, create = TRUE, check.empty = TRUE) # fun dir fun.dir = getFunDir(file.dir) checkDir(fun.dir, create = TRUE, check.empty = TRUE) # resources, pending, exports, work.dir checkDir(getResourcesDir(file.dir), create = TRUE, check.empty = TRUE) checkDir(getPendingDir(file.dir), create = TRUE, check.empty = TRUE) checkDir(getExportDir(file.dir), create = TRUE, check.empty = TRUE) sourceRegistryFilesInternal(work.dir, src.dirs, src.files) packages = setNames(lapply(packages, function(pkg) list(version = packageVersion(pkg))), packages) conf = getConfig() setClasses(list( id = id, version = R.version, RNGkind = RNGkind(), db.driver = conf$db.driver, db.options = conf$db.options, seed = seed, file.dir = file.dir, sharding = sharding, work.dir = work.dir, src.dirs = src.dirs, src.files = src.files, multiple.result.files = multiple.result.files, packages = packages[order(names(packages))] ), "Registry") } #' Construct a registry object. #' #' Note that if you don't want links in your paths (\code{file.dir}, \code{work.dir}) to get resolved and have #' complete control over the way the path is used internally, pass an absolute path which begins with \dQuote{/}. #' #' Every object is a list that contains the passed arguments of the constructor. # #' @param id [\code{character(1)}]\cr #' Name of registry. Displayed e.g. in mails or in cluster queue. #' @param file.dir [\code{character(1)}]\cr #' Path where files regarding the registry / jobs should be saved. #' Default is \dQuote{-files} in current working directory if \code{id} is set. #' @param sharding [\code{logical(1)}]\cr #' Enable sharding to distribute result files into different subdirectories? #' Important if you have many experiments. #' Default is \code{TRUE}. #' @param work.dir [\code{character(1)}]\cr #' Working directory for R process when experiment is executed. #' Default is the current working directory when registry is created. #' @param multiple.result.files [\code{logical(1)}]\cr #' Should a result file be generated for every list element of the #' returned list of the job function? #' Note that the function provided to \code{\link{batchMap}} or #' \code{\link{batchReduce}} must return a named list if this is set to \code{TRUE}. #' The result file will be named \dQuote{-result-.RData} #' instead of \dQuote{-result.RData}. #' Default is \code{FALSE}. #' @param seed [\code{integer(1)}]\cr #' Start seed for experiments. The first experiment in the registry will use this #' seed, for the subsequent ones the seed is incremented by 1. #' Default is a random number from 1 to \code{.Machine$integer.max/2}. #' @param packages [\code{character}]\cr #' Packages that will always be loaded on each node. #' Default is \code{character(0)}. #' @param src.dirs [\code{character}]\cr #' Directories containing R scripts #' to be sourced on registry load (both on slave and master). #' Files not matching the pattern \dQuote{\\.[Rr]$} are ignored. #' Useful if you have many helper functions that are needed during the execution of your jobs. #' These files should only contain function definitions and no executable code. #' Default is \code{character(0)}. #' @param src.files [\code{character}]\cr #' R scripts files #' to be sourced on registry load (both on slave and master). #' Useful if you have many helper functions that are needed during the execution of your jobs. #' These files should only contain function and constant definitions and no long running, executable code. #' These paths are considered to be relative to your \code{work.dir}. #' As a last remedy in problematic cases you can use absolute paths, by passing paths that #' start with \dQuote{/}, see the comment about \code{file.dir} and \code{work.dir} above, #' where we allow the same thing. #' Note that this is a less portable approach and therefore usually a less good idea. #' Default is \code{character(0)}. #' @param skip [\code{logical(1)}]\cr #' Skip creation of a new registry if a registry is found in \code{file.dir}. #' Defaults to \code{TRUE}. #' @return [\code{\link{Registry}}] #' @aliases Registry #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' print(reg) makeRegistry = function(id, file.dir, sharding = TRUE, work.dir, multiple.result.files = FALSE, seed, packages = character(0L), src.dirs = character(0L), src.files = character(0L), skip = TRUE) { if (missing(file.dir)) file.dir = file.path(getwd(), paste0(id, "-files")) assertFlag(skip) if (skip && isRegistryDir(file.dir)) return(loadRegistry(file.dir = file.dir)) reg = makeRegistryInternal(id, file.dir, sharding, work.dir, multiple.result.files, seed, packages, src.dirs, src.files) dbCreateJobStatusTable(reg) dbCreateJobDefTable(reg) saveRegistry(reg) reg } #' @export print.Registry = function(x, ...) { cat("Job registry: ", x$id, "\n") cat(" Number of jobs: ", dbGetJobCount(x), "\n") cat(" Files dir:", x$file.dir, "\n") cat(" Work dir:", x$work.dir, "\n") cat(" Multiple result files:", x$multiple.result.files, "\n") cat(" Seed:", x$seed, "\n") cat(" Required packages:", collapse(names(x$packages), ", "), "\n") } #' @title Load a previously saved registry. #' #' @details #' Loads a previously created registry from the file system. #' The \code{file.dir} is automatically updated upon load, so be careful #' if you use the registry on multiple machines simultaneously, e.g. #' via sshfs or a samba share. #' #' @param file.dir [\code{character(1)}]\cr #' Location of the file.dir to load the registry from. #' @param work.dir [\code{character(1)}]\cr #' Location of the work. Unchanged if missing. #' @return [\code{\link{Registry}}]. #' @export loadRegistry = function(file.dir, work.dir) { fn = getRegistryFilePath(file.dir) if (!file.exists(fn)) stopf("No registry found in '%s'", file.dir) info("Loading registry: %s", fn) reg = load2(fn, "reg") requirePackages(names(reg$packages), why = sprintf("registry %s", reg$id), default.method = "attach") if (!isOnSlave()) { # FIXME: check that no jobs are running, if possible, before updating adjusted = adjustRegistryPaths(reg, file.dir, work.dir) if (!isFALSE(adjusted)) reg = adjusted updated = updateRegistry(reg) if (!isFALSE(updated)) reg = updated if (!isFALSE(adjusted) || !isFALSE(updated)) saveRegistry(reg) } loadExports(reg) sourceRegistryFiles(reg) return(reg) } saveRegistry = function(reg) { fn = getRegistryFilePath(reg$file.dir) info("Saving registry: %s", fn) save(file = fn, reg) reg } isRegistryDir = function(dir) { isDirectory(dir) && file.exists(getRegistryFilePath(dir)) } checkRegistry = function(reg, strict = FALSE) { cl = class(reg) expected = "Registry" if (strict) { if (head(cl, 1L) != expected) stopf("Registry class mismatch: Expected argument with first class '%s'", expected) } else { if (expected %nin% cl) stopf("Registry class mismatch: Expected argument of class '%s'", expected) } invisible(TRUE) } #' Remove a registry object. #' #' If there are no live/running jobs, the registry will be closed #' and all of its files will be removed from the file system. #' If there are live/running jobs, an informative error is generated. #' The default is to prompt the user for confirmation. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ask [\code{character(1)}]\cr #' If \code{"yes"} the user is prompted to confirm the action. #' If trying to prompt the user this way in a non-interactive #' session, then an informative error is generated. #' If \code{"no"}, the registry will be removed without #' further confirmation. #' #' @return [\code{logical[1]}] #' #' @export removeRegistry = function(reg, ask = c("yes", "no")) { ask = match.arg(ask) if (ask == "yes") { if (!interactive()) stopf("removeRegistry(..., ask = \"yes\") only works in interactive sessions.") prompt = sprintf("Are you sure you wish to delete BatchJobs registry '%s' and all of it's files in directory '%s'? [y/N]: ", reg$id, reg$file.dir) ans = 2L repeat { ans = tolower(readline(prompt)) ans = gsub("[ ]", "", ans) if (ans == "") ans = "no" ans = pmatch(ans, table=c("yes", "no"), nomatch=0L) if (ans > 0L) break } if (ans != 1L) return(invisible(FALSE)) } checkRegistry(reg) syncRegistry(reg) running = findOnSystem(reg) if (length(running) > 0L) stopf("Can't remove registry, because there are %d live jobs on the system.", length(running)) ## FIXME: Close database first? removeDirs(reg$file.dir, recursive=TRUE, must.work=TRUE) } BatchJobs/R/filterResults.R0000644000176000001440000000222612415510734015340 0ustar ripleyusers#' Find all results where a specific condition is true. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs whose results you want to test for the condition. #' Default is all jobs for which results are available. #' @param fun [\code{fun(job, res)}]\cr #' Predicate function that returns \code{TRUE} or \code{FALSE}. #' @param ... [any]\cr #' Additional arguments to \code{fun}. #' @return [\code{integer}]. Ids of jobs where \code{fun(job, result)} returns \code{TRUE}. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:10) #' submitJobs(reg) #' waitForJobs(reg) #' #' # which square numbers are even: #' filterResults(reg, fun = function(job, res) res %% 2 == 0) filterResults = function(reg, ids, fun, ...) { checkRegistry(reg) syncRegistry(reg) assertFunction(fun, c("job", "res")) if (missing(ids)) ids = dbFindDone(reg) else ids = checkIds(reg, ids) Filter(function(id) { fun(job = getJob(reg, id, check.id = FALSE), res = getResult(reg, id), ...) }, ids) } BatchJobs/R/reduceResults.R0000644000176000001440000002044712415510734015327 0ustar ripleyusers#' Reduce results from result directory. #' #' @description #' The following functions provide ways to reduce result files into either specific R objects (like #' vectors, lists, matrices or data.frames) or to arbitrarily aggregate them, which is a more general #' operation. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of selected jobs. #' Default is all jobs for which results are available. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param fun [\code{function}]\cr #' For \code{reduceResults}, a function \code{function(aggr, job, res, ...)} to reduce things, #' for all others, a function \code{function(job, res, ...)} to select stuff. #' Here, \code{job} is the current job descriptor (see \code{\link{Job}}), \code{result} is the current result object and #' \code{aggr} are the so far aggregated results. When using \code{reduceResults}, #' your function should add the stuff you want to have from \code{job} and #' \code{result} to \code{aggr} and return that. #' When using the other reductions, you should select the stuff you want to have from \code{job} and #' \code{result} and return something that can be coerced to an element of the selected return data structure #' (reasonable conversion is tried internally). #' Default behavior for this argument is to return \code{res}, except for \code{reduceResults} where no #' default is available. #' @param init [\code{ANY}]\cr #' Initial element, as used in \code{\link{Reduce}}. #' Default is first result. #' @param ... [any]\cr #' Additional arguments to \code{fun}. #' @param use.names [\code{character(1)}]\cr #' Name the results with job ids (\dQuote{ids}), stored job names (\dQuote{names}) #' or return a unnamed result (\dQuote{none}). #' Default is \code{ids}. #' @param impute.val [any]\cr #' For \code{reduceResults}: If not missing, the value of \code{impute.val} is passed to function \code{fun} #' as argument \code{res} for jobs with missing results.\cr #' For the specialized reduction functions \code{reduceResults[Type]}: If not missing, \code{impute.val} is #' used as a replacement for the return value of \code{fun} on missing results. #' @template arg_progressbar #' @param rows [\code{logical(1)}]\cr #' Should the selected vectors be used as rows (or columns) in the result matrix? #' Default is \code{TRUE}. #' @param strings.as.factors [\code{logical(1)}] #' Should all character columns in result be converted to factors? #' Default is \code{default.stringsAsFactors()}. #' @return Aggregated results, return type depends on function. If \code{ids} is empty: \code{reduceResults} #' returns \code{init} (if available) or \code{NULL}, \code{reduceResultsVector} returns \code{c()}, #' \code{reduceResultsList} returns \code{list()}, \code{reduceResultsMatrix} returns \code{matrix(0,0,0)}, #' \code{reduceResultsDataFrame} returns \code{data.frame()}. #' @export #' @examples #' # generate results: #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:5) #' submitJobs(reg) #' waitForJobs(reg) #' #' # reduce results to a vector #' reduceResultsVector(reg) #' # reduce results to sum #' reduceResults(reg, fun = function(aggr, job, res) aggr+res) #' #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) list(a = x, b = as.character(2*x), c = x^2) #' batchMap(reg, f, 1:5) #' submitJobs(reg) #' waitForJobs(reg) #' #' # reduce results to a vector #' reduceResultsVector(reg, fun = function(job, res) res$a) #' reduceResultsVector(reg, fun = function(job, res) res$b) #' # reduce results to a list #' reduceResultsList(reg) #' # reduce results to a matrix #' reduceResultsMatrix(reg, fun = function(job, res) res[c(1,3)]) #' reduceResultsMatrix(reg, fun = function(job, res) c(foo = res$a, bar = res$c), rows = TRUE) #' reduceResultsMatrix(reg, fun = function(job, res) c(foo = res$a, bar = res$c), rows = FALSE) #' # reduce results to a data.frame #' print(str(reduceResultsDataFrame(reg))) #' # reduce results to a sum #' reduceResults(reg, fun = function(aggr, job, res) aggr+res$a, init = 0) reduceResults = function(reg, ids, part = NA_character_, fun, init, impute.val, progressbar = TRUE, ...) { checkRegistry(reg) syncRegistry(reg) if (missing(ids)) { ids = done = dbFindDone(reg) with.impute = FALSE } else { ids = checkIds(reg, ids) done = dbFindDone(reg, ids) with.impute = !missing(impute.val) if (!with.impute) { if (length(ids) > length(done)) stopf("No results available for jobs with ids: %s", collapse(setdiff(ids, done))) } } assertFunction(fun, c("aggr", "job", "res")) assertFlag(progressbar) n = length(ids) info("Reducing ", n, " results...") if (n == 0L) { if (missing(init)) return(NULL) return(init) } bar = getProgressBar(progressbar, max = n, label = "reduceResults") tryCatch({ if (missing(init)) { # fetch first result as init aggr = getResult(reg, ids[1L], part) ids = tail(ids, -1L) bar$inc(1L) } else { aggr = init } for (id in ids) { # use lazy evaluation aggr = fun(aggr, job = dbGetJobs(reg, id)[[1L]], res = if (with.impute && id %nin% done) impute.val else getResult(reg, id, part), ...) bar$inc(1L) } }, error = bar$error) return(aggr) } #' @export #' @rdname reduceResults reduceResultsList = function(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val, progressbar = TRUE) { checkRegistry(reg) syncRegistry(reg) if (missing(ids)) { ids = done = dbFindDone(reg) with.impute = FALSE } else { ids = checkIds(reg, ids) done = dbFindDone(reg, ids) with.impute = !missing(impute.val) if (!with.impute) { if (length(ids) > length(done)) stopf("No results available for jobs with ids: %s", collapse(setdiff(ids, done))) } } if (missing(fun)) fun = function(job, res) res else assertFunction(fun, c("job", "res")) use.names = convertUseNames(use.names) assertFlag(progressbar) n = length(ids) info("Reducing %i results...", n) if (n == 0L) return(list()) res = vector("list", n) if (with.impute) { res = replace(res, ids %nin% done, impute.val) it = match(done, ids) } else { it = seq_len(n) } bar = getProgressBar(progressbar, max = n, label = "reduceResults") tryCatch({ for (i in it) { # use lazy evaluation! tmp = fun(job = dbGetJobs(reg, ids[i])[[1L]], res = getResult(reg, ids[i], part), ...) if (!is.null(tmp)) res[[i]] = tmp bar$inc(1L) } }, error = bar$error) names(res) = switch(use.names, "none" = NULL, "ids" = as.character(ids), "names" = dbGetJobNames(reg, ids)) return(res) } #' @export #' @rdname reduceResults reduceResultsVector = function(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val) { unlist(reduceResultsList(reg, ids, part, fun, ..., use.names = use.names, impute.val = impute.val)) } #' @export #' @rdname reduceResults reduceResultsMatrix = function(reg, ids, part = NA_character_, fun, ..., rows = TRUE, use.names = "ids", impute.val) { assertFlag(rows) use.names = convertUseNames(use.names) res = reduceResultsList(reg, ids, part, fun, ..., use.names = use.names, impute.val = impute.val) if (length(res) == 0L) return(matrix(0, nrow = 0L, ncol = 0L)) n = length(res) dn = if (use.names != "none") list(names(res), names(res[[1L]])) else NULL res = unlist(res, use.names = FALSE) if (rows) matrix(res, nrow = n, byrow = TRUE, dimnames = dn) else matrix(res, ncol = n, byrow = FALSE, dimnames = rev(dn)) } #' @export #' @rdname reduceResults reduceResultsDataFrame = function(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val, strings.as.factors = default.stringsAsFactors()) { assertFlag(strings.as.factors) res = reduceResultsList(reg, ids, part, fun, ..., use.names = use.names, impute.val = impute.val) if (!length(res)) return(data.frame()) convertListOfRowsToDataFrame(res, strings.as.factors = strings.as.factors) } BatchJobs/R/killJobs.R0000644000176000001440000001031312437616146014246 0ustar ripleyusers#' Kill some jobs on the batch system. #' #' @description #' Kill jobs which have already been submitted to the batch system. #' If a job is killed its internal state is reset as if it had not been submitted at all. #' #' The function informs if #' (a) the job you want to kill has not been submitted, #' (b) the job has already terminated, #' (c) for some reason no batch job id is available. #' In all 3 cases above, nothing is changed for the state of this job and no call #' to the internal kill cluster function is generated. #' #' In case of an error when killing, the function tries - after a short sleep - to kill the remaining #' batch jobs again. If this fails again for some jobs, the function gives up. Only jobs that could be #' killed are reset in the DB. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs to kill. #' Default is none. #' @template arg_progressbar #' @return [\code{integer}]. Ids of killed jobs. #' @export #' @family debug #' @examples #' \dontrun{ #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) Sys.sleep(x) #' batchMap(reg, f, 1:10 + 5) #' submitJobs(reg) #' waitForJobs(reg) #' #' # kill all jobs currently _running_ #' killJobs(reg, findRunning(reg)) #' # kill all jobs queued or running #' killJobs(reg, findNotTerminated(reg)) #' } killJobs = function(reg, ids, progressbar = TRUE) { checkRegistry(reg) syncRegistry(reg) if (missing(ids)) return(invisible(integer(0L))) else ids = checkIds(reg, ids) assertFlag(progressbar) conf = getBatchJobsConf() killfun = getKillJob("Cannot kill jobs") # FIXME select and order (see below) could be done more efficiently in SQLite data = dbGetJobStatusTable(reg, ids = dbFindOnSystem(reg, ids), cols = c("job_id", "batch_job_id", "submitted", "started", "done", "error")) # print first summary information on jobs to kill info("Trying to kill %i jobs.", length(ids)) info("Jobs on system: %i", nrow(data)) info("Of these: %i not submitted, %i with no batch.job.id, %i already terminated", sum(is.na(data$submitted)), sum(is.na(data$batch_job_id)), sum(!is.na(data$done) | !is.na(data$error))) # subset data: restrict to jobs submitted, not done, no error, has bji # other jobs can be ignored -> overwrite ids data = subset(data, !is.na(data$submitted) & is.na(data$done) & is.na(data$error) & !is.na(data$batch_job_id), select = c("job_id", "batch_job_id", "started")) # kill queued jobs first, otherwise they might get started while killing running jobs data = data[order(data$started, na.last = FALSE),, drop = FALSE] ids = data$job_id bjids = unique(data$batch_job_id) info("Killing real batch jobs: %i", length(bjids)) if (length(bjids) == 0L) { info("No batch jobs to kill.") return(invisible(integer(0L))) } doKill = function(bjids) { n = length(bjids) old.warn = getOption("warn") bar = getProgressBar(progressbar, min = 0L, max = n, label = "killJobs") on.exit({ options(warn = old.warn); bar$kill() }) options(warn = 0L) bar$set() notkilled = logical(n) for (i in seq_len(n)) { ok = try(killfun(conf, reg, bjids[i])) if (is.error(ok)) { notkilled[i] = TRUE warning(as.character(ok)) } bar$set(i) } bjids[notkilled] } # first try to kill info(clipString(collapse(bjids), 200L, ",...")) bjids.notkilled = doKill(bjids) # second try to kill if (length(bjids.notkilled) > 0L) { info("Could not kill %i batch jobs, trying again.", length(bjids.notkilled)) Sys.sleep(2) bjids.notkilled = doKill(bjids.notkilled) } # second try also not successful if (length(bjids.notkilled) > 0L) { fn = file.path(reg$file.dir, sprintf("killjobs_failed_ids_%i", now())) warningf("Could not kill %i batch jobs, kill them manually!\nTheir ids have been saved in %s.", length(bjids.notkilled), fn) writeLines(as.character(bjids.notkilled), con = fn) } # reset killed jobs ids = ids[data$batch_job_id %nin% bjids.notkilled] info("Resetting %i jobs in DB.", length(ids)) dbSendMessage(reg, dbMakeMessageKilled(reg, ids, type = "last")) invisible(ids) } BatchJobs/R/database.R0000644000176000001440000004425512502316642014244 0ustar ripleyusers############################################ ### Common database functions ############################################ sqlQuote = function(x) { sprintf("'%s'", x) } dbGetConnection = function(drv, reg, ...) { # method dispatch to support different DBMS UseMethod("dbGetConnection") } dbGetConnection.SQLiteDriver = function(drv, reg, flags = "ro", ...) { flags = switch(flags, "ro" = SQLITE_RO, "rw" = SQLITE_RW, "rwc" = SQLITE_RWC) opts = list(dbname = file.path(reg$file.dir, "BatchJobs.db"), flags = flags, drv = drv) con = do.call(dbConnect, args = c(dropNamed(reg$db.options, "pragmas"), opts)) for (pragma in reg$db.options$pragmas) dbClearResult(dbSendQuery(con, sprintf("PRAGMA %s", pragma))) return(con) } dbConnectToJobsDB = function(reg, flags = "ro") { drv = do.call(reg$db.driver, list()) dbGetConnection(drv, reg, flags) } dbDoQueries = function(reg, queries, flags = "ro", max.retries = 100L, sleep = function(r) 1.025^r) { for (i in seq_len(max.retries)) { con = try(dbConnectToJobsDB(reg, flags), silent = TRUE) if (is.error(con)) { if (!grepl("(lock|i/o|readonly)", tolower(con))) stopf("Error while etablishing the connection: %s", as.character(con)) } else { ok = try ({ dbBegin(con) ress = lapply(queries, dbGetQuery, con = con) }, silent = TRUE) if (!is.error(ok)) { # this can fail because DB is locked ok2 = dbCommit(con) if (ok2) { dbDisconnect(con) return(ress) } else { dbRollback(con) dbDisconnect(con) } } else { ok = as.character(ok) dbRollback(con) dbDisconnect(con) # catch known temporary errors: # - database is still locked # - disk I/O error # - disk I/O error # - database is only readable if(!grepl("(lock|i/o|readonly)", tolower(ok))) stopf("Error in dbDoQueries. Displaying only 1st query. %s (%s)", ok, queries[1L]) } } # if we reach this here, DB was locked or temporary I/O error Sys.sleep(runif(1L, min = 1, max = sleep(i))) } stopf("dbDoQueries: max retries (%i) reached, database is still locked!", max.retries) } dbDoQuery = function(reg, query, flags = "ro", max.retries = 100L, sleep = function(r) 1.025^r) { for (i in seq_len(max.retries)) { con = try(dbConnectToJobsDB(reg, flags), silent = TRUE) if (is.error(con)) { if (!grepl("(lock|i/o|readonly)", tolower(con))) stopf("Error while etablishing the connection: %s", as.character(con)) } else { res = try(dbGetQuery(con, query), silent = TRUE) dbDisconnect(con) if (!is.error(res)) return(res) res = as.character(res) if(!grepl("(lock|i/o|readonly)", tolower(res))) { stopf("Error in dbDoQuery. %s (%s)", res, query) } } # if we reach this here, DB was locked or temporary I/O error Sys.sleep(runif(1L, min = 1, max = sleep(i))) } stopf("dbDoQuery: max retries (%i) reached, database is still locked!", max.retries) } dbAddData = function(reg, tab, data) { query = sprintf("INSERT INTO %s_%s (%s) VALUES(%s)", reg$id, tab, collapse(colnames(data)), collapse(rep.int("?", ncol(data)))) con = dbConnectToJobsDB(reg, flags = "rw") on.exit(dbDisconnect(con)) dbBegin(con) ok = try(dbGetPreparedQuery(con, query, bind.data = data)) if(is.error(ok)) { dbRollback(con) stopf("Error in dbAddData: %s", as.character(ok)) } dbCommit(con) as.integer(dbGetQuery(con, "SELECT total_changes()")) } dbSelectWithIds = function(reg, query, ids, where = TRUE, group.by, limit = NULL, reorder = TRUE) { if(!missing(ids)) query = sprintf("%s %s job_id IN (%s)", query, ifelse(where, "WHERE", "AND"), collapse(ids)) if(!missing(group.by)) query = sprintf("%s GROUP BY %s", query, collapse(group.by)) if(!is.null(limit)) query = sprintf("%s LIMIT %i", query, limit) res = dbDoQuery(reg, query) if(missing(ids) || !reorder) return(res) return(res[na.omit(match(ids, res$job_id)),, drop = FALSE]) } ############################################ ### CREATE ############################################ #' ONLY FOR INTERNAL USAGE. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @return Nothing. #' @keywords internal #' @export dbCreateJobDefTable = function(reg) { UseMethod("dbCreateJobDefTable") } #' @export dbCreateJobDefTable.Registry = function(reg) { query = sprintf("CREATE TABLE %s_job_def (job_def_id INTEGER PRIMARY KEY, fun_id TEXT, pars TEXT, jobname TEXT)", reg$id) dbDoQuery(reg, query, flags = "rwc") dbCreateExpandedJobsView(reg) } dbCreateJobStatusTable = function(reg, extra.cols = "", constraints = "") { query = sprintf(paste("CREATE TABLE %s_job_status (job_id INTEGER PRIMARY KEY, job_def_id INTEGER,", "first_job_in_chunk_id INTEGER, seed INTEGER, resources_timestamp INTEGER, memory REAL, submitted INTEGER,", "started INTEGER, batch_job_id TEXT, node TEXT, r_pid INTEGER,", "done INTEGER, error TEXT %s %s)"), reg$id, extra.cols, constraints) dbDoQuery(reg, query, flags = "rwc") query = sprintf("CREATE INDEX job_def_id ON %s_job_status(job_def_id)", reg$id) dbDoQuery(reg, query, flags = "rw") return(invisible(TRUE)) } dbCreateExpandedJobsView = function(reg) { query = sprintf("CREATE VIEW %1$s_expanded_jobs AS SELECT * FROM %1$s_job_status LEFT JOIN %1$s_job_def USING(job_def_id)", reg$id) dbDoQuery(reg, query, flags = "rw") } ############################################ ### SELECT ############################################ #' ONLY FOR INTERNAL USAGE. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of selected jobs. #' @return [list of \code{\link{Job}}]. Retrieved jobs from DB. #' @keywords internal #' @export dbGetJobs = function(reg, ids) { UseMethod("dbGetJobs") } # note that this does not load the job function from disk to increase speed #' @method dbGetJobs Registry #' @export dbGetJobs.Registry = function(reg, ids) { query = sprintf("SELECT job_id, fun_id, pars, jobname, seed FROM %s_expanded_jobs", reg$id) tab = dbSelectWithIds(reg, query, ids) lapply(seq_row(tab), function(i) { makeJob(id = tab$job_id[i], fun.id = tab$fun_id[i], fun = NULL, pars = unserialize(charToRaw(tab$pars[i])), name = tab$jobname[i], seed = tab$seed[i]) }) } dbGetExpandedJobsTable = function(reg, ids, cols = "*") { # Note: job_id must be in cols! query = sprintf("SELECT %s FROM %s_expanded_jobs", collapse(cols), reg$id) tab = dbSelectWithIds(reg, query, ids) setRowNames(tab, tab$job_id) } dbGetJobStatusTable = function(reg, ids, cols = "*") { # Note: job_id must be in cols! query = sprintf("SELECT %s FROM %s_job_status", collapse(cols), reg$id) tab = dbSelectWithIds(reg, query, ids) setRowNames(tab, tab$job_id) } dbGetJobCount = function(reg) { query = sprintf("SELECT COUNT(*) AS count FROM %s_job_status", reg$id) dbDoQuery(reg, query)$count } dbGetJobId = function(reg) { query = sprintf("SELECT job_id FROM %s_job_status LIMIT 1", reg$id) as.integer(dbDoQuery(reg, query)$job_id) } dbGetJobIds = function(reg) { query = sprintf("SELECT job_id FROM %s_job_status", reg$id) dbDoQuery(reg, query)$job_id } dbCheckJobIds = function(reg, ids) { not.found = setdiff(ids, dbGetJobIds(reg)) if (length(not.found) > 0L) stopf("Ids not present in registry: %s", collapse(not.found)) } dbGetJobIdsIfAllDone = function(reg) { query = sprintf("SELECT job_id, done FROM %s_job_status", reg$id) res = dbDoQuery(reg, query) if (all(! is.na(res$done))) return(res$job_id) stop("Not all jobs finished (yet)!") } dbGetLastAddedIds = function(reg, tab, id.col, n) { query = sprintf("SELECT %s AS id_col FROM %s_%s ORDER BY %s DESC LIMIT %i", id.col, reg$id, tab, id.col, n) rev(dbDoQuery(reg, query)$id_col) } dbFindDone = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (done IS NOT NULL)", reg$id, if(negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindErrors = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (error IS NOT NULL)", reg$id, if(negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindTerminated = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (done IS NOT NULL OR error IS NOT NULL)", reg$id, if(negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindSubmitted = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (submitted IS NOT NULL)", reg$id, if (negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindStarted = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (started IS NOT NULL)", reg$id, if (negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindOnSystem = function(reg, ids, negate = FALSE, limit = NULL, batch.ids) { if (missing(batch.ids)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (batch_job_id IN (%s))", reg$id, if (negate) "NOT" else "", collapse(sqlQuote(batch.ids))) dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindSubmittedNotTerminated = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (submitted IS NOT NULL AND done IS NULL AND error IS NULL)", reg$id, if (negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindRunning = function(reg, ids, negate = FALSE, limit = NULL, batch.ids) { if (missing(batch.ids)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (batch_job_id IN (%s) AND started IS NOT NULL AND done IS NULL AND error IS NULL)", reg$id, if (negate) "NOT" else "", collapse(sqlQuote(batch.ids))) dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindExpiredJobs = function(reg, ids, negate = FALSE, limit = NULL, batch.ids) { if (missing(batch.ids)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") # started, not terminated, not running query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (started IS NOT NULL AND done IS NULL AND error is NULL AND batch_job_id NOT IN (%s))", reg$id, if (negate) "NOT" else "", collapse(sqlQuote(batch.ids))) dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindDisappeared = function(reg, ids, negate = FALSE, limit = NULL, batch.ids) { if (missing(batch.ids)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (submitted IS NOT NULL AND started IS NULL AND batch_job_id NOT IN (%s))", reg$id, if (negate) "NOT" else "", collapse(sqlQuote(batch.ids))) dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbGetFirstJobInChunkIds = function(reg, ids){ query = sprintf("SELECT job_id, first_job_in_chunk_id FROM %s_job_status", reg$id) dbSelectWithIds(reg, query, ids)$first_job_in_chunk_id } dbGetErrorMsgs = function(reg, ids, filter = FALSE, limit = NULL) { query = sprintf("SELECT job_id, error from %s_job_status", reg$id) if (filter) query = sprintf("%s WHERE error IS NOT NULL", query) dbSelectWithIds(reg, query, ids, where = !filter, limit = limit) } dbGetStats = function(reg, ids, running = FALSE, expired = FALSE, times = FALSE, batch.ids) { cols = c(n = "COUNT(job_id)", submitted = "COUNT(submitted)", started = "COUNT(started)", done = "COUNT(done)", error = "COUNT(error)", running = "NULL", expired = "NULL", t_min = "NULL", t_avg = "NULL", t_max = "NULL") if (missing(batch.ids) && (expired || running)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") if(running) cols["running"] = sprintf("SUM(started IS NOT NULL AND done IS NULL AND error IS NULL AND batch_job_id IN (%s))", collapse(sqlQuote(batch.ids))) if(expired) cols["expired"] = sprintf("SUM(started IS NOT NULL AND done IS NULL AND error IS NULL AND batch_job_id NOT IN (%s))", collapse(sqlQuote(batch.ids))) if (times) cols[c("t_min", "t_avg", "t_max")] = c("MIN(done - started)", "AVG(done - started)", "MAX(done - started)") query = sprintf("SELECT %s FROM %s_job_status", collapse(paste(cols, "AS", names(cols)), sep = ", "), reg$id) df = dbSelectWithIds(reg, query, ids, reorder = FALSE) # Convert to correct type. Null has no type and casts tend to not work properly with RSQLite x = c("n", "submitted", "started", "done", "error", "running", "expired") df[x] = lapply(df[x], as.integer) x = c("t_min", "t_avg", "t_max") df[x] = lapply(df[x], as.double) df } dbGetJobNames = function(reg, ids) { query = sprintf("SELECT job_id, jobname FROM %s_expanded_jobs", reg$id) as.character(dbSelectWithIds(reg, query, ids)$jobname) } dbMatchJobNames = function(reg, ids, jobnames) { query = sprintf("SELECT job_id FROM %s_expanded_jobs WHERE jobname IN (%s)", reg$id, collapse(sqlQuote(jobnames))) dbSelectWithIds(reg, query, ids, where = FALSE)$job_id } ############################################ ### DELETE ############################################ dbRemoveJobs = function(reg, ids) { query = sprintf("DELETE FROM %s_job_status WHERE job_id IN (%s)", reg$id, collapse(ids)) dbDoQuery(reg, query, flags = "rw") query = sprintf("DELETE FROM %1$s_job_def WHERE job_def_id NOT IN (SELECT DISTINCT job_def_id FROM %1$s_job_status)", reg$id) dbDoQuery(reg, query, flags = "rw") return(invisible(TRUE)) } ############################################ ### Messages ############################################ dbSendMessage = function(reg, msg, staged = useStagedQueries(), fs.timeout = NA_real_) { if (staged) { fn = getPendingFile(reg, msg$type, msg$ids[1L]) writeSQLFile(msg$msg, fn) waitForFiles(fn, timeout = fs.timeout) } else { dbDoQuery(reg, msg$msg, flags = "rw") } } dbSendMessages = function(reg, msgs, max.retries = 200L, sleep = function(r) 1.025^r, staged = useStagedQueries(), fs.timeout = NA_real_) { if (length(msgs) == 0L) return(TRUE) if (staged) { chars = .OrderChars # reorder messages in sublist msgs = split(msgs, extractSubList(msgs, "type")) msgs = msgs[order(match(names(msgs), names(chars)))] fns = vcapply(msgs, function(cur) { first = cur[[1L]] fn = getPendingFile(reg, first$type, first$ids[1L], chars[first$type]) writeSQLFile(extractSubList(cur, "msg"), fn) fn }) waitForFiles(fns, timeout = fs.timeout) } else { ok = try(dbDoQueries(reg, extractSubList(msgs, "msg"), flags = "rw", max.retries, sleep)) if (is.error(ok)) { ok = as.character(ok) if (ok == "dbDoQueries: max retries reached, database is still locked!") { return(FALSE) } else { # throw exception again stopf("Error in dbSendMessages: %s", ok) } } } return(TRUE) } dbMakeMessageSubmitted = function(reg, job.ids, time = now(), batch.job.id, first.job.in.chunk.id = NULL, resources.timestamp, type = "submitted") { if(is.null(first.job.in.chunk.id)) first.job.in.chunk.id = "NULL" updates = sprintf("first_job_in_chunk_id=%s, submitted=%i, batch_job_id='%s', resources_timestamp=%i", first.job.in.chunk.id, time, batch.job.id, resources.timestamp) list(msg = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbMakeMessageStarted = function(reg, job.ids, time = now(), type = "started") { node = gsub("'", "\"", Sys.info()["nodename"], fixed = TRUE) updates = sprintf("started=%i, node='%s', r_pid=%i, error=NULL, done=NULL", time, node, Sys.getpid()) list(msg = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbMakeMessageError = function(reg, job.ids, err.msg, memory = -1, type = "error") { # FIXME how to escape ticks (')? Just replaced with double quotes for the moment err.msg = gsub("'", "\"", err.msg, fixed = TRUE) err.msg = gsub("[^[:print:]]", " ", err.msg) updates = sprintf("error='%s', done=NULL, memory='%.4f'", err.msg, memory) list(msg = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbMakeMessageDone = function(reg, job.ids, time = now(), memory = -1, type = "done") { updates = sprintf("done=%i, error=NULL, memory='%.04f'", time, memory) list(msg = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbMakeMessageKilled = function(reg, job.ids, type = "last") { updates = "resources_timestamp=NULL, memory=NULL, submitted=NULL, started=NULL, batch_job_id=NULL, node=NULL, r_pid=NULL, done=NULL, error=NULL" list(msgs = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbConvertNumericToPOSIXct = function(x) { now = Sys.time() as.POSIXct(x, origin = now - as.integer(now)) } dbSetJobFunction = function(reg, ids, fun.id) { query = sprintf("UPDATE %1$s_job_def SET fun_id = '%2$s' WHERE job_def_id IN (SELECT job_def_id FROM %1$s_job_status WHERE job_id IN (%3$s))", reg$id, fun.id, collapse(ids)) dbDoQuery(reg, query, flags = "rw") } dbSetJobNames = function(reg, ids, jobnames) { queries = sprintf("UPDATE %1$s_job_def SET jobname = '%2$s' WHERE job_def_id IN (SELECT job_def_id FROM %1$s_job_status WHERE job_id IN (%3$i))", reg$id, jobnames, ids) dbDoQueries(reg, queries, flags = "rw") } BatchJobs/R/writeRscripts.R0000644000176000001440000000265212423457121015357 0ustar ripleyuserswriteRscripts = function(reg, cf, ids, chunks.as.arrayjobs, resources.timestamp, disable.mail, delays) { template = paste( "Sys.sleep(%%f)", "options(BatchJobs.on.slave = TRUE, BatchJobs.resources.path = '%s')", "library(checkmate)", "library(BatchJobs)", "res = BatchJobs:::doJob(", "\treg = loadRegistry('%s'),", "\tids = c(%%s),", "\tmultiple.result.files = %s,", "\tdisable.mail = %s,", "\tfirst = %iL,", "\tlast = %iL,", "\tarray.id = %s)", "BatchJobs:::setOnSlave(FALSE)", sep = "\n") fids = viapply(ids, head, 1L) # first job id in chunk first = head(fids, 1L) last = tail(fids, 1L) ids = vcapply(ids, function(id) collapse(paste0(id, "L"))) # ids as collapsed strings # print the constant arguments (of length 1) into the template resources.path = getResourcesFilePath(reg, resources.timestamp) array.str = if (chunks.as.arrayjobs) sprintf("Sys.getenv(\"%s\", NA)", cf$getArrayEnvirName()) else NA template = sprintf(template, resources.path, reg$file.dir, reg$multiple.result.files, disable.mail, first, last, array.str) # print delays and ids into template. sprintf will return a string of length length(delays) == length(ids) # put this together with file names into an mapply on cat. files = getRScriptFilePath(fids, reg = reg) mapply(FUN = cat, SIMPLIFY = FALSE, USE.NAMES = FALSE, sprintf(template, delays, ids), file = files) invisible(files) } BatchJobs/R/callFunctionOnSSHWorkers.R0000644000176000001440000001462012415510734017343 0ustar ripleyusers#' Call an arbitrary function on specified SSH workers. #' #' @description #' Calls can be made in parallel or consecutively, #' the function waits until all calls have finished and #' returns call results. #' In consecutive mode the output on the workers can also be shown on the master #' during computation. #' #' Please read and understand the comments for argument \code{dir}. #' #' Note that this function should only be used for short administrative #' tasks or information gathering on the workers, the true work horse for #' real computation is \code{\link{submitJobs}}. #' #' In \code{\link{makeSSHWorker}} various options for load #' management are possible. Note that these will be #' ignored for the current call to execute it immediatly. #' #' @param nodenames [\code{character}]\cr #' Nodenames of workers to call function on. #' Only workers which were specified in your #' \code{\link{makeClusterFunctionsSSH}} configuration can be used. #' @param fun [\code{function}]\cr #' Function to call on workers. #' @param ... [any]\cr #' Arguments for \code{fun}. #' @param consecutive [\code{logical(1)}]\cr #' Do calls consecutively and always wait until each worker is done. #' Default is \code{FALSE}. #' @param show.output [\code{logical(1)}]\cr #' Show output of workers on master during call. #' Can be useful to see what is happening. #' Can only be used in consecutive mode. #' Default is \code{consecutive}. #' @param simplify [\code{logical(1)}]\cr #' Should the result be simplified? See \code{\link{sapply}}. #' Default is \code{TRUE}. #' @param use.names [\code{logical(1)}]\cr #' Name results by \code{nodenames}. #' Default is \code{TRUE}. #' @param dir [\code{character(1)}]\cr #' Directory under which a temporary registry will be #' created in a subdirectory for communication. #' This has to be somewhere on the shared #' filesystem. The created subdirectory will be cleaned up on exit. #' Default is current working directory. #' @return Results of function calls, either a list or simplified. #' @export callFunctionOnSSHWorkers = function(nodenames, fun, ..., consecutive = FALSE, show.output = consecutive, simplify = TRUE, use.names = TRUE, dir = getwd()) { assertCharacter(nodenames, any.missing = FALSE) assertFunction(fun) assertFlag(consecutive) assertFlag(show.output) assertFlag(simplify) assertFlag(use.names) if (!consecutive && show.output) stop("show.output = TRUE can only be used in consecutive mode.") conf = getBatchJobsConf() cf = conf$cluster.functions mail.old = c(conf$mail.start, conf$mail.done, conf$mail.error) if (cf$name != "SSH") stop("callFunctionOnSSHWorkers can only be used in SSH mode!") # create dummy registry, we submit our command from this regid = sprintf("BatchJobs_callFunctionOnSSHWorkers_%i", as.integer(Sys.time())) regdir = file.path(dir, regid) # we will change mailing and cluster funs, reset them on exit # also kill all still running jobs and remove reg dir on.exit({ conf$cluster.functions = cf conf$mail.start = mail.old[1L] conf$mail.done = mail.old[2L] conf$mail.error = mail.old[3L] }) # no mails during for the following jobs, also get the nodenames ssh workers conf$mail.start = conf$mail.done = conf$mail.error = "none" workers = environment(cf$submitJob)$workers d = setdiff(nodenames, names(workers)) if(length(d) > 0L) stopf("For some nodenames no workers exist: %s", collapse(d)) workers = workers[nodenames] # ignore load constraints old.worker.settings = list() wsettings = c("ncpus", "max.jobs", "max.load") for (wn in names(workers)) { w = workers[[wn]] old.worker.settings[[wn]] = mget(wsettings, w) for (ws in wsettings) w[[ws]] = Inf } on.exit({ # reset load settings for (wn in names(old.worker.settings)) { for (ws in wsettings) workers[[wn]][[ws]] = old.worker.settings[[wn]][[ws]] } }, add = TRUE) args = if (consecutive) args = list(fun) else replicate(length(nodenames), fun) suppressMessages({ reg = makeRegistry(regid, file.dir = regdir, sharding = FALSE) more.args = list(...) batchMap(reg, function(fun, ...) { print("###logstart###") res = fun(...) print("###logend###") res }, args, more.args = more.args) }) on.exit({ if (length(findOnSystem(reg)) > 0L) killJobs(reg, getJobIds(reg)) if (file.exists(regdir)) unlink(regdir, recursive = TRUE) }, add = TRUE) # read log as char string, get part between stamps and print only new chars printLog = function(log.old) { log.fn = getLogFiles(reg, 1L) log.new = readChar(log.fn, file.info(log.fn)$size) j = gregexpr("###logstart###", log.new)[[1L]][1L] if (j == -1) { # start not found log.new = "" } else { # start found, clip log.new = substr(log.new, j+15, nchar(log.new)) j = gregexpr("###logend###", log.new)[[1L]][1L] if (j != -1L) { # end found, clip log.new = substr(log.new, 1L, j-7L) } } cat(substr(log.new, nchar(log.old)+1L, nchar(log.new))) log.new } mysubmit = function(nodenames, reg) { messagef("Calling function on: %s.", collapse(nodenames)) conf$cluster.functions = makeClusterFunctionsSSH(workers = workers) capture.output(suppressMessages(submitJobs(reg, getJobIds(reg)))) } # while job not done, sleep and maybe print log waitTillJobsDone = function(reg) { log.old = "" while(length(findOnSystem(reg) > 0L)) { if (show.output) log.old = printLog(log.old) Sys.sleep(1) } if (show.output) { log.old = printLog(log.old) cat("\n\n") } } # if error, throw it on master checkJobErrors = function(reg, nodenames) { errids = findErrors(reg) if (length(errids) > 0L) { j = errids[1L] stopf("Error on %s: %s", nodenames[j], getErrorMessages(reg, j)) } } doit = function(reg, nodenames) { mysubmit(nodenames, reg) waitTillJobsDone(reg) checkJobErrors(reg, nodenames) } if (consecutive) { # loop though nodes individually and call function results = lapply(nodenames, function(nn) { log.old = "" doit(reg, nn) loadResult(reg, 1L) }) } else { doit(reg, nodenames) results = loadResults(reg, simplify = FALSE, use.names = FALSE) } if (use.names) names(results) = nodenames if (simplify) results = simplify2array(results) return(results) } BatchJobs/R/getJobParamDf.R0000644000176000001440000000116212415510734015134 0ustar ripleyusers#' @title Returns parameters for all jobs as the rows of a data.frame. #' #' @template arg_reg #' @template arg_ids #' @return [\code{data.frame}]. Rows are named with job ids. #' @export #' @examples #' # see batchExpandGrid getJobParamDf = function(reg, ids) { syncRegistry(reg) if (!missing(ids)) ids = checkIds(reg, ids) tab = dbGetExpandedJobsTable(reg, ids, cols = c("job_id", "pars")) # rownames are set by db* call # unserialize parameters res = convertListOfRowsToDataFrame(lapply(tab$pars, function(x) unserialize(charToRaw(x))), row.names = rownames(tab)) return(dropNamed(res, "job_id")) } BatchJobs/R/helpers.R0000644000176000001440000001034612465167456014153 0ustar ripleyuserscheckIds = function(reg, ids, check.present = TRUE) { ids = asInteger(ids, any.missing = FALSE, unique = TRUE) if (check.present) dbCheckJobIds(reg, ids) return(ids) } checkId = function(reg, id, check.present = TRUE) { id = asInt(id) if (check.present) dbCheckJobIds(reg, id) return(id) } checkMoreArgs = function(more.args, reserved) { assertList(more.args, names = "strict") n = names(more.args) if(is.null(n) || missing(reserved)) return(invisible(TRUE)) check = reserved %in% n if (any(check)) stopf("more.args uses element names which are internally reserved: %s", collapse(reserved[check])) return(invisible(TRUE)) } checkPart = function(reg, part) { if (reg$multiple.result.files) { if (!testScalarNA(part) && !testCharacter(part, any.missing = FALSE)) stop("'part' must be NA or a character vector without NAs!") } else { if (!testScalarNA(part)) stop("'part' must be NA because multiple.result.files is FALSE!") } } getListJobs = function(msg = NULL) { conf = getBatchJobsConf() cf = getClusterFunctions(conf) fun = cf$listJobs if (is.null(fun) && !is.null(msg)) stopf("%s because %s cluster functions do not support listing of jobs!", msg, cf$name) return(fun) } getKillJob = function(msg = NULL) { conf = getBatchJobsConf() cf = getClusterFunctions(conf) fun = cf$killJob if (is.null(fun) && !is.null(msg)) stopf("%s because %s cluster functions do not support killing of jobs!", msg, cf$name) return(fun) } getBatchIds = function(reg, msg = NULL) { fun = getListJobs(msg) fun(getBatchJobsConf(), reg) } getRandomSeed = function(n = 1L) { as.integer(runif(n, 1, .Machine$integer.max / 2L)) } seeder = function(reg, seed) { if(!exists(".Random.seed", envir = .GlobalEnv)) runif(1L) prev.seed = get(".Random.seed", envir = .GlobalEnv) prev.kind = RNGkind() set.seed(seed, kind = reg$RNGkind[1L], normal.kind = reg$RNGkind[2L]) return(list( reset = function() { RNGkind(kind = prev.kind[1L], normal.kind = prev.kind[2L]) assign(".Random.seed", prev.seed, envir = .GlobalEnv) })) } switchWd = function(reg) { cur = getwd() message("Setting work dir: ", reg$work.dir) setwd(reg$work.dir) return(list(reset = function() { message("Setting work back to: ", cur) setwd(cur) })) } addIntModulo = function(x, y, mod = .Machine$integer.max) { as.integer((as.double(x) + as.double(y)) %% mod) } isOnSlave = function() { getOption("BatchJobs.on.slave", default = FALSE) } setOnSlave = function(x, resources.path = as.character(NA)) { options(BatchJobs.on.slave = x) options(BatchJobs.resources.path = resources.path) } now = function() { as.integer(Sys.time()) } getArgNames = function(args) { if (!length(args)) return(NULL) if (is.null(names(args[[1L]])) && is.character(args[[1L]])) return(args[[1L]]) return(names(args[[1L]])) } convertUseNames = function(use.names) { if (is.character(use.names) && length(use.names) == 1L && use.names %in% c("none", "ids", "names")) return(use.names) assertFlag(use.names) c("none", "ids")[use.names+1L] } waitForFiles = function(fn, timeout = NA_real_, sleep = 1) { if (is.na(timeout)) return(invisible(TRUE)) fn = fn[!file.exists(fn)] if (length(fn)) { start = now() repeat { Sys.sleep(sleep) fn = fn[!file.exists(fn)] if (!length(fn)) break if (now() - start > timeout) stopf("Error waiting for file system. File '%s' timed out after %.1f seconds", head(fn, 1L), timeout) } } invisible(TRUE) } info = function(...) { if (getOption("BatchJobs.verbose", default = TRUE)) message(sprintf(...)) } getProgressBar = function(condition, ...) { if (condition) { pb = makeProgressBar(...) pb$set() } else { pb = makeProgressBar(style = "off") } pb } checkUserFunction = function(fun) { fun = match.fun(fun) if (getOption("BatchJobs.clear.function.env")) { environment(fun) = .GlobalEnv } else { ee = environment(fun) if (!is.null(ee) && !isNamespace(ee)) { nn = ls(ee, all.names = TRUE) if (sum(vnapply(nn, function(nn) object.size(ee[[nn]])) / 1024^2) > 10) warning("The environment of provided function exceeds 10Mb.") } } fun } BatchJobs/R/Worker.R0000644000176000001440000001107612465167456013763 0ustar ripleyusers# ******************** Constructors ******************** # Abstract base class constructor for general workers. # # @param nodename [\code{character(1)}]\cr # Host name of node. # @param rhome [\code{character(1)}]\cr # Path to R installation on worker. # \dQuote{} means R installation on the PATH is used. # @param r.options [\code{character}] # Options for R and Rscript, one option per element of the vector, # a la \dQuote{--vanilla}. # @param script [\code{character(1)}]\cr # Path to helper script on worker. # Default means to call \code{\link{findHelperScriptLinux}}. # @param ncpus [\code{integers(1)}]\cr # Number of VPUs of worker. # Default means to query the worker via \code{\link{getWorkerNumberOfCPUs}}. # @param max.jobs [\code{integer(1)}]\cr # Maximal number of jobs that can run concurrently for the current registry. # Default is \code{ncpus}. # @param max.load [\code{numeric(1)}]\cr # Load average (of the last 5 min) at which the worker is considered occupied, # so that no job can be submitted. # Default is \code{ncpus-1}. # @param nice [\code{integer(1)}]\cr # Process priority to run R with set via nice. Integers between -20 and 19 are allowed. # If missing, processes are not nice'd and the system default applies (usually 0). # @param classes [\code{character}]\cr # Extra classes, more specific than dQuote{Worker}. # Will be added to the class attribute of the object. # @return [\code{\link{Worker}}]. makeWorker = function(ssh, nodename, rhome, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script, ncpus, max.jobs, max.load, nice, classes) { assertFlag(ssh) assertString(nodename) assertString(rhome) assertCharacter(r.options, any.missing = FALSE) if (missing(script)) { # FIXME: dont use linux specific in base class script = findHelperScriptLinux(rhome, r.options, ssh, nodename) } else { assertString(script) } # construct object partially so we can query ncpus w = as.environment(list( ssh = ssh, nodename = nodename, rhome = rhome, r.options = r.options, script = script, last.update = -Inf, available = "A", # worker is available, we can submit, set in update loop in scheduleWorkerJobs.R status = NULL)) class(w) = c(classes, "Worker") if (missing(ncpus)) { ncpus = getWorkerNumberOfCPUs(w) messagef("Setting for worker %s: ncpus=%i", w$nodename, ncpus) } else { ncpus = asCount(ncpus) } if (missing(max.jobs)) { max.jobs = ncpus } else { max.jobs = asCount(max.jobs) } if (missing(max.load)) { max.load = ncpus-1L } else { assertNumber(max.load, lower = 0) } if (missing(nice)) { nice = "" } else { nice = asInt(nice, lower = -20, upper = 19) } w$ncpus = ncpus w$max.jobs = max.jobs w$max.load = max.load w$nice = nice return(w) } # ******************** Interface definition ******************** # Return number of cores on worker. # @param worker [\code{\link{Worker}}]. # Worker. # @return [\code{integer(1)}]. getWorkerNumberOfCPUs = function(worker) { UseMethod("getWorkerNumberOfCPUs") } # Return 4 numbers to describe worker status. # - load average of last 1 min, as given by e.g. uptime # - number of R processes by _all_ users # - number of R processes by _all_ users which have a load of >= 50% # - number of R processes by current user which match $FILEDIR/jobs in the cmd call of R # @param worker [\code{\link{Worker}}]. # Worker. # @param file.dir [\code{character(1)}}]. # File dir of registry. # @return [named \code{list} of \code{numeric(1)}]. getWorkerStatus = function(worker, file.dir) { UseMethod("getWorkerStatus") } # Start a job on worker, probably with R CMD BATCH. # @param worker [\code{\link{Worker}}]. # Worker. # @param rfile [\code{character(1)}]. # Path to R file to execute. # @param outfile [\code{character(1)}]. # Path to log file for R process. # @return [\code{character(1)}]. Relevant process id. startWorkerJob = function(worker, rfile, outfile) { UseMethod("startWorkerJob") } # Kill a job on worker. Really do it. # @param worker [\code{\link{Worker}}]. # Worker. # @param pid [\code{character(1)}]. # Process id from DB/batch.job.id to kill. # @return Nothing. killWorkerJob = function(worker, pid) { UseMethod("killWorkerJob") } # List all jobs on worker belonging to the current registry. # @param worker [\code{\link{Worker}}]. # Worker. # @param file.dir [\code{character(1)}}]. # File dir of registry. # @return [\code{character}]. Vector of process ids. listWorkerJobs = function(worker, file.dir) { UseMethod("listWorkerJobs") } BatchJobs/R/Job.R0000644000176000001440000000307112415510734013202 0ustar ripleyusers#' Creates a job description. #' #' Usually you will not do this manually. #' Every object is a list that contains the passed arguments of the constructor. #' #' @param id [\code{integer(1)}]\cr #' Job id, determined by DB autoincrement. #' Default is \code{NA}. #' @param fun [\code{function}]\cr #' Job function to apply on parameters. #' @param fun.id [\code{character(1)}]\cr #' Id used to store function on disk. #' Default is \code{digest(fun)}. #' @param pars [\code{list}]\cr #' Parameter list for job function. #' @param name [\code{character(1)}]\cr #' Alias name for this job. #' @param seed [\code{integer(1)}]\cr #' Random seed for job. #' @aliases Job #' @export makeJob = function(id = NA_integer_, fun, fun.id = digest(fun), pars, name, seed) { setClasses(list(id = id, fun = fun, fun.id = fun.id, pars = pars, name = name, seed = seed), "Job") } #' Get number of jobs in registry. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @return [\code{integer(1)}]. #' @export getJobNr = function(reg) { checkRegistry(reg) dbGetJobCount(reg) } #' Get ids of jobs in registry. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @return [\code{character}]. #' @export getJobIds = function(reg) { checkRegistry(reg) dbGetJobIds(reg) } #' @export print.Job = function(x, ...) { cat("BatchJobs job:\n") catf(" Job id: %s", x$id) catf(" Fun id: %s", x$fun.id) catf(" Fun formals: %s", collapse(names(formals(x$fun)))) catf(" Name: %s", x$name) catf(" Seed: %i", x$seed) catf(" Pars: %s", convertToShortString(x$pars)) } BatchJobs/R/conf.R0000644000176000001440000001713212502316642013417 0ustar ripleyusers#' BatchJobs configuration. #' #' In order to understand how the package should be configured #' please read #' \url{https://github.com/tudo-r/BatchJobs/wiki/Configuration}. #' #' @name configuration #' @rdname configuration #' @family conf #' @aliases .BatchJobs.R NULL # sources 1 config file and returns the envir sourceConfFile = function(conffile) { assertFile(conffile) if (getOption("BatchJobs.verbose", default = TRUE)) packageStartupMessage(sprintf("Sourcing configuration file: '%s'", conffile)) conf = new.env() x = try(sys.source(conffile, envir = conf)) if (is.error(x)) stopf("There was an error in sourcing your configuration file '%s': %s!", conffile, as.character(x)) checkConf(conf) do.call(checkConfElements, as.list(conf)) return(conf) } # sources multiple config files, the later overwrite the first, and returns the envir sourceConfFiles = function(conffiles) { conf = new.env() for (cf in conffiles) { conf2 = sourceConfFile(cf) lapply(ls(conf2), function(x) assign(x, conf2[[x]], envir = conf)) } return(conf) } # assigns a conf to namespace assignConf = function(conf) { conf.in.ns = getBatchJobsConf() lapply(ls(conf), function(x) assign(x, conf[[x]], envir = conf.in.ns)) } # locates package conf, userhome conf, working dir conf findConfigs = function(path=find.package("BatchJobs")) { fn.pack = file.path(path, "etc", "BatchJobs_global_config.R") fn.user = path.expand("~/.BatchJobs.R") fn.wd = suppressWarnings(normalizePath(".BatchJobs.R")) Filter(file.exists, unique(c(fn.pack, fn.user, fn.wd))) } # reads available config files and assigns them to namespace readConfs = function(path=find.package("BatchJobs")) { conffiles = findConfigs(path) if (length(conffiles) == 0L) { warning("No configuation found at all. Not in package, not in user.home, not in work dir!") assignConfDefaults() } # really do this in 2 steps # otherwise weird things might happen due to lazy eval combined with envirs # and we might not see the error msg triggered in the checking of the config file conf = sourceConfFiles(conffiles) assignConf(conf) } assignConfDefaults = function() { conf = getBatchJobsConf() conf$cluster.functions = makeClusterFunctionsInteractive() conf$mail.start = "none" conf$mail.done = "none" conf$mail.error = "none" conf$db.driver = "SQLite" conf$db.options = list(pragmas = "busy_timeout=5000") conf$default.resources = list() conf$debug = FALSE conf$raise.warnings = FALSE conf$staged.queries = TRUE conf$max.concurrent.jobs = Inf conf$fs.timeout = NA_real_ } # loads conf into namespace on slave loadConf = function(reg) { fn = getConfFilePath(reg) info("Loading conf: ", fn) ee = new.env() load(fn, envir = ee) ns = ls(ee$conf) # assign all stuff to conf in namespace conf = getBatchJobsConf() lapply(ns, function(x) assign(x, ee$conf[[x]], envir = conf)) invisible(NULL) } getBatchJobsConf = function() { get(".BatchJobs.conf", envir = getNamespace("BatchJobs")) } saveConf = function(reg) { fn = getConfFilePath(reg) info("Saving conf: %s", fn) conf = getBatchJobsConf() save(file = fn, conf) } getConfNames = function() { c("cluster.functions", "mail.start", "mail.done", "mail.error", "mail.from", "mail.to", "mail.control", "db.driver", "db.options", "default.resources", "debug", "raise.warnings", "staged.queries", "max.concurrent.jobs", "fs.timeout") } checkConf = function(conf) { ns = if (is.list(conf)) names(conf) else ls(conf, all.names = TRUE) ns2 = getConfNames() if (any(ns %nin% ns2)) stopf("You are only allowed to define the following R variables in your config:\n%s\nBut you also had:\n%s", collapse(ns2, sep = ", "), collapse(setdiff(ns, ns2), sep = ", ")) } checkConfElements = function(cluster.functions, mail.to, mail.from, mail.start, mail.done, mail.error, mail.control, db.driver, db.options, default.resources, debug, raise.warnings, staged.queries, max.concurrent.jobs, fs.timeout) { mail.choices = c("none", "first", "last", "first+last", "all") if (!missing(cluster.functions)) assertClass(cluster.functions, "ClusterFunctions") if (!missing(mail.from)) assertString(mail.from) if (!missing(mail.to)) assertString(mail.to) if (!missing(mail.start)) assertChoice(mail.start, mail.choices) if (!missing(mail.done)) assertChoice(mail.done, mail.choices) if (!missing(mail.error)) assertChoice(mail.error, mail.choices) if (!missing(mail.control)) assertList(mail.control) if (!missing(db.driver)) assertString(db.driver) if (!missing(db.options)) assertList(db.options, names = "named") if (!missing(default.resources)) assertList(default.resources, names = "named") if (!missing(debug)) assertFlag(debug) if (!missing(raise.warnings)) assertFlag(raise.warnings) if (!missing(staged.queries)) assertFlag(staged.queries) if (!missing(max.concurrent.jobs)) assertCount(max.concurrent.jobs) if (!missing(fs.timeout)) assertNumber(fs.timeout) } getClusterFunctions = function(conf) { conf$cluster.functions } # Function which returns a printable string describing the config # Used in packageStartupMessage and in print.Config printableConf = function(conf) { x = as.list(conf) x[setdiff(getConfNames(), names(x))] = "" fmt = paste( "BatchJobs configuration:", " cluster functions: %s", " mail.from: %s", " mail.to: %s", " mail.start: %s", " mail.done: %s", " mail.error: %s", " default.resources: %s", " debug: %s", " raise.warnings: %s", " staged.queries: %s", " max.concurrent.jobs: %s", " fs.timeout: %s\n", sep = "\n") sprintf(fmt, x$cluster.functions$name, x$mail.from, x$mail.to, x$mail.start, x$mail.done, x$mail.error, convertToShortString(x$default.resources), x$debug, x$raise.warnings, x$staged.queries, x$max.concurrent.jobs, x$fs.timeout) } #' @export print.Config = function(x, ...) { cat(printableConf(x)) } #' Load a specific configuration file. #' #' @param conffile [\code{character(1)}]\cr #' Location of the configuration file to load. #' Default is \dQuote{.BatchJobs.conf} in the current working directory. #' @return Invisibly returns a list of configuration settings. #' @family conf #' @export loadConfig = function(conffile = ".BatchJobs.R") { # checks are done in sourceConfFile conf = sourceConfFile(conffile) assignConf(conf) invisible(setClasses(as.list(conf), "Config")) } #' Set and overwrite configuration settings #' #' @param conf [\code{Config} or \code{list}]\cr #' List of configuration parameters as returned by \code{\link{loadConfig}} or \code{\link{getConfig}}. #' @param ... [\code{ANY}]\cr #' Named configuration parameters. Overwrites parameters in \code{conf}, if provided. #' @return Invisibly returns a list of configuration settings. #' @family conf #' @export setConfig = function(conf = list(), ...) { if (!is.list(conf) && !inherits(conf, "Config")) stopf("Argument 'conf' must be of class 'list' or 'Config', not %s", head(conf, 1L)) overwrites = insert(conf, list(...)) if (length(overwrites) == 0L) return(invisible(getConfig())) if (!isProperlyNamed(overwrites)) stopf("All configuration arguments in '...' must be properly named") checkConf(overwrites) conf = insert(as.list(getBatchJobsConf()), overwrites) assignConf(as.environment(conf)) invisible(setClasses(conf, "Config")) } #' Returns a list of BatchJobs configuration settings #' #' @return \code{list} of current configuration variables with classs \dQuote{Config}. #' @family conf #' @export getConfig = function() { setClasses(as.list(getBatchJobsConf()), "Config") } BatchJobs/R/debugMulticore.R0000644000176000001440000000527312415510734015450 0ustar ripleyusers#' Helper function to debug multicore mode. #' #' @description #' Useful in case of severe errors. #' Tries different operations of increasing difficulty #' and provides debug output on the console #' #' @param r.options [\code{list}] #' Options for R and Rscript, one option per element of the vector, #' a la \dQuote{--vanilla}. #' Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}. #' @return Nothing. #' @family debug #' @export debugMulticore = function(r.options) { if (missing(r.options)) r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file") conf = getBatchJobsConf() conf$debug = TRUE conf$mail.start = conf$mail.done = conf$mail.error = "none" rhome = R.home() messagef("*** System info: ***") print(Sys.info()) catf("\n") messagef("*** which R: ***") res = runOSCommandLinux("which", "R") messagef("which R result: %s", res$output) catf("\n") messagef("*** Find helper script: ***") script = findHelperScriptLinux(rhome = rhome, r.options = r.options) messagef("Find helper script result: %s", script) catf("\n") messagef("*** Auto-detecting ncpus: ***") worker = makeWorkerLocalLinux(r.options = r.options, script = script, ncpus = 1) ncpus = runWorkerCommand(worker, "number-of-cpus") messagef("Auto-detecting ncpus result: %s", ncpus) catf("\n") messagef("*** Query worker status: ***") res = runWorkerCommand(worker, "status", args = "") messagef("Query worker status result: %s", res) catf("\n") messagef("*** Submitting 1 job: ***") conf$cluster.functions = makeClusterFunctionsMulticore() fd = tempfile() reg = makeRegistry(id = "debug_multicore", file.dir = fd, sharding = FALSE) batchMap(reg, identity, 1) submitJobs(reg) Sys.sleep(3) messagef("Submitting 1 job result: %i", loadResult(reg, 1)) messagef("Query worker status:") res = runWorkerCommand(worker, "status", args = reg$file.dir) messagef("Query worker status result: %s", res) catf("\n") messagef("*** Killing 2 jobs: ***") fd = tempfile() reg = makeRegistry(id = "debug_multicore", file.dir = fd, sharding = FALSE) f = function(i) if(i <= 1) i else f(i-1) + f(i-2) xs = 50 + seq(1,2) ids = 1:2 batchMap(reg, f, xs) submitJobs(reg) Sys.sleep(3) messagef("Query worker status:") res = runWorkerCommand(worker, "status", args = reg$file.dir) messagef("Query worker status result: %s", res) messagef("Running jobs: %s", collapse(findRunning(reg))) killJobs(reg, ids) messagef("Query worker status:") res = runWorkerCommand(worker, "status", args = reg$file.dir) messagef("Query worker status result: %s", res) messagef("Running jobs: %s", collapse(findRunning(reg))) catf("\n") } BatchJobs/R/clusterFunctionsHelpers.R0000644000176000001440000001143012415510734017363 0ustar ripleyusers#' Cluster functions helper: Read in your brew template file. #' #' @description #' This function is only intended for use in your own cluster functions implementation. #' #' Simply reads your template and returns it as a character vector. #' If you do this in the constructor of your cluster functions once, you can avoid this #' repeated file access later on. #' #' @param template.file [\code{character(1)}]\cr #' File path. #' @return [\code{character}]. #' @export cfReadBrewTemplate = function(template.file) { assertFile(template.file, "r") tmpl = readLines(template.file) if (length(tmpl) == 0L) stopf("Error reading template '%s' or empty template", template.file) collapse(tmpl, "\n") } #' Cluster functions helper: Brew your template into a job description file. #' #' @description #' This function is only intended for use in your own cluster functions implementation. #' #' Calls brew silently on your template, any error will lead to an exception. #' If debug mode is turned on in the configuration, the file is stored at the same place as the #' corresponding R script in the \dQuote{jobs}-subdir of your files directory, #' otherwise in the temp dir via \code{\link{tempfile}}. #' #' @param conf [\code{environment}]\cr #' BatchJobs configuration. #' @param template [\code{character(1)}]\cr #' Job desfription template as a char vecrtor, #' possibly read in via \code{\link{cfReadBrewTemplate}}. #' @param rscript [\code{character(1)}]\cr #' File path to you your corresponding R script for the job. #' @param extension [\code{character(1)}]\cr #' Extension for the job description file, e.g. \dQuote{pbs}. #' @return [\code{character(1)}]. File path of result. #' @export cfBrewTemplate = function(conf, template, rscript, extension) { assertEnvironment(conf) assertString(template) assertString(rscript) assertString(extension) if (conf$debug) { # if debug, place in jobs dir outfile = sub("\\.R$", sprintf(".%s", extension), rscript) } else { outfile = tempfile("template") } pf = parent.frame() old = getOption("show.error.messages") on.exit(options(show.error.messages = old)) options(show.error.messages = FALSE) z = suppressAll(try(brew(text = template, output = outfile, envir = pf), silent = TRUE)) if (is.error(z)) stopf("Error brewing template: %s", as.character(z)) waitForFiles(outfile, conf$fs.timeout) return(outfile) } #' Cluster functions helper: Handle an unknown error during job submission. #' #' @description #' This function is only intended for use in your own cluster functions implementation. #' #' Simply constructs a \code{\link{SubmitJobResult}} object with status code 101, #' NA as batch job id and an informative error message containing the output of the OS command in \code{output}. #' #' @param cmd [\code{character(1)}]\cr #' OS command used to submit the job, e.g. qsub. #' @param exit.code [\code{integer(1)}]\cr #' Exit code of the OS command, should not be 0. #' @param output [\code{character}]\cr #' Output of the OS command, hopefully an informative error message. #' If these are mutiple lines in a vector, they are automatically pasted together. #' @return [\code{\link{SubmitJobResult}}]. #' @export cfHandleUnknownSubmitError = function(cmd, exit.code, output) { assertString(cmd) exit.code = asInt(exit.code) assertCharacter(output, any.missing = FALSE) msg = sprintf("%s produced exit code %i; output %s", cmd, exit.code, collapse(output, sep = "\n")) makeSubmitJobResult(status = 101L, batch.job.id = NA_character_, msg = msg) } #' Cluster functions helper: Kill a batch job via OS command #' #' @description #' This function is only intended for use in your own cluster functions implementation. #' #' Calls the OS command to kill a job via \code{system} like this: \dQuote{cmd batch.job.id}. #' If the command returns an exit code > 0, the command is repeated #' after a 1 second sleep \code{max.tries-1} times. #' If the command failed in all tries, an exception is generated. #' #' @param cmd [\code{character(1)}]\cr #' OS command, e.g. \dQuote{qdel}. #' @param batch.job.id [\code{character(1)}]\cr #' Id of the batch job on the batch system. #' @param max.tries [\code{integer(1)}]\cr #' Number of total times to try execute the OS command in cases of failures. #' Default is \code{3}. #' @return Nothing. #' @export cfKillBatchJob = function(cmd, batch.job.id, max.tries = 3L) { assertString(cmd) assertString(batch.job.id) max.tries = asCount(max.tries) assertCount(max.tries) for (tmp in seq_len(max.tries)) { res = runOSCommandLinux(cmd, batch.job.id, stop.on.exit.code = FALSE) if (res$exit.code == 0L) return() Sys.sleep(1) } stopf("Really tried to kill job, but could not do it. batch job id is %s.\nMessage: %s", batch.job.id, collapse(res$output, sep = "\n")) } BatchJobs/R/showLog.R0000644000176000001440000000512612437616220014116 0ustar ripleyusers#' Display the contents of a log file. #' #' @description #' Display the contents of a log file, useful in case of errors. #' #' Note this rare special case: When you use chunking, submit some jobs, some jobs fail, #' then you resubmit these jobs again in different chunks, the log files will contain the log #' of the old, failed job as well. \code{showLog} tries to jump to the correct part #' of the new log file with a supported pager. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param id [\code{integer(1)}]\cr #' Id of selected job. #' Default is first id in registry. #' @param pager [\code{any}]\cr #' Pager to use to display the log. Defaults to \code{getOption("pager")}. #' This option is passed to \code{file.show} and is highly OS dependant and GUI dependant. #' If either R's pager or the environment variable \dQuote{PAGER} is set to \dQuote{less} #' or \dQuote{vim}, the correct part of the log file will be shown. #' Otherwise you find information about the correct part in the beginning of the displayed file. #' @return [\code{character(1)}]. Invisibly returns path to log file. #' @family debug #' @export showLog = function(reg, id, pager = getOption("pager")) { checkRegistry(reg) syncRegistry(reg) if (missing(id)) { id = dbGetJobId(reg) if (length(id) == 0L) stop("No jobs in registry!") } else { id = checkId(reg, id) } fn = getLogFiles(reg, id) if (!file.exists(fn)) stopf("Log file does not exist: %s", fn) exact1 = sprintf("Executing jid=%i", id) exact2 = sprintf("########## %s ##########", exact1) header = c(sprintf("Showing log file for job with id=%i", id), sprintf("The string '%s' indicates start of requested job", exact1)) if (!is.function(pager)) { sys.pager = Sys.getenv("PAGER") if ((grepl("/pager$", pager) || pager == "internal") && sys.pager != "") { # prefer the system pager pager = sys.pager } bn = basename(strsplit(pager, " ", fixed = TRUE)[[1L]][1L]) # check for less or vim as pager # if we find the pattern, we jump to the matching line if (bn %in% c("less", "vim")) { pos = grep(exact2, readLines(fn), fixed = TRUE) if (length(pos) == 1L) { pos = pos + length(header) + 1L if (pager == "less") pager = function(files, header, title, delete.file) system2("less", c(sprintf("+%ig", pos), files)) else pager = function(files, header, title, delete.file) system2("vim", c(sprintf("+%i", pos), files)) } } } file.show(fn, pager = pager, header = collapse(header, sep = "\n")) invisible(fn) } BatchJobs/R/debugSSH.R0000644000176000001440000000675612415510734014151 0ustar ripleyusers#' @title Helper function to debug SSH mode. #' #' @description #' Useful in case of configuration problems. #' Tries different operations of increasing difficulty #' and provides debug output on the console. #' #' Note that this function does not access nor use information specified for #' your cluster functions in your configuration. #' #' @param nodename [\code{character(1)}]\cr #' Node on which worker should be constructed for the test. #' @param rhome [\code{character(1)}]\cr #' Path to R installation on the worker. #' \dQuote{} means R installation on the PATH is used, #' of course this implies that it must be on the PATH #' (also for non-interactive shells)! #' Default is \dQuote{}. #' @param r.options [\code{list}] #' Options for R and Rscript, one option per element of the vector, #' a la \dQuote{--vanilla}. #' Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}. #' @param dir [\code{character(1)}]\cr #' Path where internally used test registries can be created. #' Note that this must be shared for the worker. #' Default is current working directory. #' @return Nothing. #' @family debug #' @export debugSSH = function(nodename, rhome = "", r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), dir = getwd()) { assertString(nodename) assertString(rhome) assertString(dir) conf = getBatchJobsConf() conf$debug = TRUE conf$mail.start = conf$mail.done = conf$mail.error = "none" wd = dir messagef("*** System info on master: ***") print(Sys.info()) catf("\n") messagef("*** which R on slave: ***") res = runOSCommandLinux(cmd = "which", args = "R", ssh = TRUE, nodename = nodename, stop.on.exit.code = TRUE) messagef("which R result:") print(res) catf("\n") messagef("*** Find helper script on slave: ***") res = findHelperScriptLinux(rhome, r.options, ssh = TRUE, nodename = nodename) messagef("Find helper script result:") print(res) catf("\n") messagef("*** Auto-detecting ncpus for slave: ***") worker = makeWorkerRemoteLinux(nodename = nodename, rhome = rhome, r.options = r.options, ncpus = 1) res = runWorkerCommand(worker = worker, command = "number-of-cpus") messagef("Auto-detecting ncpus result:") print(res) catf("\n") queryWorkerStatus = function() { messagef("*** Query worker status: ***") res = runWorkerCommand(worker = worker, command = "status", args = "") messagef("Query worker status result:") message("load n.rprocs n.rprocs.50 n.jobs") print(res) catf("\n") } queryWorkerStatus() messagef("*** Submitting 1 job: ***") ssh.workers = list(makeSSHWorker(nodename = nodename, rhome = rhome, r.options = r.options)) conf$cluster.functions = do.call(makeClusterFunctionsSSH, ssh.workers) id = "debug_ssh_1" reg = makeRegistry(id = id, file.dir = file.path(dir, id), work.dir = wd, sharding = FALSE) batchMap(reg, identity, 1) submitJobs(reg) Sys.sleep(3) messagef("Submitting 1 job result: %i", loadResult(reg, 1)) queryWorkerStatus() messagef("*** Killing 2 jobs: ***") id = "debug_ssh_2" reg = makeRegistry(id = id, file.dir = file.path(dir, id), work.dir = wd, sharding = FALSE) f = function(i) if(i <= 1) i else f(i-1) + f(i-2) xs = 50 + seq(1,2) ids = 1:2 batchMap(reg, f, xs) submitJobs(reg) Sys.sleep(3) queryWorkerStatus() messagef("Running jobs: %s", collapse(findRunning(reg))) killJobs(reg, ids) queryWorkerStatus() messagef("Running jobs: %s", collapse(findRunning(reg))) catf("\n") } BatchJobs/R/batchApply.R0000644000176000001440000000547412465167456014606 0ustar ripleyusers# @title Apply a function on maticies or arrays # # A simple wrapper to batchMap to define jobs as an application of a function. # on margins of matrices or arrays. # # @param reg [\code{\link{Registry}}]\cr # Empty Registry that will store jobs for the mapping. # @param X [\code{\link[base]{matrix}} | \code{\link[base]{array}}]\cr # A matrix or an array. # @param margin [\code{integer(1)}]\cr # Margin of the matrix. 1 for rows, 2 for columns. # @param fun [\code{function}]\cr # Function to map over \code{...}. # @param chunk.size [\code{integer(1)}]\cr # Preferred number of jobs in each chunk. # Can not be used in combination with \code{n.chunks}. # Default is \code{chunk.size = 1} if \code{n.chunks} is also not provided and # results in \code{\link{nrow}} or \code{\link{ncol}} jobs, respectively. # @param n.chunks [\code{integer(1)}]\cr # Preferred number of chunks. # Can not be used in combination with \code{chunk.size}. # @param ... [any]\cr # Additional arguments passed to \code{fun}. # @return [\code{integer}]. Ids of created jobs. # @examples # reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) # X = matrix(1:16, 4) # # Define two jobs to calculate the row sums: # batchApply(reg, X, 1, sum, n.chunks = 2) # submitJobs(reg) # waitForJobs(reg) # reduceResultsVector(reg, use.names = FALSE) == rowSums(X) batchApply = function(reg, X, margin, fun, chunk.size, n.chunks, ..., use.names = FALSE) { assert(checkMatrix(X), checkArray(X)) dX = dim(X) margin = asInt(margin, lower = 1L, upper = length(dX)) fun = checkUserFunction(fun) more.args = list(...) checkMoreArgs(more.args, c(".X", ".inds", ".user.fun")) assertFlag(use.names) inds = chunk(seq_len(dX[margin]), chunk.size = chunk.size, n.chunks = n.chunks, shuffle = FALSE) if (use.names && !is.null(dimnames(X)[[margin]])) names(inds) = dimnames(X)[[margin]] more.args = c(more.args, list(.X = aperm(X, c(margin, seq_along(dX)[-margin])), .user.fun = fun)) batchMap(reg, batchApplyWrapper, .inds = inds, more.args = more.args, use.names = use.names) } batchApplyWrapper = function(.inds, .X, .user.fun, ...) { apply(.X[.inds,, drop = FALSE], 1L, .user.fun, ...) } # FIXME: we should split the matrix on the master first. # here is a piece of code which allows this, but there is no sufficient # mechanism in BJ to find these files again on the slave? # maybe store in work.dir? # splitX = function(X, margin, chunks) { # n = length(chunks) # X = aperm(X, c(margin, seq_along(dim(X))[-margin])) # dn = file.path(reg$file.dir, "data-chunks") # dir = checkDir(dn, create = TRUE) # dest = file.path(dn, sprintf("chunk-%i.RData", seq_len(n))) # # info("Splitting input into %i files ...", n) # for (i in seq_len(n)) { # save2(.X = X[chunks[[i]],, drop = FALSE], file = dest[i]) # } # dest # } BatchJobs/R/batchExpandGrid.R0000644000176000001440000000403112415510734015514 0ustar ripleyusers#' @title Map function over all combinations. #' #' @description #' Maps an n-ary-function over a list of all combinations which are given by some vectors. #' Internally \code{\link{expand.grid}} is used to compute the combinations, then #' \code{\link{batchMap}} is called. #' #' #' @param reg [\code{\link{Registry}}]\cr #' Empty Registry that will store jobs for the mapping. #' @param fun [\code{function}]\cr #' Function to map over the combinations. #' @param ... [any]\cr #' Vectors that are used to compute all combinations. #' If the arguments are named, these names are used to bind to arguments of \code{fun}. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @return [\code{data.frame}]. Expanded grid of combinations produced by \code{\link{expand.grid}}. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x, y, z) x * y + z #' # lets store the param grid #' grid = batchExpandGrid(reg, f, x = 1:2, y = 1:3, more.args = list(z = 10)) #' submitJobs(reg) #' waitForJobs(reg) #' y = reduceResultsVector(reg) #' # later, we can always access the param grid like this #' grid = getJobParamDf(reg) #' cbind(grid, y = y) batchExpandGrid = function(reg, fun, ..., more.args = list()) { checkRegistry(reg, strict = TRUE) assertFunction(fun) args = list(...) ns = names(args) if (length(args) == 0L) return(invisible(integer(0L))) if(!all(vlapply(args, is.vector))) stop("All args in '...' must be vectors!") checkMoreArgs(more.args) reserved = c("KEEP.OUT.ATTRS", "stringsAsFactors") if (any(reserved %in% ns)) stopf("You cannot use the reserved arg names %s in ... args!", collapse(reserved)) args$KEEP.OUT.ATTRS = FALSE args$stringsAsFactors = FALSE grid = do.call(expand.grid, args) if (is.null(ns)) colnames(grid) = NULL do.call(batchMap, c(as.list(grid), list(reg = reg, fun = fun, more.args = more.args))) return(setRowNames(grid, as.character(getJobIds(reg)))) } BatchJobs/R/filenames.R0000644000176000001440000001277212502316642014442 0ustar ripleyuserscheckDir = function(path, create = FALSE, check.empty = FALSE, check.posix = FALSE, msg = FALSE) { if (create) { if (file.exists(path)) { if (!isDirectory(path)) stop("File in place where dir should be created: ", path) } else { if (msg) info("Creating dir: %s", path) if (!dir.create(path, recursive = TRUE)) stopf("Could not create dir: %s", path) } } if (!isDirectory(path)) stopf("Directory '%s' does not exists", path) if (!is.accessible(path)) stopf("Directory '%s' is not readable/writable!", path) if (check.empty && any(list.files(path, all.files = TRUE) %nin% c(".", ".."))) stopf("Directory '%s' does not seem to be empty: %s", path, paste(setdiff(list.files(path, all.files = TRUE), c(".", "..")), collapse=", ")) if (check.posix && getOption("BatchJobs.check.posix", TRUE)) { path.abs = sanitizePath(path, make.absolute = TRUE) if(! grepl("^[[:alnum:]:/_.-]+$", path.abs)) stopf("Directory '%s' contains characters that are not fully portable according to POSIX standards. Allowed: a-z A-Z 0-9 : / . - _", path.abs) } } checkDirs = function(paths, ...) { vcapply(paths, checkDir) } createShardedDirs = function(reg, ids) { if (reg$sharding) { lapply(getJobDirs(reg, ids, unique = TRUE), checkDir, create = TRUE) } } # tests a directory for read and write permissions # uses a heuristic for windows is.accessible = function(path) { if (isWindows()) { # Workaround: No POSIX file system informations available, use a heuristic rnd = basename(tempfile("")) tf1 = file.path(path, sprintf("test_write_access_file_%s", rnd)) td1 = file.path(path, sprintf("test_write_access_dir_%s", rnd)) tf2 = file.path(td1, "test_write_access_subfile") td2 = file.path(td1, "test_write_access_subdir") # on exit, try to clean up the mess we might have caused on.exit(try(removeDirs(c(tf2, td2, tf1, td1), recursive = TRUE))) fileCreate = function(pathname) { if (!file.create(pathname)) return(FALSE) msg0 = "Hello world!" cat(file=pathname, msg0) msg = readLines(pathname, warn=FALSE) if (identical(msg, msg0)) return(TRUE) stop(sprintf("Created test file does not contain expected content ('%s'): '%s'", msg, msg0)) } # perform the checks ok = try({ fileCreate(tf1) && dir.create(td1) && dir.create(td2) && length(list.files(td1)) == 1L && fileCreate(tf2) && removeDirs(c(tf2, td2, tf1, td1), recursive = TRUE) }) if (is.error(ok) || !isTRUE(ok)) return(FALSE) # we don't need the on exit handler anymore, everything should be fine on.exit(NULL) return(TRUE) } return(file.access(path, mode = c(2L, 4L)) == 0L) } # a more trusty version than file.exists() fileExists = function(path) { if (file.exists(path)) return(TRUE) ## Double check with what dir() reports, because (at least) on Windows ## there seems to be a delay between a file/directory being removed ## and the directory listings being updated. /HB 2015-02-08 filename = basename(path) dirs = dir(path = dirname(path), all.files = TRUE, include.dirs = TRUE) return(filename %in% dirs) } # a more robust and insisting version than unlink() removeDirs = function(paths, recursive=FALSE, ..., must.work=TRUE, max.tries=30L, interval=0.2) { exists = vlapply(paths, fileExists) for (ii in which(exists)) { path = paths[ii] for (kk in seq_len(max.tries)) { unlink(path, recursive=recursive, ...) exists[ii] = fileExists(path) if (!exists[ii]) break if (kk < max.tries) Sys.sleep(interval) } } exists = vlapply(paths, fileExists) failed = path[exists] if (must.work && length(failed) > 0L) stop("Failed to remove files/directories: ", paste(sQuote(path), collapse=", ")) return(!exists) } isPathFromRoot = function(path) { (isWindows() & grepl("^[[:alpha:]]:", path)) | grepl("^[/\\]", path) } getJobDirs = function(reg, ids, unique = FALSE) { if (reg$sharding) { shards = sprintf("%02i", ids %% 100L) if(unique) shards = unique(shards) return(file.path(reg$file.dir, "jobs", shards)) } file.path(reg$file.dir, "jobs") } getFilePaths = function(reg, id, suffix, ext) { fn = sprintf("%i%s.%s", id, ifelse(is.null(suffix), "", paste0("-", suffix)), ext) file.path(getJobDirs(reg, id), fn) } getJobParentDir = function(file.dir) file.path(file.dir, "jobs") getFunDir = function(file.dir) file.path(file.dir, "functions") getFunFilePath = function(reg, fun.id) file.path(getFunDir(reg$file.dir), sprintf("%s.RData", fun.id)) getConfFilePath = function(reg) file.path(reg$file.dir, "conf.RData") getRegistryFilePath = function(file.dir) file.path(file.dir, "registry.RData") getRScriptFilePath = function(reg, id) getFilePaths(reg, id, NULL, "R") getLogFilePath = function(reg, id) getFilePaths(reg, id, NULL, "out") getResultFilePath = function(reg, id, part = NA_character_) { s = if (is.na(part)) "result" else paste0("result-", part) getFilePaths(reg, id, s, "RData") } getResourcesDir = function(file.dir) file.path(file.dir, "resources") getResourcesFilePath = function(reg, timestamp) file.path(getResourcesDir(reg$file.dir), sprintf("resources_%i.RData", timestamp)) getPendingDir = function(file.dir) file.path(file.dir, "pending") getExportDir = function(file.dir) file.path(file.dir, "exports") getPendingFile = function(reg, type, id, char = .OrderChars[type]) { file.path(getPendingDir(reg$file.dir), sprintf("pending_%s_%s_%i.sql", char, type, id)) } BatchJobs/R/Packages.R0000644000176000001440000000211612415510734014205 0ustar ripleyusers#' @title Add packages to registry. #' #' @description #' Mutator function for \code{packages} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param packages [\code{character}]\cr #' Packages to add to registry. #' @template ret_reg_mut #' @family exports #' @export addRegistryPackages = function(reg, packages) { checkRegistry(reg) assertCharacter(packages, any.missing = FALSE) packages = setNames(lapply(packages, function(pkg) list(version = packageVersion(pkg))), packages) p = c(reg$packages, packages) p = p[unique(names(p))] reg$packages = p saveRegistry(reg) } #' @title Remove packages from registry. #' #' @description #' Mutator function for \code{packages} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param packages [\code{character}]\cr #' Packages to remove from registry. #' @template ret_reg_mut #' @family exports #' @export removeRegistryPackages = function(reg, packages) { checkRegistry(reg) assertCharacter(packages, any.missing = FALSE) reg$packages = reg$packages[setdiff(names(reg$packages), packages)] saveRegistry(reg) } BatchJobs/R/getErrorMessages.R0000644000176000001440000000111212415510734015743 0ustar ripleyusers#' Get error messages of jobs. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs with errors. #' @return [\code{character}]. Error messages for jobs as character vector\cr #' \code{NA} if job has terminated successfully. #' @family debug #' @export getErrorMessages = function(reg, ids) { checkRegistry(reg) syncRegistry(reg) if (missing(ids)) ids = dbFindErrors(reg) else ids = checkIds(reg, ids) tab = dbGetErrorMsgs(reg, ids, filter = FALSE) setNames(tab$error, tab$job_id) } BatchJobs/R/loadResults.R0000644000176000001440000000566112415510734015000 0ustar ripleyusers#' Loads result files for id vector. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all done jobs. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param simplify [\code{logical(1)}]\cr #' Should the result be simplified to a vector, matrix or higher dimensional array if possible? #' Default is \code{TRUE}. #' @param use.names [\code{character(1)}]\cr #' Name the results with job ids (\dQuote{ids}), stored job names (\dQuote{names}) #' or return a unnamed result (\dQuote{none}). #' Default is \code{ids}. #' @param missing.ok [\code{logical(1)}]\cr #' If \code{FALSE} an error is thrown if the results are not found. #' Otherwise missing results are imputed to \code{NULL}. #' Default is \code{FALSE}. #' @return [\code{list}]. Results of jobs as list, possibly named by ids. #' @seealso \code{\link{reduceResults}} #' @export loadResults = function(reg, ids, part = NA_character_, simplify = FALSE, use.names = "ids", missing.ok = FALSE) { checkRegistry(reg) syncRegistry(reg) if (missing(ids)) { ids = dbFindDone(reg) } else { ids = checkIds(reg, ids) } checkPart(reg, part) assertFlag(simplify) use.names = convertUseNames(use.names) assertFlag(missing.ok) res = getResults(reg, ids, part, missing.ok) names(res) = switch(use.names, "none" = NULL, "ids" = as.character(ids), "names" = dbGetJobNames(reg, ids)) if(simplify && length(res) > 0L) res = simplify2array(res, higher = (simplify=="array")) return(res) } getResults = function(reg, ids, part = NA_character_, missing.ok = FALSE) { if (reg$multiple.result.files) { read.files = function(id, dir, pattern) { fns = list.files(dir, pattern, full.names = TRUE) found.parts = sub(".+-(.+)\\.RData$", "\\1", basename(fns)) if(length(found.parts) == 0L) { if (missing.ok) return(list()) stop("No partial result files found for job with id ", id) } setNames(lapply(fns, load2, "result"), found.parts) } dirs = getJobDirs(reg, ids) if(length(part) == 1L && is.na(part)) { patterns = sprintf("^%i-result-.+\\.RData$", ids) } else { patterns = sprintf("^%i-result-(%s)\\.RData$", ids, collapse(part, "|")) } return(mapply(read.files, id = ids, dir = dirs, pattern = patterns, SIMPLIFY = FALSE, USE.NAMES = FALSE)) } fns = getResultFilePath(reg, ids, part) miss = !file.exists(fns) if (any(miss)) { if (!missing.ok) stopf("Some job result files do not exist, showing up to first 10:\n%s", collapse(head(fns[miss], 10L), "\n")) return(replace(vector("list", length(ids)), !miss, lapply(fns[!miss], load2, "result"))) } return(lapply(fns, load2, "result")) } BatchJobs/R/setJobNames.R0000644000176000001440000000161712415510734014706 0ustar ripleyusers#' Set job names. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs. #' @param jobnames [\code{character}]\cr #' Character vector with length equal to \code{length(ids)}. #' \code{NA} removes the names stored in the registry. #' A single \code{NA} is replicated to match the length of ids provided. #' @return Named vector of job ids. #' @export setJobNames = function(reg, ids, jobnames) { checkRegistry(reg, strict = TRUE) if (missing(ids)) ids = dbGetJobIds(reg) else ids = checkIds(reg, ids) if (isScalarNA(jobnames)) jobnames = rep.int(NA_character_, length(ids)) else assertCharacter(jobnames, any.missing = FALSE) if (length(ids) != length(jobnames)) stop("Arguments 'ids' and 'jobnames' must have same length") dbSetJobNames(reg, ids, jobnames) setNames(ids, jobnames) } BatchJobs/R/sanitizePath.R0000644000176000001440000000166612415510734015143 0ustar ripleyusers#' Sanitize a path #' #' @description #' Replaces backward slashes with forward slashes and optionally #' normalizes the path. #' #' @param path [\code{character}]\cr #' Vector of paths to sanitize. #' @param make.absolute [\code{logical}]\cr #' If \code{TRUE} convert to an absolute path. #' @param normalize.absolute [\code{logical}]\cr #' Also call \code{\link[base]{normalizePath}} on absolute paths? #' This will immediately resolve symlinks. #' @return \code{character} with sanitized paths. #' @export sanitizePath = function(path, make.absolute = TRUE, normalize.absolute = FALSE) { assertCharacter(path, any.missing = FALSE) assertFlag(make.absolute) assertFlag(normalize.absolute) if (make.absolute) { normalize = if (normalize.absolute) rep.int(TRUE, length(path)) else !isPathFromRoot(path) path[normalize] = normalizePath(path[normalize], mustWork = FALSE, winslash = "/") } gsub("\\", "/", path, fixed = TRUE) } BatchJobs/R/findStatus.R0000644000176000001440000001070012502316642014610 0ustar ripleyusersfindState = function(reg, ids, fun, negate, limit) { checkRegistry(reg) syncRegistry(reg) if (!missing(ids)) ids = checkIds(reg, ids) if (!is.null(limit)) assertCount(limit) fun(reg, ids, negate, limit) } #' Find jobs depending on computional state. #' #' \code{findDone}: Find jobs which succesfully terminated. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Subset of job ids to restrict the result to. #' Default is all jobs. #' @param limit [\code{integer(1)}]\cr #' Limit the number of returned ids. #' Default is all ids. #' @return [\code{integer}]. Ids of jobs. #' @export #' @rdname findState findDone = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindDone, FALSE, limit) } #' \code{findNotDone}: Find jobs for which results are still missing. #' @export #' @rdname findState findNotDone = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindDone, TRUE, limit) } #' \code{findMissingResults}: Deprecated. Alias for findNotDone. #' @export #' @rdname findState findMissingResults = function(reg, ids, limit = NULL) { findNotDone(reg, ids, limit) } #' \code{findErrors}: Find jobs where errors occured. #' @export #' @rdname findState findErrors = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindErrors, FALSE, limit) } #' \code{findNotErrors}: Find jobs where no errors occured. #' @export #' @rdname findState findNotErrors = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindErrors, TRUE, limit) } #' \code{findTerminated}: Find jobs which have terminated (done / error). #' @export #' @rdname findState findTerminated = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindTerminated, FALSE, limit) } #' \code{findNotTerminated}: Find jobs which have not terminated (not done / no error). #' @export #' @rdname findState findNotTerminated = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindTerminated, TRUE, limit) } #' \code{findSubmitted}: Find jobs which have been submitted. #' @export #' @rdname findState findSubmitted = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindSubmitted, FALSE, limit) } #' \code{findNotSubmitted}: Find jobs which have not been submitted. #' @export #' @rdname findState findNotSubmitted = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindSubmitted, TRUE, limit) } #' \code{findOnSystem}: Find jobs which are present on the batch system at the moment. #' @export #' @rdname findState findOnSystem = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindOnSystem, FALSE, limit) } #' \code{findNotOnSystem}: Find jobs which are not present on the batch system at the moment. #' @export #' @rdname findState findNotOnSystem = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindOnSystem, TRUE, limit) } #' \code{findRunning}: Find jobs which are running. #' @export #' @rdname findState findRunning = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindRunning, FALSE, limit) } #' \code{findNotRunning}: Find jobs which are not running. #' @export #' @rdname findState findNotRunning = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindRunning, TRUE, limit) } #' \code{findStarted}: Find jobs which have been started on the batch system. #' @export #' @rdname findState findStarted = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindStarted, FALSE, limit) } #' \code{findStarted}: Find jobs which have not been started on the batch system. #' \code{findNotRunning}: Find jobs which are not running. #' @export #' @rdname findState findNotStarted = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindStarted, TRUE, limit) } #' \code{findExpired}: Find jobs where walltime was probably hit. #' Right now the heuristic is as follows: #' Find all jobs that have started, did not abort with an error, #' did not complete with a result and are not submitted or running anymore. #' Note that this heuristic does not include jobs the scheduler looses before starting. #' @export #' @rdname findState findExpired = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindExpiredJobs, FALSE, limit) } #' \code{findDisappeared}: Find jobs which disappeared from the system. #' Right now the heuristic is as follows: #' Find all jobs that are submitted but not started nor on the system anymore. #' @export #' @rdname findState findDisappeared = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindDisappeared, FALSE, limit) } BatchJobs/R/submitJobs.R0000644000176000001440000002555012465167456014635 0ustar ripleyusers#' @title Submit jobs or chunks of jobs to batch system via cluster function. #' #' @description #' If the internal submit cluster function completes successfully, the \code{retries} #' counter is set back to 0 and the next job or chunk is submitted. #' If the internal submit cluster function returns a fatal error, the submit process #' is completely stopped and an exception is thrown. #' If the internal submit cluster function returns a temporary error, the submit process #' waits for a certain time, which is determined by calling the user-defined #' \code{wait}-function with the current \code{retries} counter, the counter is #' increased by 1 and the same job is submitted again. If \code{max.retries} is #' reached the function simply terminates. #' #' Potential temporary submit warnings and errors are logged inside your file #' directory in the file \dQuote{submit.log}. #' To keep track you can use \code{tail -f [file.dir]/submit.log} in another #' terminal. #' #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Vector for job id or list of vectors of chunked job ids. #' Only corresponding jobs are submitted. Chunked jobs will get executed #' sequentially as a single job for the scheduler. #' Default is all jobs which were not yet submitted to the batch system. #' @param resources [\code{list}]\cr #' Required resources for all batch jobs. The elements of this list #' (e.g. something like \dQuote{walltime} or \dQuote{nodes} are defined by your template job file. #' Defaults can be specified in your config file. #' Default is empty list. #' @param wait [\code{function(retries)}]\cr #' Function that defines how many seconds should be waited in case of a temporary error. #' Default is exponential back-off with \code{10*2^retries}. #' @param max.retries [\code{integer(1)}]\cr #' Number of times to submit one job again in case of a temporary error #' (like filled queues). Each time \code{wait} is called to wait a certain #' number of seconds. #' Default is 10 times. #' @param chunks.as.arrayjobs [\code{logical(1)}]\cr #' If ids are passed as a list of chunked job ids, execute jobs in a chunk #' as array jobs. Note that your scheduler and your template must be adjusted to #' use this option. Default is \code{FALSE}. #' @param job.delay [\code{function(n, i)} or \code{logical(1)}]\cr #' Function that defines how many seconds a job should be delayed before it starts. #' This is an expert option and only necessary to change when you want submit #' extremely many jobs. We then delay the jobs a bit to write the submit messages as #' early as possible to avoid writer starvation. #' \code{n} is the number of jobs and \code{i} the number of the ith job. #' The default function used with \code{job.delay} set to \code{TRUE} is no delay for #' 100 jobs or less and otherwise \code{runif(1, 0.1*n, 0.2*n)}. #' If set to \code{FALSE} (the default) delaying jobs is disabled. #' @template arg_progressbar #' @return [\code{integer}]. Vector of submitted job ids. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:10) #' submitJobs(reg) #' waitForJobs(reg) #' #' # Submit the 10 jobs again, now randomized into 2 chunks: #' chunked = chunk(getJobIds(reg), n.chunks = 2, shuffle = TRUE) #' submitJobs(reg, chunked) submitJobs = function(reg, ids, resources = list(), wait, max.retries = 10L, chunks.as.arrayjobs = FALSE, job.delay = FALSE, progressbar = TRUE) { ### helper function to calculate the delay getDelays = function(cf, job.delay, n) { if (is.logical(job.delay)) { if (job.delay && n > 100L && cf$name %nin% c("Interactive", "Multicore", "SSH")) { return(runif(n, n*0.1, n*0.2)) } return(delays = rep.int(0, n)) } vnapply(seq_along(ids), job.delay, n = n) } ### argument checks on registry and ids checkRegistry(reg) syncRegistry(reg) if (missing(ids)) { ids = dbFindSubmitted(reg, negate = TRUE) if (length(ids) == 0L) { info("All jobs submitted, nothing to do!") return(invisible(integer(0L))) } } else { if (is.list(ids)) { ids = lapply(ids, checkIds, reg = reg, check.present = FALSE) dbCheckJobIds(reg, unlist(ids)) } else if(is.numeric(ids)) { ids = checkIds(reg, ids) } else { stop("Parameter 'ids' must be a integer vector of job ids or a list of chunked job ids (list of integer vectors)!") } } ### initialization of some helping vars conf = getBatchJobsConf() cf = getClusterFunctions(conf) limit.concurrent.jobs = is.finite(conf$max.concurrent.jobs) n = length(ids) ### argument checks for other parameters assertList(resources) resources = resrc(resources) if (missing(wait)) wait = function(retries) 10 * 2^retries else assertFunction(wait, "retries") if (is.finite(max.retries)) max.retries = asCount(max.retries) assertFlag(chunks.as.arrayjobs) if (chunks.as.arrayjobs && is.na(cf$getArrayEnvirName())) { warningf("Cluster functions '%s' do not support array jobs, falling back on chunks", cf$name) chunks.as.arrayjobs = FALSE } assert(checkFlag(job.delay), checkFunction(job.delay, c("n", "i"))) assertFlag(progressbar) if (!is.null(cf$listJobs)) { ### check for running jobs ids.intersect = intersect(unlist(ids), dbFindOnSystem(reg, unlist(ids))) if (length(ids.intersect) > 0L) { stopf("Some of the jobs you submitted are already present on the batch system! E.g. id=%i.", ids.intersect[1L]) } } if (limit.concurrent.jobs && (cf$name %in% c("Interactive", "Local", "Multicore", "SSH") || is.null(cf$listJobs))) { warning("Option 'max.concurrent.jobs' is enabled, but your cluster functions implementation does not support the listing of system jobs.\n", "Option disabled, sleeping 5 seconds for safety reasons.") limit.concurrent.jobs = FALSE Sys.sleep(5) } ### quick sanity check if (n > 5000L) { warningf(collapse(c("You are about to submit %i jobs.", "Consider chunking them to avoid heavy load on the scheduler.", "Sleeping 5 seconds for safety reasons."), sep = "\n"), n) Sys.sleep(5) } ### save config, start the work saveConf(reg) is.chunked = is.list(ids) info("Submitting %i chunks / %i jobs.", n, if(is.chunked) sum(viapply(ids, length)) else n) info("Cluster functions: %s.", cf$name) info("Auto-mailer settings: start=%s, done=%s, error=%s.", conf$mail.start, conf$mail.done, conf$mail.error) # use staged queries on master if fs.timeout is set # -> this way we are relatively sure that db transactions are performed in the intended order fs.timeout = conf$fs.timeout staged = conf$staged.queries && !is.na(fs.timeout) interrupted = FALSE submit.msgs = buffer(type = "list", capacity = 1000L, value = dbSendMessages, reg = reg, max.retries = 10000L, sleep = function(r) 5, staged = staged, fs.timeout = fs.timeout) logger = makeSimpleFileLogger(file.path(reg$file.dir, "submit.log"), touch = FALSE, keep = 1L) ### set on exit handler to avoid inconsistencies caused by user interrupts on.exit({ # we need the second case for errors in brew (e.g. resources) if(interrupted && exists("batch.result", inherits = FALSE)) { submit.msgs$push(dbMakeMessageSubmitted(reg, id, time = submit.time, batch.job.id = batch.result$batch.job.id, first.job.in.chunk.id = if(is.chunked) id1 else NULL, resources.timestamp = resources.timestamp)) } # send remaining msgs now info("Sending %i submit messages...\nMight take some time, do not interrupt this!", submit.msgs$pos()) submit.msgs$clear() # message the existance of the log file if (logger$getSize()) messagef("%i temporary submit errors logged to file '%s'.\nFirst message: %s", logger$getSize(), logger$getLogfile(), logger$getMessages(1L)) }) ### write R scripts info("Writing %i R scripts...", n) resources.timestamp = saveResources(reg, resources) rscripts = writeRscripts(reg, cf, ids, chunks.as.arrayjobs, resources.timestamp, disable.mail = FALSE, delays = getDelays(cf, job.delay, n)) waitForFiles(rscripts, timeout = fs.timeout) ### reset status of jobs: delete errors, done, ... dbSendMessage(reg, dbMakeMessageKilled(reg, unlist(ids), type = "first"), staged = staged, fs.timeout = fs.timeout) ### initialize progress bar bar = getProgressBar(progressbar, max = n, label = "SubmitJobs") bar$set() tryCatch({ for (i in seq_along(ids)) { id = ids[[i]] id1 = id[1L] retries = 0L repeat { # max.retries may be Inf if (limit.concurrent.jobs && length(cf$listJobs(conf, reg)) >= conf$max.concurrent.jobs) { # emulate a temporary erroneous batch result batch.result = makeSubmitJobResult(status = 10L, batch.job.id = NA_character_, "Max concurrent jobs exhausted") } else { # try to submit the job interrupted = TRUE submit.time = now() batch.result = cf$submitJob( conf = conf, reg = reg, job.name = sprintf("%s-%i", reg$id, id1), rscript = rscripts[i], log.file = getLogFilePath(reg, id1), job.dir = getJobDirs(reg, id1), resources = resources, arrayjobs = if(chunks.as.arrayjobs) length(id) else 1L ) } ### validate status returned from cluster functions if (batch.result$status == 0L) { submit.msgs$push(dbMakeMessageSubmitted(reg, id, time = submit.time, batch.job.id = batch.result$batch.job.id, first.job.in.chunk.id = if(is.chunked) id1 else NULL, resources.timestamp = resources.timestamp)) interrupted = FALSE bar$inc(1L) break } ### submitJob was not successful, handle the return status interrupted = FALSE if (batch.result$status > 0L && batch.result$status <= 100L) { if (is.finite(max.retries) && retries > max.retries) stopf("Retried already %i times to submit. Aborting.", max.retries) # temp error, wait and increase retries, then submit again in next iteration Sys.sleep(wait(retries)) # log message to file logger$log(batch.result$msg) retries = retries + 1L } else if (batch.result$status > 100L && batch.result$status <= 200L) { # fatal error, abort at once stopf("Fatal error occured: %i. %s", batch.result$status, batch.result$msg) } else { # illeagal status code stopf("Illegal status code %s returned from cluster functions!", batch.result$status) } } } }, error = bar$error) ### return ids (on.exit handler kicks now in to submit the remaining messages) return(invisible(ids)) } BatchJobs/R/batchReduceResults.R0000644000176000001440000000624612465167456016310 0ustar ripleyusers#' Reduces results via a binary function and adds jobs for this to a registry. #' #' @description #' Each jobs reduces a certain number of results on one slave. #' You can then submit these jobs to the batch system. #' Later, you can do a final reduction with \code{\link{reduceResults}} on the master. #' @param reg [\code{\link{Registry}}]\cr #' Registry whose results should be reduced by \code{fun}. #' @param reg2 [\code{\link{Registry}}]\cr #' Empty registry that should store the job for the mapping. #' @param fun [\code{function(aggr, job, res, ...)}]\cr #' Function to reduce results with. #' @param ids [\code{integer}]\cr #' Ids of jobs whose results should be reduced with \code{fun}. #' Default is all jobs. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param init [any]\cr #' Initial object for reducing. #' @param block.size [\code{integer(1)}]\cr #' Number of results reduced in one job. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @return Vector of type \code{integer} with job ids. #' @export #' @examples #' # generating example results: #' reg1 = makeRegistry(id = "BatchJobsExample1", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg1, f, 1:20) #' submitJobs(reg1) #' waitForJobs(reg1) #' #' # define function to reduce on slave, we want to sum the squares #' myreduce = function(aggr, job, res) aggr + res #' #' # sum 5 results on each slave process, i.e. 4 jobs #' reg2 = makeRegistry(id = "BatchJobsExample2", file.dir = tempfile(), seed = 123) #' batchReduceResults(reg1, reg2, fun = myreduce, init = 0, block.size = 5) #' submitJobs(reg2) #' waitForJobs(reg2) #' #' # now reduce one final time on master #' reduceResults(reg2, fun = myreduce) batchReduceResults = function(reg, reg2, fun, ids, part = NA_character_, init, block.size, more.args = list()) { checkRegistry(reg) checkRegistry(reg2) syncRegistry(reg) syncRegistry(reg2) assertFunction(fun, c("aggr", "job", "res")) if (missing(ids)) { ids = dbGetJobIdsIfAllDone(reg) } else { ids = checkIds(reg, ids) if (length(dbFindDone(reg, ids, negate = TRUE, limit = 1L)) > 0L) stop("Not all jobs with corresponding ids finished (yet)!") } block.size = asCount(block.size) checkMoreArgs(more.args, reserved = c("..reg", "..fun", "..part")) if (dbGetJobCount(reg2) > 0L) stop("Registry 'reg2' is not empty!") if(reg$file.dir == reg2$file.dir) stop("Both registries cannot point to the same file dir. Files would get overwritten!") reg2$packages = insert(reg2$packages, reg$packages) saveRegistry(reg2) batchReduce(reg2, batchReduceResultsWrapper, ids, init = init, block.size = block.size, more.args = c(more.args, list(..reg = reg, ..fun = fun, ..part = part))) } batchReduceResultsWrapper = function(aggr, x, ..reg, ..fun, ..part) { # x is id # use lazy evaluation, if fun doesn't access job or res (unlikely) ..fun(aggr = aggr, job = getJob(..reg, x, check.id = FALSE), res = getResult(..reg, x, ..part)) } BatchJobs/R/clusterFunctionsMulticore.R0000644000176000001440000000647012471624612017737 0ustar ripleyusers#' Use multiple cores on local Linux machine to spawn parallel jobs. #' #' @description #' Jobs are spawned by starting multiple R sessions on the commandline #' (similar like on true batch systems). #' Packages \code{parallel} or \code{multicore} are not used in any way. #' #' @param ncpus [\code{integer(1)}]\cr #' Number of VPUs of worker. #' Default is to use all cores but one, where total number of cores #' "available" is given by option \code{\link[base:options]{mc.cores}} #' and if that is not set it is inferred by #' \code{\link[parallel]{detectCores}}. #' @param max.jobs [\code{integer(1)}]\cr #' Maximal number of jobs that can run concurrently for the current registry. #' Default is \code{ncpus}. #' @param max.load [\code{numeric(1)}]\cr #' Load average (of the last 5 min) at which the worker is considered occupied, #' so that no job can be submitted. #' Default is \code{ncpus-1}. #' @param nice [\code{integer(1)}]\cr #' Process priority to run R with set via nice. Integers between -20 and 19 are allowed. #' If missing, processes are not nice'd and the system default applies (usually 0). #' @param r.options [\code{character}] #' Options for R and Rscript, one option per element of the vector, #' a la \dQuote{--vanilla}. #' Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}. #' @param script [\code{character(1)}]\cr #' Path to helper bash script which interacts with the worker. #' You really should not have to touch this, as this would imply that we have screwed up and #' published an incompatible version for your system. #' This option is only provided as a last resort for very experienced hackers. #' Note that the path has to be absolute. #' This is what is done in the package: #' \url{https://github.com/tudo-r/BatchJobs/blob/master/inst/bin/linux-helper} #' Default means to take it from package directory. #' @return [\code{\link{ClusterFunctions}}]. #' @family clusterFunctions #' @export #' @importFrom parallel detectCores makeClusterFunctionsMulticore = function(ncpus = max(getOption("mc.cores", detectCores()) - 1, 1), max.jobs, max.load, nice, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script) { if (isWindows()) stop("ClusterFunctionsMulticore do not work in Windows") worker = makeWorkerLocalLinux(r.options, script, ncpus, max.jobs, max.load, nice) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { updateWorker(worker, reg$file.dir, tdiff = 0L) s = worker$available if (s != "A") { makeSubmitJobResult(status = 1L, batch.job.id = NULL, msg = sprintf("Multicore busy: %s", s)) } else { pid = try(startWorkerJob(worker, rscript, log.file)) if (is.error(pid)) makeSubmitJobResult(status = 101L, batch.job.id = NULL, msg = "Submit failed.") else makeSubmitJobResult(status = 0L, batch.job.id = pid) } } killJob = function(conf, reg, batch.job.id) { killWorkerJob(worker, batch.job.id) } listJobs = function(conf, reg) { listWorkerJobs(worker, reg$file.dir) } getArrayEnvirName = function() NA_character_ makeClusterFunctions(name = "Multicore", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/Exports.R0000644000176000001440000000575012465170614014146 0ustar ripleyusers#' @title Load exported R data objects. #' #' @description #' Loads exported \code{RData} object files in the \dQuote{exports} subdirectory of your \code{file.dir} #' and assigns the objects to the global environment. #' #' @template arg_reg #' @param what [\code{character}]\cr #' Names of objects and corresponding \dQuote{RData} files which should be loaded. #' Default \code{NULL} loads all files. #' @return [\code{character}]. Invisibly returns a character vector of loaded objects. #' @family exports #' @export loadExports = function(reg, what = NULL) { checkRegistry(reg) if (!is.null(what)) assertCharacter(what, any.missing = FALSE) f = fail(getExportDir(reg$file.dir), extension = "RData", simplify = FALSE) keys = f$ls() if (!is.null(what)) keys = intersect(keys, what) if (length(keys) > 0L) { messagef("Loading RData files: %s", collapse(keys)) f$assign(keys, envir = .GlobalEnv) } invisible(keys) } #' @title Export R object to be available on the slaves. #' #' @description #' Saves objects as \code{RData} files in the \dQuote{exports} subdirectory of your \code{file.dir} #' to be later loaded on the slaves. #' #' @template arg_reg #' @param ... [any]\cr #' Objects to export. You must provide a valid name. #' @param li [\code{list}]\cr #' More objects to export provided as a named list. #' @param overwrite [\code{logical(1)}]\cr #' If set to \code{FALSE} (default), exported objects are protected from being overwritten #' by multiple calls of this function. Setting this to \code{TRUE} disables this check. #' @return [\code{character}]. Invisibly returns a character vector of exported objects. #' @family exports #' @export batchExport = function(reg, ..., li = list(), overwrite = FALSE) { checkRegistry(reg) ddd = list(...) assertList(li, names = "strict") assertList(ddd, names = "strict") assertFlag(overwrite) keys = c(names(li), names(ddd)) dup = anyDuplicated(keys) if (dup > 0L) stopf("Object for export provided more than once: '%s'", keys[dup]) f = fail(getExportDir(reg$file.dir), extension = "RData", simplify = FALSE) if (!overwrite) { collision = which.first(keys %in% f$ls()) if (length(collision) > 0L) stopf("Object named '%s' already exported and 'overwrite' is set to FALSE", keys[collision]) } f$put(li = li) f$put(li = ddd) invisible(keys) } #' @title Unload exported R objects. #' #' @description #' Removes \code{RData} files from the \dQuote{exports} subdirectory of your \code{file.dir} #' and thereby prevents loading on the slave. #' #' @template arg_reg #' @param what [\code{character}]\cr #' Names of objects to remove. #' @return [\code{character}]. Invisibly returns a character vector of unexported objects. #' @family exports #' @export batchUnexport = function(reg, what) { checkRegistry(reg) assertCharacter(what, any.missing = FALSE) f = fail(getExportDir(reg$file.dir), extension = "RData", simplify = FALSE) keys = intersect(f$ls(), what) f$remove(keys) invisible(keys) } BatchJobs/R/syncRegistry.R0000644000176000001440000000224312415510734015175 0ustar ripleyuserssyncRegistry = function(reg) { conf = getBatchJobsConf() if (conf$staged.queries) { if (conf$debug && isOnSlave()) stop("SQL query sent from Worker") fns = lsort(list.files(getPendingDir(reg$file.dir), full.names = TRUE)) if (length(fns) == 0L) return(invisible(TRUE)) info("Syncing registry ...") queries = lapply(fns, readSQLFile) ok = !vlapply(queries, isFALSE) tryCatch(dbDoQueries(reg, unlist(queries[ok]), "rw"), error = function(e) stopf("Error syncing registry (%s)", e)) fns = fns[ok] ok = file.remove(fns) if (!all(ok)) warningf("Some pending result sets could not be removed, e.g. '%s'", head(fns[ok], 1L)) } invisible(TRUE) } readSQLFile = function(con) { x = try(readLines(con), silent = TRUE) n = length(x) if (is.error(x) || n == 0L || x[n] != "--EOF--") return(FALSE) x = x[-n] substr(x, 1L, nchar(x) - 1L) } writeSQLFile = function(x, con) { writeLines(c(paste0(x, ";"), "--EOF--"), con = con) } useStagedQueries = function() { getBatchJobsConf()$staged.queries } .OrderChars = setNames(letters[1L:6L], c("first", "submitted", "started", "done", "error", "last")) BatchJobs/R/getLogFiles.R0000644000176000001440000000100512415510734014667 0ustar ripleyusers#' Get log file paths for jobs. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs. #' @return [\code{character}]. Vector of file paths to log files. #' @family debug #' @export getLogFiles = function(reg, ids) { checkRegistry(reg) syncRegistry(reg) if (missing(ids)) ids = getJobIds(reg) else ids = checkIds(reg, ids) fids = dbGetFirstJobInChunkIds(reg, ids) getLogFilePath(reg, ifelse(is.na(fids), ids, fids)) } BatchJobs/R/sourceRegistryFiles.R0000644000176000001440000000751412415510734016512 0ustar ripleyusers#' Source registry files #' #' @description #' Sources all files found in \code{src.dirs} and specified via \code{src.files}. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param envir [\code{environment}]\cr #' Environment to source the files into. Default is the global environment. #' @return Nothing. #' @export sourceRegistryFiles = function(reg, envir = .GlobalEnv) { checkRegistry(reg) assertEnvironment(envir) sourceRegistryFilesInternal(reg$work.dir, reg$src.dirs, reg$src.files) } sourceRegistryFilesInternal = function(work.dir, dirs, files, envir = .GlobalEnv) { # add work dir if not /foo/bar path w = !isPathFromRoot(files) files[w] = file.path(work.dir, files[w]) w = which.first(!file.exists(files)) if (length(w)) stopf("Files to source not found, e.g. %s", files[w]) w = !isPathFromRoot(dirs) dirs[w] = file.path(work.dir, dirs[w]) w = which.first(!isDirectory(dirs)) if (length(w)) stopf("Directories to source not found, e.g. %s", dirs[w]) lapply(c(getRScripts(dirs), files), sys.source, envir = envir) invisible(TRUE) } getRScripts = function(dirs) { if (length(dirs)) { ok = isDirectory(dirs) if (any(!ok)) stopf("Directories not found: %s", collapse(dirs[!ok])) unlist(lapply(dirs, list.files, pattern = "\\.[Rr]$", full.names = TRUE)) } else { character(0L) } } #' @title Add source files to registry. #' #' @description #' Mutator function for \code{src.files} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param src.files [\code{character}]\cr #' Paths to add to registry. #' See \code{\link{makeRegistry}}. #' @param src.now [\code{logical(1)}] #' Source files now on master? #' Default is \code{TRUE}. #' @template ret_reg_mut #' @family exports #' @export addRegistrySourceFiles = function(reg, src.files, src.now = TRUE) { checkRegistry(reg) assertCharacter(src.files, any.missing = FALSE) assertFlag(src.now) src.files = sanitizePath(src.files, make.absolute = FALSE) if (src.now) sourceRegistryFilesInternal(reg$work.dir, character(0L), src.files) reg$src.files = union(reg$src.files, src.files) saveRegistry(reg) } #' @title Add source dirs to registry. #' #' @description #' Mutator function for \code{src.dirs} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param src.dirs [\code{character}]\cr #' Paths to add to registry. #' See \code{\link{makeRegistry}}. #' @param src.now [\code{logical(1)}] #' Source files now on master? #' Default is \code{TRUE}. #' @template ret_reg_mut #' @family exports #' @export addRegistrySourceDirs = function(reg, src.dirs, src.now = TRUE) { checkRegistry(reg) assertCharacter(src.dirs, any.missing = FALSE) src.dirs = sanitizePath(src.dirs, make.absolute = FALSE) if (src.now) sourceRegistryFilesInternal(reg$work.dir, src.dirs, character(0L)) reg$src.dirs = c(reg$src.dirs, src.dirs) saveRegistry(reg) } #' @title Remove source files from registry. #' #' @description #' Mutator function for \code{src.files} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param src.files [\code{character}]\cr #' Paths to remove from registry. #' @template ret_reg_mut #' @family exports #' @export removeRegistrySourceFiles = function(reg, src.files) { checkRegistry(reg) assertCharacter(src.files, any.missing = FALSE) reg$src.files = setdiff(reg$src.files, src.files) saveRegistry(reg) } #' @title Remove packages from registry. #' #' @description #' Mutator function for \code{src.dirs} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param src.dirs [\code{character}]\cr #' Paths to remove from registry. #' @template ret_reg_mut #' @family exports #' @export removeRegistrySourceDirs = function(reg, src.dirs) { checkRegistry(reg) assertCharacter(src.dirs, any.missing = FALSE) reg$src.dirs = setdiff(reg$src.dirs, src.dirs) saveRegistry(reg) } BatchJobs/R/showStatus.R0000644000176000001440000000542712471624612014666 0ustar ripleyusers#' @title Retrieve or show status information about jobs. #' #' @description #' E.g.: How many there are, how many are done, any errors, etc. #' \code{showStatus} displays on the console, \code{getStatus} returns an informative result #' without console output. # #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of selected jobs. #' Default is all jobs. #' @param errors [\code{integer(1)}]\cr #' How many of the error messages should be displayed if any errors occured in the jobs? #' Default is 10. #' @param run.and.exp [\code{logical(1)}]\cr #' Show running and expired jobs? #' Requires to list the job on the batch system. If not possible, because #' that cluster function is not avilable, this option is ignored anyway. #' Default is \code{TRUE}. #' @return [\code{list}]. List of absolute job numbers. \code{showStatus} returns them #' invisibly. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:10) #' submitJobs(reg) #' waitForJobs(reg) #' #' # should show 10 submitted jobs, which are all done. #' showStatus(reg) showStatus = function(reg, ids, run.and.exp = TRUE, errors = 10L) { errors = asCount(errors) stats = getStatus(reg, ids = ids, run.and.exp = run.and.exp) procent = function(x, n) { if(is.na(x)) return("") p = ifelse(n == 0L, 0, x / n * 100) sprintf("(%6.2f%%)", p) } output = collapse(c("Status for %%i jobs at %%s", "Submitted: %%%1$ii %%s", "Started: %%%1$ii %%s", "Running: %%%1$ii %%s", "Done: %%%1$ii %%s", "Errors: %%%1$ii %%s", "Expired: %%%1$ii %%s", "Time: min=%%.2fs avg=%%.2fs max=%%.2fs"), "\n") output = sprintf(output, min(4L, nchar(sprintf("%i", stats$n + 1L)))) with(stats, catf(output, n, Sys.time(), submitted, procent(submitted, n), started, procent(started, n), running, procent(running, n), done, procent(done, n), error, procent(error, n), expired, procent(expired, n), t_min, t_avg, t_max)) m = min(errors, stats$error) if(m > 0L) { msgs = dbGetErrorMsgs(reg, ids, filter = TRUE, limit = m) catf("\nShowing first %i errors:", m) cat(sprintf("Error in %i: %s", msgs$job_id, msgs$error), sep = "\n") } return(invisible(stats)) } #' @rdname showStatus #' @export getStatus getStatus = function(reg, ids, run.and.exp = TRUE) { checkRegistry(reg) syncRegistry(reg) if (!missing(ids)) ids = checkIds(reg, ids) run.and.exp = run.and.exp && !is.null(getListJobs()) dbGetStats(reg, ids, running = run.and.exp, expired = run.and.exp, times = TRUE) } BatchJobs/R/sendMail.R0000644000176000001440000000346612415510734014234 0ustar ripleyuserssendMail = function(reg, ids, result.str, extra.msg = "", disable.mail, condition, first, last, conf) { if (disable.mail) return(invisible(NULL)) conf = getBatchJobsConf() ischunk = (length(ids) > 1L) first.id = if(ischunk) ids[[1L]] else ids # should we mail mail.conds = list(start = conf$mail.start, done = conf$mail.done, error = conf$mail.error) mail.cond = mail.conds[[condition]] if (mail.cond == "all" || (mail.cond %in% c("first", "first+last") && first.id == first) || (mail.cond %in% c("last", "first+last") && first.id == last)) { myformat = function(title, lines) { collapse(c(sprintf("### %s", title), lines, "", ""), "\n") } cstr = switch(condition, "start" = "started", "done" = "finished", "error" = "terminated with exception") subj = sprintf("[%s]: %s %s has %s", reg$id, ifelse(ischunk, "Chunk with first job ", "Job"), first.id, cstr) msg = myformat("Ids", ids) # append result and status information if (condition != "start") { if (extra.msg != "") msg = paste0(msg, myformat("Message", extra.msg)) msg = paste0(msg, myformat("Results", result.str)) if(first.id == last) msg = paste0(msg, myformat("Status", capture.output(showStatus(reg, run.and.exp = FALSE)))) } # if a mail problem occurs, we only warn but do not terminate ok = try (sendmail(conf$mail.from, conf$mail.to, subj, msg, control = conf$mail.control)) if (is.error(ok)) { warningf("Could not send mail to signal condition '%s'!\nFrom: %s\nTo: %s\nControl: %s\nError message: %s", condition, conf$mail.from, conf$mail.to, convertToShortString(conf$mail.control), as.character(ok)) } else { messagef("Mail signaling condition '%s' send to %s", condition, conf$mail.to) } } invisible(NULL) } BatchJobs/R/clusterFunctionsSLURM.R0000644000176000001440000000525012465167456016704 0ustar ripleyusers#' @title Create cluster functions for SLURM-based systems. #' #' @description #' Job files are created based on the brew template #' \code{template.file}. This file is processed with brew and then #' submitted to the queue using the \code{sbatch} command. Jobs are #' killed using the \code{scancel} command and the list of running jobs #' is retrieved using \code{squeue}. The user must have the #' appropriate privileges to submit, delete and list jobs on the #' cluster (this is usually the case). #' #' The template file can access all arguments passed to the #' \code{submitJob} function, see here \code{\link{ClusterFunctions}}. #' It is the template file's job to choose a queue for the job #' and handle the desired resource allocations. #' Examples can be found on #' \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfSLURM}. #' #' @template arg_template #' @template arg_list_jobs_cmd #' @template ret_cf #' @family clusterFunctions #' @export makeClusterFunctionsSLURM = function(template.file, list.jobs.cmd = c("squeue", "-h", "-o %i", "-u $USER")) { assertCharacter(list.jobs.cmd, min.len = 1L, any.missing = FALSE) template = cfReadBrewTemplate(template.file) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { outfile = cfBrewTemplate(conf, template, rscript, "sb") res = runOSCommandLinux("sbatch", outfile, stop.on.exit.code = FALSE) max.jobs.msg = "sbatch: error: Batch job submission failed: Job violates accounting policy (job submit limit, user's size and/or time limits)" temp.error = "Socket timed out on send/recv operation" output = collapse(res$output, sep = "\n") if (grepl(max.jobs.msg, output, fixed = TRUE)) { makeSubmitJobResult(status = 1L, batch.job.id = NA_character_, msg = max.jobs.msg) } else if (grepl(temp.error, output, fixed = TRUE)) { # another temp error we want to catch makeSubmitJobResult(status = 2L, batch.job.id = NA_character_, msg = temp.error) } else if (res$exit.code > 0L) { cfHandleUnknownSubmitError("sbatch", res$exit.code, res$output) } else { makeSubmitJobResult(status = 0L, batch.job.id = str_trim(strsplit(output, split = " ")[[1L]][4L])) } } killJob = function(conf, reg, batch.job.id) { cfKillBatchJob("scancel", batch.job.id) } listJobs = function(conf, reg) { # Result is lines of fully quantified batch.job.ids runOSCommandLinux(list.jobs.cmd[1L], list.jobs.cmd[-1L])$output } getArrayEnvirName = function() "SLURM_ARRAY_TASK_ID" makeClusterFunctions(name = "SLURM", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/installPackagesOnSSHWorkers.R0000644000176000001440000000263012437616146020035 0ustar ripleyusers#' Install packages on SSH workers. #' #' @description #' Installation is done via \code{\link{callFunctionOnSSHWorkers}} #' and \code{\link{install.packages}}. #' #' Note that as usual the function tries to install #' the packages into the first path of \code{.libPaths()} #' of each each worker. #' #' @param nodenames [\code{character}]\cr #' Nodenames of workers. #' @param pkgs [\code{character}]\cr #' See \code{\link{install.packages}}. #' @param repos [\code{character}]\cr #' See \code{\link{install.packages}}. #' If the user must be queried this is of course done on the master. #' @param consecutive [\code{logical(1)}]\cr #' See \code{\link{callFunctionOnSSHWorkers}}. #' Default is \code{TRUE}. #' @param show.output [\code{logical(1)}]\cr #' See \code{\link{callFunctionOnSSHWorkers}}. #' Default is \code{consecutive}. #' @param ... [any]\cr #' Passed to \code{\link{install.packages}}. #' @return Nothing. #' @export #' @seealso \code{\link{callFunctionOnSSHWorkers}} installPackagesOnSSHWorkers = function(nodenames, pkgs, repos = getOption("repos"), consecutive = TRUE, show.output = consecutive, ...) { assertCharacter(pkgs, min.len = 1L, any.missing = FALSE) if (repos == "@CRAN@") repos = chooseCRANmirror() callFunctionOnSSHWorkers(nodenames, fun = install.packages, pkgs = pkgs, repos = repos, consecutive = consecutive, show.output = show.output, ...) invisible(NULL) } BatchJobs/R/getJob.R0000644000176000001440000000205312415510734013701 0ustar ripleyusers#' Get job from registry by id. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param id [\code{integer(1)}]\cr #' Id of job. #' @param check.id [\code{logical(1)}]\cr #' Check the job id? #' Default is \code{TRUE}. #' @return [\code{\link{Job}}]. #' @export getJob = function(reg, id, check.id = TRUE) { if (check.id) id = checkId(reg, id) getJobs(reg, id, check.ids = FALSE)[[1L]] } #' Get jobs from registry by id. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs. #' @param check.ids [\code{logical(1)}]\cr #' Check the job ids? #' Default is \code{TRUE}. #' @return [list of \code{\link{Job}}]. #' @export getJobs = function(reg, ids, check.ids = TRUE) { checkRegistry(reg) # syncRegistry(reg) NOT! assertFlag(check.ids) UseMethod("getJobs") } #' @method getJobs Registry #' @export getJobs.Registry = function(reg, ids, check.ids = TRUE) { if (! missing(ids) && check.ids) { ids = checkIds(reg, ids) } dbGetJobs(reg, ids) } BatchJobs/R/clusterFunctionsSGE.R0000644000176000001440000000537512415510734016412 0ustar ripleyusers#' @title Create cluster functions for Sun Grid Engine systems. #' #' @description #' Job files are created based on the brew template #' \code{template.file}. This file is processed with brew and then #' submitted to the queue using the \code{qsub} command. Jobs are #' killed using the \code{qdel} command and the list of running jobs #' is retrieved using \code{qselect}. The user must have the #' appropriate privileges to submit, delete and list jobs on the #' cluster (this is usually the case). #' #' The template file can access all arguments passed to the #' \code{submitJob} function, see here \code{\link{ClusterFunctions}}. #' It is the template file's job to choose a queue for the job #' and handle the desired resource allocations. #' Examples can be found on #' \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfSGE}. #' #' @template arg_template #' @template arg_list_jobs_cmd #' @template ret_cf #' @family clusterFunctions #' @export makeClusterFunctionsSGE = function(template.file, list.jobs.cmd = c("qstat", "-u $USER")) { assertCharacter(list.jobs.cmd, min.len = 1L, any.missing = FALSE) template = cfReadBrewTemplate(template.file) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { outfile = cfBrewTemplate(conf, template, rscript, "job") # returns: "Your job 240933 (\"sleep 60\") has been submitted" res = runOSCommandLinux("qsub", outfile, stop.on.exit.code = FALSE) # FIXME filled queues if (res$exit.code > 0L) { cfHandleUnknownSubmitError("qsub", res$exit.code, res$output) } else { # collapse output strings and first number in string is batch.job.id batch.job.id = str_extract(collapse(res$output, sep = " "), "\\d+") makeSubmitJobResult(status = 0L, batch.job.id = batch.job.id) } } killJob = function(conf, reg, batch.job.id) { cfKillBatchJob("qdel", batch.job.id) } listJobs = function(conf, reg) { # looks like this # job-ID prior name user state submit/start at queue slots ja-task-ID #----------------------------------------------------------------------------------------------------------------- # 240935 0.00000 sleep 60 matthias qw 04/03/2012 15:45:54 1 # res = runOSCommandLinux("qstat", "-u $USER") res = runOSCommandLinux(list.jobs.cmd[1L], list.jobs.cmd[-1L]) # drop first 2 header lines out = tail(res$output, -2L) # first number in strings are batch.job.ids str_extract(out, "\\d+") } getArrayEnvirName = function() "SGE_TASK_ID" makeClusterFunctions(name = "SGE", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/loadResult.R0000644000176000001440000000170112415510734014604 0ustar ripleyusers#' Loads a specific result file. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param id [\code{integer(1)}]\cr #' Id of job. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param missing.ok [\code{logical(1)}]\cr #' If \code{FALSE} an error is thrown if no result file is found. #' Otherwise \code{NULL} is returned. #' Default is \code{FALSE}. #' @return [any]. Result of job. #' @seealso \code{\link{reduceResults}} #' @export loadResult = function(reg, id, part = NA_character_, missing.ok = FALSE) { checkRegistry(reg) syncRegistry(reg) id = checkId(reg, id) checkPart(reg, part) assertFlag(missing.ok) getResult(reg, id, part, missing.ok) } getResult = function(reg, id, part = NA_character_, missing.ok = FALSE) { getResults(reg, id, part, missing.ok)[[1L]] } BatchJobs/R/doJob.R0000644000176000001440000000765412423457121013537 0ustar ripleyusersdoJob = function(reg, ids, multiple.result.files, disable.mail, first, last, array.id) { saveOne = function(result, name) { fn = getResultFilePath(reg, job$id, name) message("Writing result file: ", fn) save2(file = fn, result = result) } getMemoryUsage = function() { sum(gc()[, 6L]) } # Get the conf loadConf(reg) conf = getBatchJobsConf() # Say hi. messagef("%s: Starting job on node %s.", Sys.time(), Sys.info()["nodename"]) messagef("Auto-mailer settings: start=%s, done=%s, error=%s.", conf$mail.start, conf$mail.done, conf$mail.error) # We need to see all warnings immediatly if (conf$cluster.functions$name != "Testing") options(warning.length = 8170L, warn = 1L + conf$raise.warnings) # Go to the work.dir wd = switchWd(reg) on.exit({ wd$reset() message("Memory usage according to gc:") print(gc()) }) if (!is.na(array.id)) { # FIXME better send error to database here, we don't see those errors on the master :( array.id = asInt(as.integer(array.id), lower = 1L, upper = length(ids)) messagef("Processing array id %s", array.id) ids = ids[array.id] } staged = conf$staged.queries n = length(ids) results = character(n) error = logical(n) mail.extra.msg = "" cache = makeFileCache(use.cache = n > 1L) # notify status sendMail(reg, ids, results, "", disable.mail, condition = "start", first, last) # create buffer of started messages msg.buf = buffer(init = lapply(ids, dbMakeMessageStarted, reg = reg), capacity = 2L * n, value = TRUE) next.flush = 0L for (i in seq_len(n)) { now = now() if (now > next.flush) { if (dbSendMessages(reg, msg.buf$get(), staged = staged)) msg.buf$clear() next.flush = now + runif(1L, 300, 600) } job = getJob(reg, ids[i], check.id = FALSE) messagef("########## Executing jid=%s ##########", job$id) messagef("Timestamp: %s" , Sys.time()) print(job) message("Setting seed: ", job$seed) seed = seeder(reg, job$seed) gc(reset = TRUE) result = try(applyJobFunction(reg, job, cache), silent = TRUE) mem.used = getMemoryUsage() seed$reset() catf("Result:") print(str(result, max.level = 1L, list.len = 5L)) error[i] = is.error(result) if (error[i]) { results[i] = as.character(result) } else if (multiple.result.files) { if (!is.list(result)) { results[i] = "multiple.result.files is TRUE, but your algorithm did not return a list!" error[i] = TRUE } else if (!isProperlyNamed(result)) { results[i] = "multiple.result.files is TRUE, but some the returned lists is not fully, distinctly named!" error[i] = TRUE } } if (error[i]) { msg.buf$push(dbMakeMessageError(reg, job$id, err.msg = results[i], memory = mem.used)) message("Error occurred: ", results[i]) } else { results[i] = paste0(capture.output(str(result)), collapse = "\n") msg.buf$push(dbMakeMessageDone(reg, job$id, memory = mem.used)) if (multiple.result.files) { Map(saveOne, result = result, name = names(result)) } else { saveOne(result, NA_character_) } } } # try to flush the remaining msgs at the end for (i in seq_len(10L)) { if (dbSendMessages(reg, msg.buf$get(), staged = staged)) { msg.buf$clear() break } Sys.sleep(round(runif(1L, 30, 120))) } # check if there are still remaining messages if (!msg.buf$empty()) { mail.extra.msg = paste("Some DB messages could not be flushed.", "This indicates some DB problem or too much communication with the DB.", "Everything should still be ok, you only might have to resubmit some jobs as they are not recorded as 'done'.", sep = "\n") warningf(mail.extra.msg) } sendMail(reg, ids, results, mail.extra.msg, disable.mail, condition = ifelse(any(error), "error", "done"), first, last) messagef("%s: All done.", Sys.time()) return(!any(error)) } BatchJobs/R/testJob.R0000644000176000001440000000770412437616146014121 0ustar ripleyusers#' Tests a job by running it with Rscript in a new process. #' #' @description #' Useful for debugging. #' Note that neither the registry, database or file directory are changed. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param id [\code{integer(1)}]\cr #' Id of job to test. #' Default is first job id of registry. #' @param external [\code{logical(1)}]\cr #' Run test in an independent external R session instead of current. #' The former allows to uncover missing variable definitions (which may #' accidentially be defined in the current global environment) and the latter #' is useful to get traceable execeptions. #' Default is \code{TRUE}. #' @param resources [\code{list}]\cr #' Usually not needed, unless you call the function \code{\link{getResources}} in your job. #' See \code{\link{submitJobs}}. #' Default is empty list. #' @return [any]. Result of job. If the job did not complete because of an error, NULL is returned. #' @family debug #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) if (x==1) stop("oops") else x #' batchMap(reg, f, 1:2) #' testJob(reg, 2) testJob = function(reg, id, resources = list(), external = TRUE) { checkRegistry(reg) #syncRegistry(reg) if (missing(id)) { id = dbGetJobId(reg) if (length(id) == 0L) stop("Registry is empty!") messagef("Testing job with id=%i ...", id) } else { id = checkId(reg, id) } assertList(resources) resources = resrc(resources) assertFlag(external) if (external) { # we dont want to change anything in the true registry / file dir / DB # so we have to copy stuff a little bit r = reg # get a unique, unused tempdir. tmpdir() always stays the same per session td = tempfile(pattern = "") construct = sprintf("make%s", class(r)[1L]) # copy reg reg = do.call(construct, list(id = reg$id, seed = r$seed, file.dir = td, work.dir = r$work.dir, sharding = FALSE, multiple.result.files = r$multiple.result.files, packages = names(reg$packages), src.dirs = reg$src.dirs, src.files = reg$src.files)) # copy DB file.copy(from = file.path(r$file.dir, "BatchJobs.db"), to = file.path(td, "BatchJobs.db"), overwrite = TRUE) # copy conf saveConf(reg) # copy job stuff copyRequiredJobFiles(r, reg, id) # copy exports file.copy(from = getExportDir(r$file.dir), to = td, recursive = TRUE) # write r script resources.timestamp = saveResources(reg, resources) writeRscripts(reg, getClusterFunctions(getBatchJobsConf()), id, chunks.as.arrayjobs = FALSE, resources.timestamp = resources.timestamp, disable.mail = TRUE, delays = 0) # execute now = Sys.time() message("### Output of new R process starts here ###") system3(file.path(R.home("bin"), "Rscript"), getRScriptFilePath(reg, id), wait = TRUE) message("### Output of new R process ends here ###") dt = difftime(Sys.time(), now) messagef("### Approximate running time: %.2f %s", as.double(dt), units(dt)) res = try(getResult(reg, id)) if (is.error(res)) return(NULL) } else { setOnSlave(TRUE) on.exit(setOnSlave(FALSE)) # FIXME: stuff we might need to store before: resources saveConf(reg) # trigger loadExports, sourceRegistryFiles, ... loadRegistry(reg$file.dir) res = applyJobFunction(reg, getJob(reg, id), makeFileCache(FALSE)) } return(res) } #' ONLY FOR INTERNAL USAGE. #' @param reg1 [\code{\link{Registry}}]\cr #' Source registry. #' @param reg2 [\code{\link{Registry}}]\cr #' Detination registry. #' @param id [\code{character(1)}]\cr #' Job id. #' @return Nothing. #' @keywords internal #' @export copyRequiredJobFiles = function(reg1, reg2, id) { UseMethod("copyRequiredJobFiles") } #' @export copyRequiredJobFiles.Registry = function(reg1, reg2, id) { job = getJob(reg1, id, check.id = FALSE) file.copy(getFunFilePath(reg1, job$fun.id), getFunFilePath(reg2, job$fun.id)) } BatchJobs/R/waitForJobs.R0000644000176000001440000000734412465167456014746 0ustar ripleyusers#' Wait for termination of jobs on the batch system. #' #' @description #' Waits for termination of jobs while displaying a progress bar #' containing summarizing informations of the jobs. #' The following abbreviations are used in the progress bar: #' \dQuote{S} for number of jobs on system, \dQuote{D} for number of #' jobs successfully terminated, \dQuote{E} for number ofjobs terminated #' with an R exception and \dQuote{R} for number of jobs currently running #' on the system. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Vector of job ids. #' Default is all submitted jobs not yet terminated. #' @param sleep [\code{numeric(1)}]\cr #' Seconds to sleep between status updates. Default is \code{10}. #' @param timeout [\code{numeric(1)}]\cr #' After waiting \code{timeout} seconds, show a message and return \code{FALSE}. #' This argument may be required on some systems where, e.g., expired jobs or jobs on hold #' are problematic to detect. If you don't want a timeout, set this to \code{Inf}. #' Default is \code{604800} (one week). #' @param stop.on.error [\code{logical(1)}]\cr #' Immediately return if a job terminates with an error? Default is \code{FALSE}. #' @template arg_progressbar #' @return [\code{logical(1)}]. Returns \code{TRUE} if all jobs terminated successfully #' and \code{FALSE} if either an error occurred or the timeout is reached. #' @export waitForJobs = function(reg, ids, sleep = 10, timeout = 604800, stop.on.error = FALSE, progressbar = TRUE) { checkRegistry(reg) syncRegistry(reg) if (missing(ids)) { ids = dbFindSubmittedNotTerminated(reg) } else { ids = checkIds(reg, ids) not.submitted = dbFindSubmitted(reg, ids, negate = TRUE, limit = 1L) if (length(not.submitted) > 0L) stopf("Not all jobs have been submitted, e.g. job with id %i", not.submitted) } assertNumber(sleep, lower = 1) if (is.infinite(sleep)) stop("Argument 'sleep' must be finite") assertNumber(timeout, lower = sleep) assertFlag(stop.on.error) assertFlag(progressbar) n = length(ids) if (n == 0L) return(TRUE) timeout = now() + timeout batch.ids = getBatchIds(reg, "Cannot find jobs on system") i = 1L bar = getProgressBar(progressbar, min = 0L, max = n, label = "Waiting ") on.exit(bar$kill()) repeat { stats = dbGetStats(reg, ids, running = TRUE, expired = FALSE, times = FALSE, batch.ids = batch.ids) n.sys = n - stats$done - stats$error bar$set(n - n.sys, msg = sprintf("Waiting [S:%i D:%i E:%i R:%i]", n.sys, stats$done, stats$error, stats$running)) if (stop.on.error && stats$error) { err = dbGetErrorMsgs(reg, ids, filter = TRUE, limit = 1L) warningf("Job %i terminated with an error: %s", err$job_id, err$error) return(FALSE) } if (n.sys == 0L) return(stats$error == 0L) if (i %% 5L == 0L) { # update batch ids batch.ids = getBatchIds(reg, "Cannot find jobs on system") # check if there are still jobs on the system and none has mystically disappeared # NOTE it seems like some schedulers are "laggy", we should not do this operation # in the first loop w/o a sleep if (length(dbFindOnSystem(reg, ids, limit = 1L, batch.ids = batch.ids)) == 0L) { if (length(dbFindDisappeared(reg, ids, limit = 1L, batch.ids = batch.ids)) > 0L) bar$error(stop("Some jobs disappeared, i.e. were submitted but are now gone. Check your configuration and template file.")) return(stats$error == 0L) } } if (is.finite(timeout) && now() > timeout) { warningf("Timeout reached. %i jobs still on system.", n.sys) return(FALSE) } Sys.sleep(sleep) i = i + 1L suppressMessages(syncRegistry(reg)) } } BatchJobs/R/setJobFunction.R0000644000176000001440000000320512415510734015423 0ustar ripleyusers#' Sets the job function for already existing jobs. #' #' @description #' Use this function only as last measure when there is a bug #' in a part of your job function and you have already computed a large number #' of (unaffected) results. This function allows you to fix the error and to #' associate the jobs with the corrected function. #' #' Note that by default the computational state of the affected jobs is also reset. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs. #' @param fun [\code{function}]\cr #' Replacement function. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @param reset [\code{logical(1)}]\cr #' Reset job status via \code{\link{resetJobs}}. #' Default is \code{TRUE}. #' @param force [\code{logical(1)}]\cr #' See \code{\link{resetJobs}}. #' Default is \code{FALSE}. #' @return Nothing. #' @family debug #' @export setJobFunction = function(reg, ids, fun, more.args = list(), reset = TRUE, force = FALSE) { checkRegistry(reg, strict = TRUE) syncRegistry(reg) assertFunction(fun) checkMoreArgs(more.args) assertFlag(reset) assertFlag(force) UseMethod("setJobFunction") } #' @export setJobFunction.Registry = function(reg, ids, fun, more.args = list(), reset = TRUE, force = FALSE) { if (missing(ids)) { # nothing to do ... return(invisible(NULL)) } else { ids = checkIds(reg, ids) } fun.id = saveFunction(reg, fun, more.args) dbSetJobFunction(reg, ids, fun.id) if (reset) resetJobs(reg, ids, force = force) invisible(NULL) } BatchJobs/R/clusterFunctionsSSH.R0000644000176000001440000001315712415510734016426 0ustar ripleyusers#' Create SSH worker for SSH cluster functions. #' #' @param nodename [\code{character(1)}]\cr #' Host name of node. #' @param rhome [\code{character(1)}]\cr #' Path to R installation on worker. #' \dQuote{} means R installation on the PATH is used, #' of course this implies that it must be on the PATH #' (also for non-interactive shells)! #' Default is \dQuote{}. #' @param ncpus [\code{integers(1)}]\cr #' Number of VPUs of worker. #' Default means to query the worker via \dQuote{/proc/cpuinfo}. #' @param max.jobs [\code{integer(1)}]\cr #' Maximal number of jobs that can run concurrently for the current registry. #' Default is \code{ncpus}. #' @param max.load [\code{numeric(1)}]\cr #' Load average (of the last 5 min) at which the worker is considered occupied, #' so that no job can be submitted. #' Default is \code{ncpus-1}. #' @param nice [\code{integer(1)}]\cr #' Process priority to run R with set via nice. Integers between -20 and 19 are allowed. #' If missing, processes are not nice'd and the system default applies (usually 0). #' Default is no niceing. #' @param r.options [\code{character}] #' Options for R and Rscript, one option per element of the vector, #' a la \dQuote{--vanilla}. #' Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}. #' @param script [\code{character(1)}]\cr #' Path to helper bash script which interacts with the worker. #' You really should not have to touch this, as this would imply that we have screwed up and #' published an incompatible version for your system. #' This option is only provided as a last resort for very experienced hackers. #' Note that the path has to be absolute. #' This is what is done in the package: #' \url{https://github.com/tudo-r/BatchJobs/blob/master/inst/bin/linux-helper} #' Default means to take it from package directory. #' @return [\code{\link{SSHWorker}}]. #' @export #' @aliases SSHWorker makeSSHWorker = function(nodename, rhome = "", ncpus, max.jobs, max.load, nice, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script) { worker = makeWorkerRemoteLinux(nodename, rhome, r.options, script, ncpus, max.jobs, max.load, nice) class(worker) = c("SSHWorker", class(worker)) return(worker) } #' Create an SSH cluster to execute jobs. #' #' Worker nodes must share the same file system and be accessible by ssh #' without manually entering passwords (e.g. by ssh-agent or passwordless pubkey). #' Note that you can also use this function to parallelize on multiple cores on your local machine. #' But you still have to run an ssh server and provide passwordless access to #' localhost. #' #' @param ... [\code{\link{SSHWorker}}]\cr #' Worker objects, all created with \code{\link{makeSSHWorker}}. #' @param workers [list of \code{\link{SSHWorker}}]\cr #' Alternative way to pass workers. #' @return [\code{ClusterFunctions}]. #' #' @examples \dontrun{ #' #' # Assume you have three nodes larry, curley and moe. All have 6 #' # cpu cores. On curley and moe R is installed under #' # "/opt/R/R-current" and on larry R is installed under #' # "/usr/local/R/". larry should not be used extensively because #' # somebody else wants to compute there as well. #' # Then a call to 'makeClusterFunctionsSSH' #' # might look like this: #' #' cluster.functions = makeClusterFunctionsSSH( #' makeSSHWorker(nodename = "larry", rhome = "/usr/local/R", max.jobs = 2), #' makeSSHWorker(nodename = "curley", rhome = "/opt/R/R-current"), #' makeSSHWorker(nodename = "moe", rhome = "/opt/R/R-current")) #' } #' @export #' @family clusterFunctions #' @seealso \code{\link{makeSSHWorker}} makeClusterFunctionsSSH = function(..., workers) { args = list(...) if (!xor(length(args) > 0L, !missing(workers))) stop("You must use exactly only 1 of: '...', 'workers'!") if (missing(workers)) workers = args checkListElementClass(workers, "SSHWorker") if (length(workers) == 0L) stop("You must pass at least 1 SSH worker!") nodenames = extractSubList(workers, "nodename") dup = duplicated(nodenames) if (any(dup)) stopf("Multiple definitions for worker nodenames: %s!", collapse(nodenames[dup])) names(workers) = nodenames rm(nodenames, dup) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { worker = findWorker(workers, reg$file.dir, tdiff = 5L) if (is.null(worker)) { states = collapse(extractSubList(workers, "available", simplify = TRUE), sep = "") makeSubmitJobResult(status = 1L, batch.job.id = NULL, msg = sprintf("Workers busy: %s", states)) } else { pid = try(startWorkerJob(worker, rscript, log.file)) if (is.error(pid)) makeSubmitJobResult(status = 101L, batch.job.id = NULL, msg = "Submit failed.") else makeSubmitJobResult(status = 0L,batch.job.id = paste0(worker$nodename, "#", pid)) } } killJob = function(conf, reg, batch.job.id) { parts = strsplit(batch.job.id, "#", fixed = TRUE)[[1L]] nodename = parts[1L] pid = parts[2L] worker = workers[[nodename]] if (is.null(worker)) stopf("Unknown worker node '%s'.", nodename) killWorkerJob(worker, pid) } listJobs = function(conf, reg) { res = NULL for (worker in workers) { nodename = worker[["nodename"]] pids = listWorkerJobs(worker, reg$file.dir) if (length(pids) > 0L) { res = c(res, paste0(nodename, "#", pids)) } } res } getArrayEnvirName = function() NA_character_ makeClusterFunctions(name = "SSH", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/checkIdValid.R0000644000176000001440000000101412415510734014775 0ustar ripleyusers# check for a valid registry id: # must start with an alphabetic # then alpha-numerics and underscores are allowed # pattern = "^[0-9a-zA-Z]+[0-9a-zA-Z_-]*$" # we must be quite restrictive here because we prefix # the table name with the registry name checkIdValid = function(id, allow.minus = TRUE) { assertString(id) if (allow.minus) pattern = "^[a-zA-Z]+[0-9a-zA-Z_-]*$" else pattern = "^[a-zA-Z]+[0-9a-zA-Z_]*$" if (!grepl(pattern, id)) stopf("Id does not comply with pattern %s: %s", pattern, id) } BatchJobs/R/showClusterStatus.R0000644000176000001440000000234412415510734016220 0ustar ripleyusers#' Show information about available computational resources on cluster. #' #' @description #' Currently only supported for multicore and SSH mode. #' Displays: Name of node, current load, number of running R processes, number of R processes #' with more than 50% load, number of BatchJobs jobs running. #' The latter counts either jobs belonging to \code{reg} or all BatchJobs jobs if reg was not passed. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' Must not be passed and this is the default. #' @return [\code{data.frame}]. Returns the displayed table invisibly. #' @export # FIXME: allow to get a quick overview by passing nodenames showClusterStatus = function(reg) { if (missing(reg)) { file.dir = "" } else { checkRegistry(reg) syncRegistry(reg) file.dir = reg$file.dir } conf = getBatchJobsConf() cf = conf$cluster.functions if (cf$name %nin% c("Multicore", "SSH")) stop("showWorkerStatus can only be used in multicore or SSH mode!") workers = environment(cf$submitJob)$workers data = lapply(workers, getWorkerStatus, file.dir = file.dir) data = do.call(rbind, lapply(data, as.data.frame)) data = cbind(ncpus = extractSubList(workers, "ncpus"), data) print(data) invisible(data) } BatchJobs/R/batchReduce.R0000644000176000001440000000367312415510734014711 0ustar ripleyusers#' Reduces via a binary function over a list adding jobs to a registry. #' #' @description #' Each jobs reduces a certain number of elements on one slave. #' You can then submit these jobs to the batch system. #' @param reg [\code{\link{Registry}}]\cr #' Empty Registry. #' @param fun [\code{function(aggr, x, ...)}]\cr #' Function to reduce \code{xs} with. #' @param xs [\code{vector}]\cr #' Vector to reduce. #' @param init [any]\cr #' Initial object for reducing. #' @param block.size [\code{integer(1)}]\cr #' Number of elements of \code{xs} reduced in one job. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @return Vector of type \code{integer} with job ids. #' @export #' @examples #' # define function to reduce on slave, we want to sum a vector #' f = function(aggr, x) aggr + x #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' #' # sum 20 numbers on each slave process, i.e. 5 jobs #' batchReduce(reg, fun = f, 1:100, init = 0, block.size = 5) #' submitJobs(reg) #' waitForJobs(reg) #' #' # now reduce one final time on master #' reduceResults(reg, fun = function(aggr,job,res) f(aggr, res)) batchReduce = function(reg, fun, xs, init, block.size, more.args = list()) { checkRegistry(reg, strict = TRUE) syncRegistry(reg) assertFunction(fun, c("aggr", "x")) if (!is.vector(xs)) stop("Argument xs must be a vector") block.size = asCount(block.size, positive = TRUE) if (dbGetJobCount(reg) > 0L) stop("Registry is not empty!") checkMoreArgs(more.args, reserved = c(".fun", ".init")) xs.blocks = chunk(xs, chunk.size = block.size, shuffle = FALSE) more.args = c(more.args, list(.fun = fun, .init = init)) batchMap(reg, batchReduceWrapper, xs.blocks, more.args = more.args) } batchReduceWrapper = function(xs.block, .fun, .init, ...) { fun = function(aggr, x) .fun(aggr, x, ...) Reduce(fun, xs.block, init = .init) } BatchJobs/R/batchMapQuick.R0000644000176000001440000000552512415510734015212 0ustar ripleyusers#' Combination of makeRegistry, batchMap and submitJobs. #' #' @description #' Combination of \code{\link{makeRegistry}}, \code{\link{batchMap}} #' and \code{\link{submitJobs}} #' for quick computations on the cluster. #' Should only be used by skilled users who know what they are doing. #' Creates the file.dir, maps function, potentially chunks jobs and submits them. #' #' @param fun [\code{function}]\cr #' Function to map over \code{...}. #' @param ... [any]\cr #' Arguments to vectorize over (list or vector). #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @param file.dir [\code{character}]\cr #' See \code{\link{makeRegistry}}. #' Default is \code{NULL}, which means that it is created in the current directory under the name #' \dQuote{bmq_[random alphanumerics]}. #' @param packages [\code{character}]\cr #' See \code{\link{makeRegistry}}. #' @param chunk.size [\code{integer(1)}]\cr #' Preferred number of jobs in each chunk. #' Can not be used in combination with \code{n.chunks}. #' Note that the ids will get shuffled to balance out possible run time differences. #' Default is not to use chunking. #' @param n.chunks [\code{integer(1)}]\cr #' Preferred number chunks. #' Can not be used in combination with \code{chunk.size}. #' Note that the ids will get shuffled to balance out possible run time differences. #' Default is not to use chunking. #' @param chunks.as.arrayjobs [\code{logical(1)}]\cr #' Submit chunks as array jobs? #' Default is \code{FALSE}. #' @param inds [\code{integer}]\cr #' Indices of ids / chunks to submit. #' Default is all. If ids get chunked, this subsets the list of shuffled ids. #' @param resources [\code{list}]\cr #' Required resources for all batch jobs. #' Default is empty list. #' @return [\code{\link{Registry}}] #' @export batchMapQuick = function(fun, ..., more.args = list(), file.dir = NULL, packages = character(0L), chunk.size, n.chunks, chunks.as.arrayjobs = FALSE, inds, resources = list()) { if (is.null(file.dir)) { # create name for temp file dir in current dir file.dir = tempfile(pattern = "bmq_", tmpdir = getwd()) } else { assertString(file.dir) } id = basename(file.dir) reg = makeRegistry(id = id, file.dir = file.dir, packages = packages) on.exit(messagef("Interrupted. You can find your registry in %s.", reg$file.dir)) # we want to return the reg in any case # otherwise we cannot look at it / do anything with it in case of errors try({ ids = batchMap(reg, fun, ..., more.args = more.args) if (!missing(chunk.size) || !missing(n.chunks)) ids = chunk(ids, chunk.size = chunk.size, n.chunks = n.chunks, shuffle = TRUE) if (!missing(inds)) ids = ids[inds] submitJobs(reg, ids, resources = resources) }, silent = FALSE) on.exit(NULL) return(reg) } BatchJobs/R/clusterFunctionsLSF.R0000644000176000001440000000615012415510734016410 0ustar ripleyusers#' @title Create cluster functions for LSF systems. #' #' @description #' Job files are created based on the brew template #' \code{template.file}. This file is processed with brew and then #' submitted to the queue using the \code{bsub} command. Jobs are #' killed using the \code{bkill} command and the list of running jobs #' is retrieved using \code{bjobs -u $USER -w}. The user must have the #' appropriate privileges to submit, delete and list jobs on the #' cluster (this is usually the case). #' #' The template file can access all arguments passed to the #' \code{submitJob} function, see here \code{\link{ClusterFunctions}}. #' It is the template file's job to choose a queue for the job #' and handle the desired resource allocations. #' Examples can be found on #' \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfLSF}. #' #' @template arg_template #' @template arg_list_jobs_cmd #' @template ret_cf #' @family clusterFunctions #' @export makeClusterFunctionsLSF = function(template.file, list.jobs.cmd = c("bjobs", "-u $USER", "-w")) { assertCharacter(list.jobs.cmd, min.len = 1L, any.missing = FALSE) template = cfReadBrewTemplate(template.file) # When LSB_BJOBS_CONSISTENT_EXIT_CODE = Y, the bjobs command exits with 0 only # when unfinished jobs are found, and 255 when no jobs are found, # or a non-existent job ID is entered. Sys.setenv(LSB_BJOBS_CONSISTENT_EXIT_CODE = "Y") submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { outfile = cfBrewTemplate(conf, template, rscript, "job") # returns: "Job <128952> is submitted to default queue ." res = runOSCommandLinux("bsub", stdin = outfile, stop.on.exit.code = FALSE) # FIXME filled queues if (res$exit.code > 0L) { cfHandleUnknownSubmitError("bsub", res$exit.code, res$output) } else { # collapse output strings and first number in string is batch.job.id batch.job.id = str_extract(collapse(res$output, sep = " "), "\\d+") makeSubmitJobResult(status = 0L, batch.job.id = batch.job.id) } } killJob = function(conf, reg, batch.job.id) { cfKillBatchJob("bkill", batch.job.id) } listJobs = function(conf, reg) { # JOBID USER STAT QUEUE FROM_HOST EXEC_HOST JOB_NAME SUBMIT_TIME # 106560 rogon UNKWN m_amd hpc84 hpc25 QScript Mar 19 12:18 # res = runOSyyCommandLinux("bjobs", c("-u $USER", "-w"), stop.on.exit.code = FALSE) res = runOSCommandLinux(list.jobs.cmd[1L], list.jobs.cmd[-1L], stop.on.exit.code = FALSE) if (res$exit.code == 255L && grepl("No unfinished job found", res$output, fixed = TRUE)) return(character(0L)) if (res$exit.code > 0L) stopf("bjobs produced exit code %i; output %s", res$exit.code, res$output) # drop first header line of output out = tail(res$output, -1L) # first number in strings are batch.job.ids str_extract(out, "\\d+") } getArrayEnvirName = function() "LSB_JOBINDEX" makeClusterFunctions(name = "LSF", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/Resources.R0000644000176000001440000000407612465167456014466 0ustar ripleyusersresrc = function(res.new) { res.old = getBatchJobsConf()$default.resources if(!isProperlyNamed(res.new) || !isProperlyNamed(res.old)) stop("Resources must be all be uniquely named!") insert(res.old, res.new) } saveResources = function(reg, resources, timestamp = now()) { fn = getResourcesFilePath(reg, timestamp) save2(file = fn, resources = resources) return(timestamp) } #' Function to get job resources in job function. #' #' Return the list passed to \code{\link{submitJobs}}, e.g. #' nodes, walltime, etc. #' #' Can only be called in job function during job execution on slave. #' #' @return [\code{list}]. #' @export getResources = function() { if (!isOnSlave()) stop("getResources can only be called during job execution on slave!") load2(getOption("BatchJobs.resources.path")) } #' Function to get the resources that were submitted for some jobs. #' #' Throws an error if call it for unsubmitted jobs. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all submitted jobs. #' @param as.list [\code{integer(1)}]\cr #' If \code{FALSE} a data.frame will be returned. #' Default is \code{TRUE}. #' @return [\code{list} | \code{data.frame}]. List (or data.frame) of resource lists as passed to \code{\link{submitJobs}}. #' @export getJobResources = function(reg, ids, as.list = TRUE) { checkRegistry(reg) syncRegistry(reg) if (missing(ids)) { ids = dbFindSubmitted(reg) } else { ids = checkIds(reg, ids) nsub = dbFindSubmitted(reg, ids, negate = TRUE, limit = 1L) if (length(nsub) > 0L) stopf("Some of your jobs have not been submitted, so no resources are available, e.g. for id=%i", nsub) } query = sprintf("SELECT job_id, resources_timestamp FROM %s_job_status", reg$id) df = dbSelectWithIds(reg, query, ids) res = namedList(df$job_id) for(ts in unique(df$resources_timestamp)) { res[df$resources_timestamp == ts] = load2(getResourcesFilePath(reg, ts), simplify = FALSE) } if (!as.list) res = convertListOfRowsToDataFrame(res) return(res) } BatchJobs/R/clusterFunctionsTorque.R0000644000176000001440000000470012423457121017241 0ustar ripleyusers#' @title Create cluster functions for torque-based systems. #' #' @description #' Job files are created based on the brew template #' \code{template.file}. This file is processed with brew and then #' submitted to the queue using the \code{qsub} command. Jobs are #' killed using the \code{qdel} command and the list of running jobs #' is retrieved using \code{qselect}. The user must have the #' appropriate privileges to submit, delete and list jobs on the #' cluster (this is usually the case). #' #' The template file can access all arguments passed to the #' \code{submitJob} function, see here \code{\link{ClusterFunctions}}. #' It is the template file's job to choose a queue for the job #' and handle the desired resource allocations. #' Examples can be found on #' \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfTorque}. #' #' @template arg_template #' @template arg_list_jobs_cmd #' @template ret_cf #' @family clusterFunctions #' @export makeClusterFunctionsTorque = function(template.file, list.jobs.cmd = c("qselect", "-u $USER", "-s EHQRTW")) { assertCharacter(list.jobs.cmd, min.len = 1L, any.missing = FALSE) template = cfReadBrewTemplate(template.file) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { outfile = cfBrewTemplate(conf, template, rscript, "pbs") res = runOSCommandLinux("qsub", outfile, stop.on.exit.code = FALSE) max.jobs.msg = "Maximum number of jobs already in queue" output = collapse(res$output, sep = "\n") if (grepl(max.jobs.msg, output, fixed = TRUE)) { makeSubmitJobResult(status = 1L, batch.job.id = NA_character_, msg = max.jobs.msg) } else if (res$exit.code > 0L) { cfHandleUnknownSubmitError("qsub", res$exit.code, res$output) } else { makeSubmitJobResult(status = 0L, batch.job.id = str_trim(output)) } } killJob = function(conf, reg, batch.job.id) { cfKillBatchJob("qdel", batch.job.id) } listJobs = function(conf, reg) { # Result is lines of fully quantified batch.job.ids batch.ids = runOSCommandLinux(list.jobs.cmd[1L], list.jobs.cmd[-1L])$output # simplify batch ids of array jobs, i.e. remove the array id from the batch id unique(gsub("\\[[[:digit:]]\\]", "[]", batch.ids)) } getArrayEnvirName = function() "PBS_ARRAYID" makeClusterFunctions(name = "Torque", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/resetJobs.R0000644000176000001440000000371412465167456014452 0ustar ripleyusers#' Reset computational state of jobs. #' #' @description #' Reset state of jobs in the database. Useful under two circumstances: #' Either to re-submit them because of changes in e.g. external #' data or to resolve rare issues when jobs are killed in an unfortunate state #' and therefore blocking your registry. #' #' The function internally lists all jobs on the batch system and #' if those include some of the jobs you want to reset, it informs you to kill them first by raising #' an exception. #' If you really know what you are doing, you may set \code{force} to \code{TRUE} to omit this sanity check. #' Note that this is a dangerous operation to perform which may harm #' the database integrity. In this case you HAVE to make externally sure that none of the jobs #' you want to reset are still running. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs to kill. #' Default is none. #' @param force [\code{logical(1)}]\cr #' Reset jobs without checking whether they are currently running. #' READ THE DETAILS SECTION! #' Default is \code{FALSE}. #' @return Vector of reseted job ids. #' @family debug #' @export resetJobs = function(reg, ids, force = FALSE) { checkRegistry(reg) syncRegistry(reg) if (missing(ids) || length(ids) == 0L) return(integer(0L)) ids = checkIds(reg, ids) assertFlag(force) if (!force) { if(is.null(getListJobs()) || is.null(getKillJob())) { stop("Listing or killing of jobs not supported by your cluster functions\n", "You need to set force = TRUE to reset jobs, but see the warning in ?resetJobs") } running = dbFindOnSystem(reg, ids, limit = 10L) if (length(running) > 0L) stopf("Can't reset jobs which are live on system. You have to kill them first!\nIds: %s", collapse(running)) } info("Resetting %i jobs in DB.", length(ids)) dbSendMessage(reg, dbMakeMessageKilled(reg, ids), staged = FALSE) invisible(ids) } BatchJobs/R/sweepRegistry.R0000644000176000001440000000557012465167456015370 0ustar ripleyusers#' @title Sweep obsolete files from the file system. #' #' @description #' Removes R scripts, log files, resource informations and temporarily stored configuration files #' from the registry's file directory. Assuming all your jobs completed successfully, none of these are needed #' for further work. This operation potentially releases quite a lot of disk space, depending on the number of your jobs. #' BUT A HUGE WORD OF WARNING: #' IF you later notice something strange and need to determine the reason for it, you are at a huge disadvantage. #' Only do this at your own risk and when you are sure that you have successfully completed a project and only #' want to archive your produced experiments and results. #' #' @template arg_reg #' @param sweep [\code{character}]\cr #' Possible choices: #' Temporary R scripts of jobs, #' really not needed for anything else then execution (\dQuote{scripts}), #' log file of jobs, #' think about whether you later want to inspect them (\dQuote{logs}), #' BatchJobs configuration files which are temporarily stored on submit, #' really not needed for anything else then execution (\dQuote{conf}), #' resource lists of \code{\link{submitJobs}} which are temporarily stored on submit, #' think about whether you later want to inspect them (\dQuote{resources}), #' Default is \code{c("scripts", "conf")}. #' @return [\code{logical}]. Invisibly returns \code{TRUE} on success and \code{FALSE} #' if some files could not be removed. #' @export sweepRegistry = function(reg, sweep = c("scripts", "conf")) { checkRegistry(reg) syncRegistry(reg) assertSubset(sweep, c("scripts", "logs", "resources", "conf")) if (length(dbFindRunning(reg, limit = 1L)) > 0L) stop("Can't sweep registry while jobs are running") fd = reg$file.dir jd = getJobParentDir(fd) rd = getResourcesDir(fd) # failed kill ids are always obsolete because no jobs are running anymore files = list.files(fd, pattern = "^killjobs_failed_ids_*", full.names = TRUE) # sweep configuration if ("conf" %in% sweep) files = c(files, list.files(fd, pattern = "^conf.RData$", full.names = TRUE)) # sweep resources if ("resources" %in% sweep) files = c(files, list.files(rd, full.names = TRUE)) # sweep logs and scripts (in one go if possible) if (all(c("logs", "scripts") %in% sweep)) { files = c(files, list.files(jd, pattern = "^[0-9]+\\.[out|R]$", recursive = TRUE, full.names = TRUE)) } else { if ("logs" %in% sweep) files = c(files, list.files(jd, pattern = "^[0-9]+\\.out$", recursive = TRUE, full.names = TRUE)) if ("scripts" %in% sweep) files = c(files, list.files(jd, pattern = "^[0-9]+\\.R$", recursive = TRUE, full.names = TRUE)) } info("Removing %i files ...", length(files)) ok = all(file.remove(files)) if (!ok) warning("Not all files could be deleted. Check file permissions and try again") return(invisible(ok)) } BatchJobs/R/clusterFunctionsLocal.R0000644000176000001440000000245512415510734017022 0ustar ripleyusers#' Create cluster functions for sequential execution on local host. #' #' @description #' All jobs executed under these cluster functions are executed #' sequentially, but in an independent, new R session. #' That is, \code{submitJob} does not return until the #' job has finished. The main use of this \code{ClusterFunctions} #' implementation is to test and debug programs on a local computer. #' #' Listing jobs returns an empty vector (as no jobs can be running when you call this) #' and \code{killJob} returns at once (for the same reason). #' #' @return [\code{\link{ClusterFunctions}}]. #' @family clusterFunctions #' @export makeClusterFunctionsLocal = function() { submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { system2(command = file.path(R.home("bin"), "Rscript"), args = rscript, stdout = log.file, stderr = log.file, wait = TRUE) makeSubmitJobResult(status = 0L, batch.job.id = "cfLocal") } killJob = function(conf, reg, batch.job.id) NULL listJobs = function(conf, reg) integer(0L) getArrayEnvirName = function() NA_character_ makeClusterFunctions(name = "Local", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/updateRegistry.R0000644000176000001440000000615712415510734015513 0ustar ripleyusers#' ONLY FOR INTERNAL USAGE. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @return [any]. Updated \code{\link{Registry}} or \code{FALSE} if no updates were performed. #' @keywords internal #' @export updateRegistry = function(reg) { UseMethod("updateRegistry") } #' @method updateRegistry Registry #' @export updateRegistry.Registry = function(reg) { # Fix for missing package version (package versions < 1.0.527) if ("BatchJobs" %nin% names(reg$packages)) { reg$packages$BatchJobs = list(version = package_version("1.0.527")) } version.reg = reg$packages$BatchJobs$version version.pkg = packageVersion("BatchJobs") if (version.reg == version.pkg) { return(FALSE) } if (version.reg > version.pkg) { warningf("The registry has been used with BatchJobs version %s, installed is version %s. You should update BatchJobs on this machine.", version.reg, version.pkg) return(FALSE) } # update registry info("Updating Registry and DB to newer version.") if (version.reg < package_version("1.0.606")) { # create new resources dir resources.dir = getResourcesDir(reg$file.dir) checkDir(resources.dir, create = TRUE, check.empty = TRUE) query = sprintf("ALTER TABLE %s_job_status ADD COLUMN resources_timestamp INTEGER", reg$id) dbDoQuery(reg, query, flags = "rwc") # save dummy resources query = sprintf("UPDATE %s_job_status SET resources_timestamp = 0 WHERE submitted IS NOT NULL", reg$id) dbDoQuery(reg, query, flags = "rwc") saveResources(reg, resources = list(), timestamp = 0L) } if (version.reg < package_version("1.0.723")) { checkDir(getPendingDir(reg$file.dir), create = TRUE) } if (version.reg < package_version("1.0.1071")) { checkDir(getExportDir(reg$file.dir), create = TRUE) } if (version.reg < package_version("1.1")) { query = sprintf("ALTER TABLE %s_job_def ADD COLUMN jobname TEXT", reg$id) dbDoQuery(reg, query, flags = "rwc") reg$src.dirs = character(0L) } if (version.reg < package_version("1.2")) { reg$src.files = character(0L) } if (version.reg < package_version("1.4")) { query = sprintf("ALTER TABLE %s_job_status ADD COLUMN memory REAL", reg$id) dbDoQuery(reg, query, flags = "rwc") } reg$packages$BatchJobs$version = version.pkg reg } adjustRegistryPaths = function(reg, file.dir, work.dir) { adjusted = FALSE # adjust file dir if necessary file.dir = sanitizePath(file.dir, make.absolute = TRUE) if (!isDirectory(file.dir)) stopf("file.dir does not exist or is not a directory: %s", file.dir) if (reg$file.dir != file.dir) { reg$file.dir = file.dir adjusted = TRUE } # adjust work dir if necessary if (missing(work.dir)) { if (!isDirectory(reg$work.dir)) warningf("The currently set work.dir '%s' does not exists. Use option 'work.dir' in loadRegistry to change it.", reg$work.dir) } else { work.dir = sanitizePath(work.dir, make.absolute = TRUE) if (!isDirectory(work.dir)) stopf("work.dir does not exist or is not a directory: %s", work.dir) reg$work.dir = work.dir adjusted = TRUE } if (adjusted) reg else FALSE } BatchJobs/R/batchMapResults.R0000644000176000001440000000572412465167456015616 0ustar ripleyusers#' Maps a function over the results of a registry by using batchMap. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry whose results should be mapped by \code{fun}. #' @param reg2 [\code{\link{Registry}}]\cr #' Empty registry that should store the job for the mapping. #' @param fun [\code{function(job, res, ...)}]\cr #' Function to map over results of \code{reg}. #' Further arguments come from ... of \code{batchMapResults} and \code{more.args}. #' @param ... [any]\cr #' Furher arguments to vectorize over (list or vector). #' Must all be the same length as number of results in \code{reg}. #' @param ids [\code{integer}]\cr #' Ids of jobs whose results should be mapped with \code{fun}. #' Default is all jobs. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @return Vector of type \code{integer} with job ids. #' @export #' @examples #' reg1 = makeRegistry(id = "BatchJobsExample1", file.dir = tempfile(), seed = 123) #' # square some numbers #' f = function(x) x^2 #' batchMap(reg1, f, 1:10) #' #' # submit jobs and wait for the jobs to finish #' submitJobs(reg1) #' waitForJobs(reg1) #' #' # look at results #' reduceResults(reg1, fun = function(aggr,job,res) c(aggr, res)) #' #' reg2 = makeRegistry(id = "BatchJobsExample2", file.dir = tempfile(), seed = 123) #' #' # define function to tranform results, we simply do the inverse of the squaring #' g = function(job, res) sqrt(res) #' batchMapResults(reg1, reg2, fun = g) #' #' # submit jobs and wait for the jobs to finish #' submitJobs(reg2) #' waitForJobs(reg2) #' #' # check results #' reduceResults(reg2, fun = function(aggr,job,res) c(aggr, res)) batchMapResults = function(reg, reg2, fun, ..., ids, part = NA_character_, more.args = list()) { # FIXME conserve jobnames checkRegistry(reg) syncRegistry(reg) checkRegistry(reg2) syncRegistry(reg2) assertFunction(fun, c("job", "res")) if (missing(ids)) { ids = dbGetJobIdsIfAllDone(reg) } else { ids = checkIds(reg, ids) if (length(dbFindDone(reg, ids, negate = TRUE, limit = 1L)) > 0L) stop("Not all jobs with corresponding ids finished (yet)!") } checkMoreArgs(more.args, reserved = c(".reg", ".fun", ".part")) if (dbGetJobCount(reg2) > 0L) stop("Registry 'reg2' is not empty!") if(reg$file.dir == reg2$file.dir) stop("Both registries cannot point to the same file dir. Files would get overwritten!") reg2$packages = insert(reg2$packages, reg$packages) saveRegistry(reg2) batchMap(reg2, batchMapResultsWrapper, ids, ..., more.args = c(more.args, list(.reg = reg, .fun = fun, .part = part))) } batchMapResultsWrapper = function(id, ..., .reg, .fun, .part) { .fun(job = getJob(.reg, id, check.id = FALSE), res = getResult(.reg, id, part = .part), ...) } BatchJobs/R/findJobs.R0000644000176000001440000000335312415510734014231 0ustar ripleyusers#' Finds ids of jobs that match a query. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Subset of job ids to restrict the result to. #' Default is all jobs. #' @param pars [R expression]\cr #' All jobs whose parameters match the given expression are selected. #' This implies that you have named the parameters when you passed the vectors. #' If you forgot to do this you can use \code{.arg1}, \code{.arg2}, etc., to refer to the #' the unnamed ones. #' @param jobnames [\code{character}]\cr #' Restrict to jobs with stored names. Exact matching is used. #' @return [\code{integer}]. Ids for jobs which match the query. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x, y) x * y #' batchExpandGrid(reg, f, x = 1:2, y = 1:3) #' findJobs(reg, pars = (y > 2)) findJobs = function(reg, ids, pars, jobnames) { checkRegistry(reg, strict = TRUE) syncRegistry(reg) if (!missing(ids)) checkIds(reg, ids) if (missing(pars) && missing(jobnames)) return(getJobIds(reg)) if (!missing(jobnames)) { assertCharacter(jobnames, any.missing = FALSE) ids = dbMatchJobNames(reg, ids, jobnames) } if (!missing(pars)) { jobs = dbGetJobs(reg, ids) rename = function(pars) { ns = names(pars) if (is.null(ns)) { ns = rep.int("", length(pars)) } j = which(is.na(ns) | ns == "") ns[j] = paste0(".arg", seq_along(j)) setNames(pars, ns) } ind = vlapply(jobs, function(job, pars, ee) eval(pars, rename(job$pars), ee), pars = substitute(pars), ee = parent.frame()) ids = extractSubList(jobs[!is.na(ind) & ind], "id", element.value = integer(1L)) } ids } BatchJobs/R/getSSHWorkersInfo.R0000644000176000001440000000275512415510734016026 0ustar ripleyusers#' Print and return R installation and other information for SSH workers. #' #' @description #' Workers are queried in parallel via \code{\link{callFunctionOnSSHWorkers}}. #' #' The function will display a warning if the first lib path on the worker #' is not writable as this indicates potential problems in the configuration #' and \code{\link{installPackagesOnSSHWorkers}} will not work. #' #' @param nodenames [\code{character}]\cr #' Nodenames of workers. #' @return [\code{list}]. Displayed information as a list named by nodenames. #' @export #' @seealso \code{\link{callFunctionOnSSHWorkers}} getSSHWorkersInfo = function(nodenames) { fun = function() { lib.paths = .libPaths() list( r.home = R.home(), session.info = sessionInfo(), lib.paths = lib.paths, is.lib.path.writeable = is.accessible(head(lib.paths, 1L)) ) } res = callFunctionOnSSHWorkers(nodenames, fun = fun, consecutive = FALSE, show.output = FALSE, use.names = TRUE, simplify = FALSE) for (nn in nodenames) { r = res[[nn]] messagef("Node: %s", nn) messagef(r$session.info$R.version$version.string) messagef("Platform: %s", r$session.info$platform) messagef("R Home: %s", r$r.home) messagef("First lib path: %s", head(r$lib.paths, 1L)) messagef("") } notok = names(Filter(function(r) !r$is.lib.path.writeable, res)) if (length(notok) > 0L) warningf("On the following nodes the first lib path is not writeable: %s", collapse(notok)) invisible(res) } BatchJobs/R/grepLogs.R0000644000176000001440000000527612465167456014301 0ustar ripleyusers#' Grep log files for a pattern. #' #' @description #' Searches for occurence of \code{pattern} in log files. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs to grep. #' Default is all terminated jobs (done + errors). #' @param pattern [\code{character(1)}]\cr #' Pattern to search for. See \code{\link{grep}}. #' Default is \code{"warn"}. #' @param ignore.case [\code{logical(1)}]\cr #' Ignore case. See \code{\link{grep}}. #' Default is \code{TRUE}. #' @param verbose [\code{logical(1)}]\cr #' Print matches. #' Default is \code{FALSE}. #' @param range [\code{integer(1)}]\cr #' If \code{verbose} is set to \code{TRUE}, print \code{range} #' leading and trailing lines for contextual information about the warning. #' Default is \code{2}. #' @return [\code{integer}]. Ids of jobs where pattern was found in the log file. #' @family debug #' @export grepLogs = function(reg, ids, pattern = "warn", ignore.case = TRUE, verbose = FALSE, range = 2L) { checkRegistry(reg) syncRegistry(reg) if (missing(ids)) { ids = dbFindTerminated(reg) } else { nterminated = dbFindTerminated(reg, ids, negate = TRUE, limit = 1L) if (length(nterminated) > 0L) stopf("Not all jobs with provided ids have finished yet and therefore possess no log file, e.g. id=%i.", nterminated[1L]) } assertString(pattern) assertFlag(ignore.case) assertFlag(verbose) range = asCount(range) fids = dbGetFirstJobInChunkIds(reg, ids) fns = getLogFilePath(reg, ifelse(is.na(fids), ids, fids)) matched = logical(length(ids)) getLines = function(lines, match, range) { start = max(1L, match - range) stop = min(length(lines), match + range) collapse(lines[start:stop], "\n") } for(i in seq_along(fns)) { if (!file.exists(fns[i])) stopf("File '%s' does not exist.", fns[i]) # read lines from log and trim to output of job with id 'ids[i]' lines = readLines(fns[i]) start = grep(sprintf("^########## Executing jid=%i ##########$", ids[i]), lines) if (length(start) != 1L) stop("The output of the job with id=%i could not be found in file '%s' or was found more than once", ids[i], fns[i]) end = head(grep("^########## Executing jid = [0-9]+ ##########$", tail(lines, -start)), 1L) lines = lines[start:min(start+end, length(lines))] matches = grep(pattern, lines, ignore.case = ignore.case) matched[i] = (length(matches) > 0L) if (verbose && matched[i]) { messagef("\n##### Matches for job with id=%i (%s) #####", ids[i], basename(fns[i])) message(collapse(vcapply(matches, getLines, lines = lines, range = range), "\n---\n")) } } ids[matched] } BatchJobs/R/batchMap.R0000644000176000001440000000456012415510734014213 0ustar ripleyusers#' @title Maps a function over lists or vectors, adding jobs to a registry. #' #' @description #' You can then submit these jobs to the batch system. #' #' @param reg [\code{\link{Registry}}]\cr #' Empty Registry that will store jobs for the mapping. #' @param fun [\code{function}]\cr #' Function to map over \code{...}. #' @param ... [any]\cr #' Arguments to vectorize over (list or vector). #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @param use.names [\code{logical(1)}]\cr #' Store parameter names to enable named results in \code{\link{loadResults}} and some other functions. #' Default is \code{FALSE}. #' @return Vector of type \code{integer} with job ids. #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:10) #' print(reg) #' @export batchMap = function(reg, fun, ..., more.args = list(), use.names = FALSE) { checkRegistry(reg, strict = TRUE) assertFunction(fun) ddd = list(...) if (length(ddd) == 0L) return(invisible(integer(0L))) n = unique(viapply(ddd, length)) if(length(n) != 1L) stop("All args in '...' must be of the same length!") if (n == 0L) return(invisible(integer(0L))) checkMoreArgs(more.args) assertFlag(use.names) if (dbGetJobCount(reg) > 0L) stop("Registry is not empty!") info("Adding %i jobs to DB.", n) # create seeds seed = reg$seed seeds = addIntModulo(seed, seq(0L, n-1L)) # serialize pars to char vector pars = mapply(function(...) { rawToChar(serialize(list(...), connection = NULL, ascii = TRUE)) }, ..., USE.NAMES = FALSE) fun.id = saveFunction(reg, fun, more.args) # generate jobnames col if (use.names) { jobname = getArgNames(ddd) if (is.null(jobname)) jobname = rep.int(NA_character_, n) } else { jobname = rep.int(NA_character_, n) } # add jobs to DB n = dbAddData(reg, "job_def", data = data.frame(fun_id = fun.id, pars = pars, jobname = jobname)) job.def.ids = dbGetLastAddedIds(reg, "job_def", "job_def_id", n) n = dbAddData(reg, "job_status", data = data.frame(job_def_id = job.def.ids, seed = seeds)) job.ids = dbGetLastAddedIds(reg, "job_status", "job_id", n) # we can only create the dir after we have obtained the ids from the DB createShardedDirs(reg, job.ids) invisible(job.ids) } BatchJobs/R/zzz.R0000644000176000001440000000524012471624612013330 0ustar ripleyusers#' The BatchJobs package #' #' @section Additional information: #' #' \describe{ #' \item{Homepage:}{\url{https://github.com/tudo-r/BatchJobs}} #' \item{Wiki:}{\url{https://github.com/tudo-r/BatchJobs/wiki}} #' \item{FAQ:}{\url{https://github.com/tudo-r/BatchJobs/wiki/FAQ}} #' \item{Configuration:}{\url{https://github.com/tudo-r/BatchJobs/wiki/Configuration}} #' } #' #' The package currently support the following further R options, which you can set #' either in your R profile file or a script via \code{\link{options}}: #' #' \describe{ #' \item{BatchJobs.verbose}{This boolean flag can be set to \code{FALSE} to reduce the #' console output of the package operations. Usually you want to see this output in interactive #' work, but when you use the package in e.g. knitr documents, #' it clutters the resulting document too much.} #' \item{BatchJobs.check.posix}{If this boolean flag is enabled, the package checks your #' registry file dir (and related user-defined directories) quite strictly to be POSIX compliant. #' Usually this is a good idea, you do not want to have strange chars in your file paths, #' as this might results in problems when these paths get passed to the scheduler or other #' command-line tools that the package interoperates with. #' But on some OS this check might be too strict and cause problems. #' Setting the flag to \code{FALSE} allows to disable the check entirely. #' The default is \code{FALSE} on Windows systems and \code{TRUE} else.} #' } #' #' @docType package #' @name BatchJobs NULL #' @import BBmisc #' @import checkmate #' @import utils #' @import stats #' @import DBI #' @import RSQLite #' @import fail #' @import methods #' @importFrom digest digest #' @importFrom brew brew #' @importFrom sendmailR sendmail #' @importFrom stringr str_extract #' @importFrom stringr str_trim NULL .BatchJobs.conf = new.env() .onAttach = function(libname, pkgname) { if (getOption("BatchJobs.load.config", TRUE) && !isOnSlave()) { if (missing(libname) || missing(pkgname)) { # this can happen with testthat while loading from skel/ readConfs(find.package(package = "BatchJobs")) } else { readConfs(file.path(libname, pkgname)) } if (getOption("BatchJobs.verbose", default = TRUE)) packageStartupMessage(printableConf(getConfig())) } } .onLoad = function(libname, pkgname) { # checking for posix might create problem in windwos tests options(BatchJobs.check.posix = getOption("BatchJobs.check.posix", default = !isWindows())) options(BatchJobs.clear.function.env = getOption("BatchJobs.clear.function.env", default = FALSE)) if (!isOnSlave()) { assignConfDefaults() } } BatchJobs/R/WorkerLinux.R0000644000176000001440000001104012415510734014754 0ustar ripleyusers# ******************** Constructors ******************** # Construct a remote worker for a Linux machine via SSH. makeWorkerRemoteLinux = function(nodename, rhome, r.options, script, ncpus, max.jobs, max.load, nice) { makeWorker(ssh = TRUE, nodename, rhome, r.options, script, ncpus, max.jobs, max.load, nice, c("WorkerRemoteLinux", "WorkerLinux")) } # Construct a worker for local Linux machine to spawn parallel jobs. makeWorkerLocalLinux = function(r.options, script, ncpus, max.jobs, max.load, nice) { makeWorker(ssh = FALSE, "localhost", R.home(), r.options, script, ncpus, max.jobs, max.load, nice, c("WorkerLocalLinux", "WorkerLinux")) } # ******************** Interface implementation ******************** #' @export getWorkerNumberOfCPUs.WorkerLinux = function(worker) { as.integer(runWorkerCommand(worker, "number-of-cpus")) } #' @export getWorkerStatus.WorkerLinux = function(worker, file.dir) { res = runWorkerCommand(worker, "status", file.dir) setNames(as.list(as.numeric(strsplit(res, " +")[[1L]])), c("load", "n.rprocs", "n.rprocs.50", "n.jobs")) } #' @export startWorkerJob.WorkerLinux = function(worker, rfile, outfile) { runWorkerCommand(worker, "start-job", c(worker$rhome, worker$nice, worker$r.options, rfile, outfile)) } #' @export killWorkerJob.WorkerLinux = function(worker, pid) { runWorkerCommand(worker, "kill-job", pid) } #' @export listWorkerJobs.WorkerLinux = function(worker, file.dir) { res = runWorkerCommand(worker, "list-jobs", file.dir) gsub("^\\s+|\\s+$", "", res) } # ******************** Run commands on OS ******************** # Runs a command, either by SSH or directly on localhost. # @param cmd [\code{character(1)}] # System command to run. # @param args [\code{character}] # System command arguments. # Default is \code{character(0)}. # @param stdin [\code{character(1)}] # See \code{\link{system3}}. # Default is \dQuote{}. # @param stop.on.exit.code [\code{character}] # See \code{\link{system3}}. # Default is \code{TRUE}. # @param ssh [\code{logical(1)}] # Use SSH? # Default is \code{FALSE}. # @param nodename [\code{character(1)}] # Nodename for SSH. # @return See \code{\link{system3}}. runOSCommandLinux = function(cmd, args = character(0L), stdin = "", stop.on.exit.code = TRUE, ssh = FALSE, nodename) { conf = getBatchJobsConf() if (ssh) { sys.cmd = "ssh" ssh.cmd = sprintf("'%s'", collapse(c(cmd, args), sep = " ")) sys.args = c(nodename, ssh.cmd) } else { sys.cmd = cmd sys.args = args } if (conf$debug) { catf("OS cmd: %s %s", sys.cmd, collapse(sys.args, " ")) res = try(system3(sys.cmd, sys.args, stdin = stdin, stdout = TRUE, stderr = TRUE, wait = TRUE, stop.on.exit.code = stop.on.exit.code)) catf("OS result:") print(res) } else { res = system3(sys.cmd, sys.args, stdin = stdin, stdout = TRUE, stderr = TRUE, wait = TRUE, stop.on.exit.code = stop.on.exit.code) } if(is.error(res)) stopf("Error in runLinuxOSCommand: %s (cmd: %s || args: %s)", as.character(res), sys.cmd, collapse(sys.args)) res } # Find helper script on a Linux machine in package dir. # @param rhome [\code{character(1)}] # RHOME dir. # @param r.options [\code{character}] # Options for R and Rscript, one option per element of the vector, # a la "--vanilla". # @param ssh [\code{logical(1)}] # Use SSH? # Default is \code{FALSE}. # @param nodename [\code{character(1)}] # Nodename for SSH. # @return [\code{character(1)}]. Path of script. findHelperScriptLinux = function(rhome, r.options, ssh = FALSE, nodename) { # i think we dont need to quote anything here, because system2 uses shQuote if (rhome == "") rscript = "Rscript" else rscript = file.path(rhome, "bin", "Rscript") minus.e = "-e \"message(normalizePath(system.file(\\\"bin/linux-helper\\\", package = \\\"BatchJobs\\\")))\"" args = c(r.options, minus.e) # only take the last line, if stuff is printed by Rscript before our message tail(runOSCommandLinux(rscript, args, ssh = ssh, nodename = nodename)$output, 1L) } # Perform a batch helper command on a Linux machine. # @param worker [\code{\link{WorkerLinux}}] # Worker. # @param command [\code{character(1)}] # Helper command. # @param args [\code{character}] # Arguments for helper command. # See documenation of linux-helper. runWorkerCommand = function(worker, command, args = character(0L)) { # in paths can be whitespaces and other bad stuff, quote it! args = sprintf("\"%s\"", args) script.args = c(command, args) runOSCommandLinux(worker$script, script.args, ssh = worker$ssh, nodename = worker$nodename)$output } BatchJobs/R/scheduleWorkerJobs.R0000644000176000001440000000252612415510734016300 0ustar ripleyusers# is a worker busy, see rules below getWorkerSchedulerStatus = function(worker) { # we have already used up our maximal load on this node if (worker$status$n.jobs >= worker$max.jobs) return("J") # should not have too much load average if (worker$status$load[1L] > worker$max.load) return("L") # there are already ncpus expensive R jobs running on the node if (worker$status$n.rprocs.50 >= worker$ncpus) return("R") # should not have too many R sessions open if(worker$status$n.rprocs >= 3 * worker$ncpus) return("r") # else all clear, submit the job! return("A") } # update status of worker IN PLACE updateWorker = function(worker, file.dir, tdiff) { time = now() if (worker$available == "A" || time - worker$last.update >= tdiff) { worker$last.update = time worker$status = getWorkerStatus(worker, file.dir) worker$available = getWorkerSchedulerStatus(worker) } } # find worker via isBusyWorker and update workers while looking # workers with a low load are more likely to be selected when there are # multiple workers available findWorker = function(workers, file.dir, tdiff) { lapply(workers, updateWorker, file.dir = file.dir, tdiff = tdiff) rload = vnapply(workers, function(w) w$status$load / w$ncpus) Find(function(w) w$available=="A", sample(workers, prob = 1 / (rload + 0.1)), nomatch = NULL) } BatchJobs/R/applyJobFunction.R0000644000176000001440000000126512415510734015761 0ustar ripleyusers#' applyJobFunction #' ONLY FOR INTERNAL USAGE. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param job [\code{\link{Job}}]\cr #' Job. #' @param cache [\code{FileCache}]\cr #' Instance of \code{\link[BBmisc]{makeFileCache}}. #' @return [any]. Result of job. #' @keywords internal #' @export applyJobFunction = function(reg, job, cache) { UseMethod("applyJobFunction") } #' @method applyJobFunction Registry #' @export applyJobFunction.Registry = function(reg, job, cache) { fn = file.path(getFunDir(reg$file.dir), sprintf("%s.RData", job$fun.id)) stuff = cache(fn, parts = c("fun", "more.args"), simplify = FALSE) do.call(stuff$fun, c(job$pars, stuff$more.args)) } BatchJobs/R/addJob.R0000644000176000001440000000032712465167456013672 0ustar ripleyuserssaveFunction = function(reg, fun, more.args) { fun = checkUserFunction(fun) fun.id = digest(list(fun, more.args)) save2(file = getFunFilePath(reg, fun.id), fun = fun, more.args = more.args) return(fun.id) } BatchJobs/MD50000644000176000001440000002475012502327541012462 0ustar ripleyusers669b0c563ac07342fc05c1130daba63f *DESCRIPTION 9d6fc77f9cd80b60f62389c553a386c7 *LICENSE c1d27fdb0b1ce6f1360590bfae89ebdc *NAMESPACE 1c6909e2abfa14c9c9c242373d29a585 *NEWS 830c09db600e6a97353d2b71b659389a *R/Exports.R 25659bd930b0b9e1b0a2a860fce7c24d *R/Job.R 62cd2261a54d3222957b83f8c6b2f762 *R/Packages.R ed357b4779d8f7258f0788aa8752a8eb *R/Registry.R a1bc7895ecbf51177ffbd38fe155c904 *R/Resources.R 1ee9e2a31c312e3e07a1da7b8218127e *R/Worker.R f0761f2144e454934e45c951a5cf180f *R/WorkerLinux.R 658f8c315735400ca5cf9fc0a2e33016 *R/addJob.R d0d1fd441c719e0683b07b5e3280ceb2 *R/applyJobFunction.R e2c7e5dfe2ba536e79094282d1ae7172 *R/batchApply.R b1a9f87aeaf87939402c746676a4b6ff *R/batchExpandGrid.R 8e27ab70469e436f81fa1786a77b2ff2 *R/batchMap.R 5e5a54ecbc6b5635bf40edfd268151dc *R/batchMapQuick.R 24575e5f89f1c6efde750a3087198c46 *R/batchMapResults.R 2ef351b2040e7c9c7811d91ae2798be5 *R/batchReduce.R 722ff539d18b31c0b572b950040c39aa *R/batchReduceResults.R f172e35b613a86151dbc6c82779bb18b *R/buffer.R 2b002f7db482bab14963b9218654e5bd *R/callFunctionOnSSHWorkers.R fab25278bcc83a38f8db06ae6b817afb *R/checkIdValid.R c8487612d382dbfaec57e329112b1166 *R/clusterFunctions.R 2b7177a561f594e2d5ff9a790ef476a6 *R/clusterFunctionsHelpers.R f6f6934bcbc0cc1be3a49c8aaa42d959 *R/clusterFunctionsInteractive.R 5916debf206839da198f9158d68c669a *R/clusterFunctionsLSF.R 4e507dccaf6f9b6ae58ced8e92b42018 *R/clusterFunctionsLocal.R 5dba0577239626ab59a8b602e411fff8 *R/clusterFunctionsMulticore.R 9b1df631be1f8697a55726975c6aef01 *R/clusterFunctionsSGE.R d84112981a443139cb4db28bae73be3d *R/clusterFunctionsSLURM.R 7fb9ba72f0e2fdfa2af89fb4c98a01af *R/clusterFunctionsSSH.R 6bbeef6f3e2ef20086f2c0649dcccd66 *R/clusterFunctionsTorque.R 9c59affcb0a041f21ef0a2676cd799a4 *R/conf.R 8b61693de82c96c5aa598131ef6914c5 *R/database.R 3d76275ec0683ed18c4295e65de23c92 *R/debugMulticore.R 6b6d4869bbfd8b4ec5aab2eaf949f331 *R/debugSSH.R f7d4cf1e834c131b99e19fd369d4ee7b *R/doJob.R 0280a243dae1f5a9cf9cb9af6767e774 *R/filenames.R dba6afba73c7fb1cd55ea2b51d2e6e27 *R/filterResults.R 02a6d8fc83984a873dc232c743c7295e *R/findJobs.R 1d912de46211f69f65aec610c1491778 *R/findStatus.R 612f8139e4dbff8b3efbad3717c98adf *R/getErrorMessages.R 8f62497300eff37cff063a7c653400d7 *R/getJob.R 8a44315fa63406e41f96d48dd0484878 *R/getJobInfo.R b8132fa989a124fedb8089d29ce66a7f *R/getJobParamDf.R baafacda579b53b29c8533827b1eafca *R/getLogFiles.R 02c342cac122cffdb158ef48be98d701 *R/getSSHWorkersInfo.R 10f83e76fd57c643762d6de273d7c8cf *R/grepLogs.R a63f190c1006ebeceea02e9e0f50461b *R/helpers.R fc7b467082b004f08239e5c3796a760a *R/installPackagesOnSSHWorkers.R 6eff96e443b2338447dba727010c5fb1 *R/killJobs.R 1dd977e52321c413b290710b2b2f3902 *R/loadResult.R d3c9e8a83675dfda5ae8a080d320f205 *R/loadResults.R c39ca602985fb8a35f1b624139fd1667 *R/reduceResults.R 2ae6fdfe8980f2f50627b3f893af9855 *R/resetJobs.R a0cbe2f88a748469eba52c0ebb72f498 *R/sanitizePath.R 54f6af0f47082c1d0b076d67372a2cb5 *R/scheduleWorkerJobs.R a3030fb66c111c3fbb18ad66b2b17a99 *R/sendMail.R 6b89343de03f3544da4aa8b29c74b95b *R/setJobFunction.R 7de273b8cd6ae1e370168de0b7cb472c *R/setJobNames.R 22b64461209ffe569650e3a0ecf11b21 *R/showClusterStatus.R 366422a110f5fa5212ebb20f79c1fc8e *R/showLog.R 56f2170d376015c30c3027072c59e65d *R/showStatus.R 76c43084255145485a921172e823b392 *R/sourceRegistryFiles.R 5e518117918a134d18af2cb76925c70c *R/submitJobs.R 438d9ce1b165023845180657554624a0 *R/sweepRegistry.R 399433ccf3664be08626a54958b4da71 *R/syncRegistry.R 7c5b4b2cf414c2cf54438395cb502c83 *R/testJob.R e5bc30275b667c90d242988bf462c2f9 *R/updateRegistry.R a0820fcb67377e2341a786b190503f73 *R/waitForJobs.R d39da9d38fabf5038ffb761343f3fdb0 *R/writeRscripts.R 6caeb5841206f60777a8d8b882e8e7e1 *R/zzz.R 1f72c38068bf885bbcb699519e99e8a8 *inst/CITATION d9ddf818a9837e0159ce6826293d0902 *inst/bin/linux-helper b7e4f63a659e5fe827946829a6167b2c *inst/etc/BatchJobs_global_config.R c33d92119072e029b10048bbe219983f *man/BatchJobs.Rd b5ec694af4753b04128a1f03b4487300 *man/addRegistryPackages.Rd 79bfb5c17d014d1ae4c497bb860d7c76 *man/addRegistrySourceDirs.Rd c12a96b74d388c8c96a929e95be239cf *man/addRegistrySourceFiles.Rd a7d630f41a7e7a1c6367110cec2a1646 *man/applyJobFunction.Rd a2466bb3474893e979062b65140f34d5 *man/batchExpandGrid.Rd cbc6dc06ee67a508eec5ecaac8eb3187 *man/batchExport.Rd b88aceebdf8286f24ebdac5a04cba71d *man/batchMap.Rd 32ca6738186ad91d53a7853392c0d4b7 *man/batchMapQuick.Rd 4167d1fb545151653f1613a7458b9391 *man/batchMapResults.Rd 02ebdee1e6d025a05f8ce863fa7ceacc *man/batchReduce.Rd 30f5e2b73dc38ba3ecab922cc2fff4f9 *man/batchReduceResults.Rd e059f8b801db02ed02700d8644c21c69 *man/batchUnexport.Rd 3ce60fd9038705d7a35cabb96811043e *man/callFunctionOnSSHWorkers.Rd 4a01e373a76b760ed3021dcf31352156 *man/cfBrewTemplate.Rd 336b61270f8be2d3250676d804875464 *man/cfHandleUnknownSubmitError.Rd 93aac79336772fe1cf718e3bae53f289 *man/cfKillBatchJob.Rd 98d845a3d5fa69274b2cd67b18a4b354 *man/cfReadBrewTemplate.Rd 2a4b81e0c5a2f9d6e544373cc9f58e16 *man/configuration.Rd bb3cd6dd75423a86d730805405bbad2f *man/copyRequiredJobFiles.Rd 7bd4aa32db5f879ebe7351598e33b9c4 *man/dbCreateJobDefTable.Rd e8c2d0dc5b866b2334f951af59ac1e37 *man/dbGetJobs.Rd 2e7882903b38c83cd65033fd4d369a76 *man/debugMulticore.Rd 21f948e592fabac5887519b25d878c89 *man/debugSSH.Rd 7f19b3937f65854380591392ff19e9bf *man/filterResults.Rd b12e9ddf4272c29f0b29c625f3a4bc56 *man/findJobs.Rd 75fd4484471e18998c6784d92b3068a2 *man/findState.Rd 2976b708ad6a7bdcc9cf9dfbd6bc72b3 *man/getConfig.Rd d8162a5b75b924d29e403486a0031b7b *man/getErrorMessages.Rd 9ed7a1cb8e9b419cd04fb2fe7a4d9f3c *man/getJob.Rd 540908211cbfd2ba7d74dd98da265105 *man/getJobIds.Rd 36d60023b1c04d17df177e70d1b152b2 *man/getJobInfo.Rd 4112b1643d65bdec6fba4e4d2d759b36 *man/getJobNr.Rd dcaf5fb5a90eddb424023e838df33015 *man/getJobParamDf.Rd ef3409c7b485f86d55a393085182ec40 *man/getJobResources.Rd ead2c6cb3dc2e668e18246094d663413 *man/getJobs.Rd 21142e1a3e345bc8ee797084bdbce7f2 *man/getLogFiles.Rd e12e8aa0092b1bf6880d2e6ab6d4c68c *man/getResources.Rd b02373c6f33045e3a013b50f1c128364 *man/getSSHWorkersInfo.Rd a9ce8938fa7a7914566b0134a1c94c53 *man/grepLogs.Rd 0c0dbb846edad0bba798b2f497b36e83 *man/installPackagesOnSSHWorkers.Rd 361496df105474c0f09d58589eb476c6 *man/killJobs.Rd c0342de39815e04e1207a435a8403ac2 *man/loadConfig.Rd 672a36f7d79fd7cc198a8a5dd22eb0a9 *man/loadExports.Rd 90a1bb194a00859cbf3051c6fcba8a79 *man/loadRegistry.Rd 2f5a5471181ab5deda0b1de65d60974c *man/loadResult.Rd 11c59e1ef7419886ca4046397d9b670c *man/loadResults.Rd 1c7a973e1e9ac7ec6e463e5710b23c55 *man/makeClusterFunctions.Rd db7ac7762d4a906ca221dfa4b2450437 *man/makeClusterFunctionsInteractive.Rd ce18a3a7c869e3fa39ee834e5b117c05 *man/makeClusterFunctionsLSF.Rd e20998cb9fad29de6683fa68594d1d38 *man/makeClusterFunctionsLocal.Rd efa77f9cefe3c7947467155df822b7a0 *man/makeClusterFunctionsMulticore.Rd de8a37bf1e54e51a9c9ed8ceaf8054b2 *man/makeClusterFunctionsSGE.Rd 99839a3d6a9a4bd26057bd0134535954 *man/makeClusterFunctionsSLURM.Rd b9b3293c52d0fedf7e7231e92067fdb0 *man/makeClusterFunctionsSSH.Rd d02e9dd2141c34d217bc761c82951cfa *man/makeClusterFunctionsTorque.Rd 9b29296396026dcda2e785ca2e69c59c *man/makeJob.Rd cab967bfa1f69120912d58f18917a5d6 *man/makeRegistry.Rd 5e97a4f9038447ffea8d1e9428042bba *man/makeSSHWorker.Rd 6ffe37068b4a18a073a943e25a337362 *man/makeSubmitJobResult.Rd 04bae320fddf028cba94f7e0f449abed *man/reduceResults.Rd 0a51d2bcf1f929f2abeab0c39045a0f8 *man/removeRegistry.Rd eaf146dd965101809c2a059c9decfd3b *man/removeRegistryPackages.Rd 0962449b90398d14552915ef4e42cbe4 *man/removeRegistrySourceDirs.Rd 8aff9b716ccc4daaccdc85a54153df2c *man/removeRegistrySourceFiles.Rd 7ab4ea479e9a76079d967d178695f683 *man/resetJobs.Rd 9dd84910ca4cf6f03a7ef61883279434 *man/sanitizePath.Rd 08512b5b2fef0f2eb562f3b2024420c5 *man/setConfig.Rd b16c844a949fc2def3ac2594c0588994 *man/setJobFunction.Rd 64ec4317deddde9699d50fc9dcaaac89 *man/setJobNames.Rd d25e318ec7547996d0d66e495775db39 *man/showClusterStatus.Rd 0e72f20d64e2a56d1666e7be5d28e649 *man/showLog.Rd a0e7d21e29bc6dbe7ca381fd816363c8 *man/showStatus.Rd 2e71bb5ec5ece988e6bb288100d00f00 *man/sourceRegistryFiles.Rd 8f0e1138d9193b355486954dedafdb89 *man/submitJobs.Rd c1add407fc65ef16d82fd99a630ec038 *man/sweepRegistry.Rd 3401580c8116cb01b30cbbd091ab922c *man/testJob.Rd 5eaa82245eabf38894f7b36e61045457 *man/updateRegistry.Rd e12ea267ed386605b97b989b9cd4da76 *man/waitForJobs.Rd 72cdbf3f20e95b58cd1bf26e60659e8b *tests/run-all.R 33861590f3a2cef2ee3867af89e505b5 *tests/testthat/helpers.R e2a51c9197bfd500a777d41921028658 *tests/testthat/test_batchExpandGrid.R 01d5879c5226a2afbcf1aa6b17d61456 *tests/testthat/test_batchMap.R c187957f7b4052bf6fac9c3177bdab93 *tests/testthat/test_batchMapQuick.R 21d86cbbc521c5961063e8f072649fa3 *tests/testthat/test_batchMapResults.R 0454d68de8d66a7c2c30e7497c7aae23 *tests/testthat/test_batchReduce.R cd1a71f7750f18268e2b31ef35010d43 *tests/testthat/test_batchReduceResults.R b313cd78a45958c572d2e92f9dcdaf29 *tests/testthat/test_database.R 73a2d7324b48ce42138c671ffaba64af *tests/testthat/test_doJob.R 95291d5e594cb10c153ba0c93c406955 *tests/testthat/test_export.R e56ae4875570d3c14a647fe61a76c701 *tests/testthat/test_filterResults.R ebe1497f20792bee8ff9d4bb9c938a37 *tests/testthat/test_findJobs.R e2410eaba2fc0eb3f2a013ad23f6b423 *tests/testthat/test_findStatus.R 39e6e89d03c362f58348f60d1acd85a4 *tests/testthat/test_getJob.R 3ce13ef84d30b98ea9db124312bb8b8a *tests/testthat/test_getJobInfo.R fe36cc94d24eefd02113595f1d53e966 *tests/testthat/test_getJobParamDf.R 4a01f997f9bee2d581c2daf17501c0a9 *tests/testthat/test_getLogFiles.R c5116748ae0445761773a0daa28bae12 *tests/testthat/test_grepLogs.R 167cfb3d345c6138631d828797847d51 *tests/testthat/test_loadResults.R f419c0f426346d65b586f147cf53c3ec *tests/testthat/test_makeRegistry.R adca524e07b020f34071b16dca3a6a5a *tests/testthat/test_packages.R bc777d9f9e78cece43eaadb1b0ed1781 *tests/testthat/test_reduceResults.R e41a108af2685c8f12c4ea889d2850ce *tests/testthat/test_resetJobs.R 95526b2defbf0e833f5ee1e3403dbc2b *tests/testthat/test_resources.R d4abfc3fa39eaeff66843a6e442c9360 *tests/testthat/test_sanitizePath.R 3498c69bc5a65ec0603a0704d1bbae04 *tests/testthat/test_setJobFunction.R b56e0a7adca05051de44e52954576542 *tests/testthat/test_showStatus.R dbc86bd58a082e3fd0adba10d6e12594 *tests/testthat/test_sourceRegistryFiles.R f1a8703f9f8a55a5d4c538a9138f5385 *tests/testthat/test_sqlquotes.R be0baf58829e85b7aa63a044cfd786e1 *tests/testthat/test_submitJobs.R 6c54b743a389973f4d8f73246b8b816b *tests/testthat/test_sweepRegistry.R 73bca9b56fbc16b0c48691e81ce5e21c *tests/testthat/test_testJob.R 1a7c80ff18054815b6eef0225f8ec350 *tests/testthat/test_waitForJobs.R cb842420345de10a618937f551c9fa01 *tests/testthat/test_zzz_cleanup.R BatchJobs/DESCRIPTION0000644000176000001440000000177312502327541013660 0ustar ripleyusersPackage: BatchJobs Title: Batch Computing with R Description: Provides Map, Reduce and Filter variants to generate jobs on batch computing systems like PBS/Torque, LSF, SLURM and Sun Grid Engine. Multicore and SSH systems are also supported. For further details see the project web page. Author: Bernd Bischl , Michel Lang , Henrik Bengtsson Maintainer: Bernd Bischl URL: https://github.com/tudo-r/BatchJobs BugReports: https://github.com/tudo-r/BatchJobs/issues MailingList: batchjobs@googlegroups.com License: BSD_2_clause + file LICENSE Depends: R (>= 2.15.0), BBmisc (>= 1.9), methods Imports: brew, checkmate (>= 1.5.1), DBI, digest, fail (>= 1.2), parallel, RSQLite (>= 1.0.0), sendmailR, stats, stringr, utils Suggests: MASS, testthat LazyData: yes ByteCompile: yes Version: 1.6 Packaged: 2015-03-18 16:40:05 UTC; bischl NeedsCompilation: no Repository: CRAN Date/Publication: 2015-03-18 18:01:21 BatchJobs/man/0000755000176000001440000000000012502316642012715 5ustar ripleyusersBatchJobs/man/makeClusterFunctionsLSF.Rd0000644000176000001440000000346512465167456017750 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsLSF.R \name{makeClusterFunctionsLSF} \alias{makeClusterFunctionsLSF} \title{Create cluster functions for LSF systems.} \usage{ makeClusterFunctionsLSF(template.file, list.jobs.cmd = c("bjobs", "-u $USER", "-w")) } \arguments{ \item{template.file}{[\code{character(1)}]\cr Path to a brew template file that is used for the PBS job file.} \item{list.jobs.cmd}{[\code{character}]\cr Change default system command / options to list jobs. The first entry is the command, the following the options. See \code{\link[BBmisc]{system3}}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Job files are created based on the brew template \code{template.file}. This file is processed with brew and then submitted to the queue using the \code{bsub} command. Jobs are killed using the \code{bkill} command and the list of running jobs is retrieved using \code{bjobs -u $USER -w}. The user must have the appropriate privileges to submit, delete and list jobs on the cluster (this is usually the case). The template file can access all arguments passed to the \code{submitJob} function, see here \code{\link{ClusterFunctions}}. It is the template file's job to choose a queue for the job and handle the desired resource allocations. Examples can be found on \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfLSF}. } \seealso{ Other clusterFunctions: \code{\link{ClusterFunctions}}, \code{\link{makeClusterFunctions}}; \code{\link{makeClusterFunctionsInteractive}}; \code{\link{makeClusterFunctionsLocal}}; \code{\link{makeClusterFunctionsMulticore}}; \code{\link{makeClusterFunctionsSGE}}; \code{\link{makeClusterFunctionsSLURM}}; \code{\link{makeClusterFunctionsSSH}}; \code{\link{makeClusterFunctionsTorque}} } BatchJobs/man/loadResults.Rd0000644000176000001440000000244712465167456015533 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/loadResults.R \name{loadResults} \alias{loadResults} \title{Loads result files for id vector.} \usage{ loadResults(reg, ids, part = NA_character_, simplify = FALSE, use.names = "ids", missing.ok = FALSE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all done jobs.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{simplify}{[\code{logical(1)}]\cr Should the result be simplified to a vector, matrix or higher dimensional array if possible? Default is \code{TRUE}.} \item{use.names}{[\code{character(1)}]\cr Name the results with job ids (\dQuote{ids}), stored job names (\dQuote{names}) or return a unnamed result (\dQuote{none}). Default is \code{ids}.} \item{missing.ok}{[\code{logical(1)}]\cr If \code{FALSE} an error is thrown if the results are not found. Otherwise missing results are imputed to \code{NULL}. Default is \code{FALSE}.} } \value{ [\code{list}]. Results of jobs as list, possibly named by ids. } \description{ Loads result files for id vector. } \seealso{ \code{\link{reduceResults}} } BatchJobs/man/addRegistrySourceDirs.Rd0000644000176000001440000000170312465167456017510 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{addRegistrySourceDirs} \alias{addRegistrySourceDirs} \title{Add source dirs to registry.} \usage{ addRegistrySourceDirs(reg, src.dirs, src.now = TRUE) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{src.dirs}{[\code{character}]\cr Paths to add to registry. See \code{\link{makeRegistry}}.} \item{src.now}{[\code{logical(1)}] Source files now on master? Default is \code{TRUE}.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{src.dirs} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}; \code{\link{addRegistrySourceFiles}}; \code{\link{batchExport}}; \code{\link{batchUnexport}}; \code{\link{loadExports}}; \code{\link{removeRegistryPackages}}; \code{\link{removeRegistrySourceDirs}}; \code{\link{removeRegistrySourceFiles}} } BatchJobs/man/updateRegistry.Rd0000644000176000001440000000065612465167456016245 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/updateRegistry.R \name{updateRegistry} \alias{updateRegistry} \title{ONLY FOR INTERNAL USAGE.} \usage{ updateRegistry(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} } \value{ [any]. Updated \code{\link{Registry}} or \code{FALSE} if no updates were performed. } \description{ ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/dbCreateJobDefTable.Rd0000644000176000001440000000055412465167456016762 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/database.R \name{dbCreateJobDefTable} \alias{dbCreateJobDefTable} \title{ONLY FOR INTERNAL USAGE.} \usage{ dbCreateJobDefTable(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} } \value{ Nothing. } \description{ ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/getJobs.Rd0000644000176000001440000000100012465167456014607 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/getJob.R \name{getJobs} \alias{getJobs} \title{Get jobs from registry by id.} \usage{ getJobs(reg, ids, check.ids = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} \item{check.ids}{[\code{logical(1)}]\cr Check the job ids? Default is \code{TRUE}.} } \value{ [list of \code{\link{Job}}]. } \description{ Get jobs from registry by id. } BatchJobs/man/setJobNames.Rd0000644000176000001440000000115612465167456015440 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/setJobNames.R \name{setJobNames} \alias{setJobNames} \title{Set job names.} \usage{ setJobNames(reg, ids, jobnames) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} \item{jobnames}{[\code{character}]\cr Character vector with length equal to \code{length(ids)}. \code{NA} removes the names stored in the registry. A single \code{NA} is replicated to match the length of ids provided.} } \value{ Named vector of job ids. } \description{ Set job names. } BatchJobs/man/applyJobFunction.Rd0000644000176000001440000000104512465167456016511 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/applyJobFunction.R \name{applyJobFunction} \alias{applyJobFunction} \title{applyJobFunction ONLY FOR INTERNAL USAGE.} \usage{ applyJobFunction(reg, job, cache) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{job}{[\code{\link{Job}}]\cr Job.} \item{cache}{[\code{FileCache}]\cr Instance of \code{\link[BBmisc]{makeFileCache}}.} } \value{ [any]. Result of job. } \description{ applyJobFunction ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/installPackagesOnSSHWorkers.Rd0000644000176000001440000000235412465167456020564 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/installPackagesOnSSHWorkers.R \name{installPackagesOnSSHWorkers} \alias{installPackagesOnSSHWorkers} \title{Install packages on SSH workers.} \usage{ installPackagesOnSSHWorkers(nodenames, pkgs, repos = getOption("repos"), consecutive = TRUE, show.output = consecutive, ...) } \arguments{ \item{nodenames}{[\code{character}]\cr Nodenames of workers.} \item{pkgs}{[\code{character}]\cr See \code{\link{install.packages}}.} \item{repos}{[\code{character}]\cr See \code{\link{install.packages}}. If the user must be queried this is of course done on the master.} \item{consecutive}{[\code{logical(1)}]\cr See \code{\link{callFunctionOnSSHWorkers}}. Default is \code{TRUE}.} \item{show.output}{[\code{logical(1)}]\cr See \code{\link{callFunctionOnSSHWorkers}}. Default is \code{consecutive}.} \item{...}{[any]\cr Passed to \code{\link{install.packages}}.} } \value{ Nothing. } \description{ Installation is done via \code{\link{callFunctionOnSSHWorkers}} and \code{\link{install.packages}}. Note that as usual the function tries to install the packages into the first path of \code{.libPaths()} of each each worker. } \seealso{ \code{\link{callFunctionOnSSHWorkers}} } BatchJobs/man/makeClusterFunctionsMulticore.Rd0000644000176000001440000000515312471624612021250 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsMulticore.R \name{makeClusterFunctionsMulticore} \alias{makeClusterFunctionsMulticore} \title{Use multiple cores on local Linux machine to spawn parallel jobs.} \usage{ makeClusterFunctionsMulticore(ncpus = max(getOption("mc.cores", detectCores()) - 1, 1), max.jobs, max.load, nice, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script) } \arguments{ \item{ncpus}{[\code{integer(1)}]\cr Number of VPUs of worker. Default is to use all cores but one, where total number of cores "available" is given by option \code{\link[base:options]{mc.cores}} and if that is not set it is inferred by \code{\link[parallel]{detectCores}}.} \item{max.jobs}{[\code{integer(1)}]\cr Maximal number of jobs that can run concurrently for the current registry. Default is \code{ncpus}.} \item{max.load}{[\code{numeric(1)}]\cr Load average (of the last 5 min) at which the worker is considered occupied, so that no job can be submitted. Default is \code{ncpus-1}.} \item{nice}{[\code{integer(1)}]\cr Process priority to run R with set via nice. Integers between -20 and 19 are allowed. If missing, processes are not nice'd and the system default applies (usually 0).} \item{r.options}{[\code{character}] Options for R and Rscript, one option per element of the vector, a la \dQuote{--vanilla}. Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}.} \item{script}{[\code{character(1)}]\cr Path to helper bash script which interacts with the worker. You really should not have to touch this, as this would imply that we have screwed up and published an incompatible version for your system. This option is only provided as a last resort for very experienced hackers. Note that the path has to be absolute. This is what is done in the package: \url{https://github.com/tudo-r/BatchJobs/blob/master/inst/bin/linux-helper} Default means to take it from package directory.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Jobs are spawned by starting multiple R sessions on the commandline (similar like on true batch systems). Packages \code{parallel} or \code{multicore} are not used in any way. } \seealso{ Other clusterFunctions: \code{\link{ClusterFunctions}}, \code{\link{makeClusterFunctions}}; \code{\link{makeClusterFunctionsInteractive}}; \code{\link{makeClusterFunctionsLSF}}; \code{\link{makeClusterFunctionsLocal}}; \code{\link{makeClusterFunctionsSGE}}; \code{\link{makeClusterFunctionsSLURM}}; \code{\link{makeClusterFunctionsSSH}}; \code{\link{makeClusterFunctionsTorque}} } BatchJobs/man/killJobs.Rd0000644000176000001440000000355012465167456014777 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/killJobs.R \name{killJobs} \alias{killJobs} \title{Kill some jobs on the batch system.} \usage{ killJobs(reg, ids, progressbar = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs to kill. Default is none.} \item{progressbar}{[\code{logical(1)}]\cr Set to \code{FALSE} to disable the progress bar. To disable all progress bars, see \code{\link[BBmisc]{makeProgressBar}}.} } \value{ [\code{integer}]. Ids of killed jobs. } \description{ Kill jobs which have already been submitted to the batch system. If a job is killed its internal state is reset as if it had not been submitted at all. The function informs if (a) the job you want to kill has not been submitted, (b) the job has already terminated, (c) for some reason no batch job id is available. In all 3 cases above, nothing is changed for the state of this job and no call to the internal kill cluster function is generated. In case of an error when killing, the function tries - after a short sleep - to kill the remaining batch jobs again. If this fails again for some jobs, the function gives up. Only jobs that could be killed are reset in the DB. } \examples{ \dontrun{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) Sys.sleep(x) batchMap(reg, f, 1:10 + 5) submitJobs(reg) waitForJobs(reg) # kill all jobs currently _running_ killJobs(reg, findRunning(reg)) # kill all jobs queued or running killJobs(reg, findNotTerminated(reg)) } } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{debugSSH}}; \code{\link{getErrorMessages}}; \code{\link{getJobInfo}}; \code{\link{getLogFiles}}; \code{\link{grepLogs}}; \code{\link{resetJobs}}; \code{\link{setJobFunction}}; \code{\link{showLog}}; \code{\link{testJob}} } BatchJobs/man/makeRegistry.Rd0000644000176000001440000000671712465167456015704 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Registry.R \name{makeRegistry} \alias{Registry} \alias{makeRegistry} \title{Construct a registry object.} \usage{ makeRegistry(id, file.dir, sharding = TRUE, work.dir, multiple.result.files = FALSE, seed, packages = character(0L), src.dirs = character(0L), src.files = character(0L), skip = TRUE) } \arguments{ \item{id}{[\code{character(1)}]\cr Name of registry. Displayed e.g. in mails or in cluster queue.} \item{file.dir}{[\code{character(1)}]\cr Path where files regarding the registry / jobs should be saved. Default is \dQuote{-files} in current working directory if \code{id} is set.} \item{sharding}{[\code{logical(1)}]\cr Enable sharding to distribute result files into different subdirectories? Important if you have many experiments. Default is \code{TRUE}.} \item{work.dir}{[\code{character(1)}]\cr Working directory for R process when experiment is executed. Default is the current working directory when registry is created.} \item{multiple.result.files}{[\code{logical(1)}]\cr Should a result file be generated for every list element of the returned list of the job function? Note that the function provided to \code{\link{batchMap}} or \code{\link{batchReduce}} must return a named list if this is set to \code{TRUE}. The result file will be named \dQuote{-result-.RData} instead of \dQuote{-result.RData}. Default is \code{FALSE}.} \item{seed}{[\code{integer(1)}]\cr Start seed for experiments. The first experiment in the registry will use this seed, for the subsequent ones the seed is incremented by 1. Default is a random number from 1 to \code{.Machine$integer.max/2}.} \item{packages}{[\code{character}]\cr Packages that will always be loaded on each node. Default is \code{character(0)}.} \item{src.dirs}{[\code{character}]\cr Directories containing R scripts to be sourced on registry load (both on slave and master). Files not matching the pattern \dQuote{\\.[Rr]$} are ignored. Useful if you have many helper functions that are needed during the execution of your jobs. These files should only contain function definitions and no executable code. Default is \code{character(0)}.} \item{src.files}{[\code{character}]\cr R scripts files to be sourced on registry load (both on slave and master). Useful if you have many helper functions that are needed during the execution of your jobs. These files should only contain function and constant definitions and no long running, executable code. These paths are considered to be relative to your \code{work.dir}. As a last remedy in problematic cases you can use absolute paths, by passing paths that start with \dQuote{/}, see the comment about \code{file.dir} and \code{work.dir} above, where we allow the same thing. Note that this is a less portable approach and therefore usually a less good idea. Default is \code{character(0)}.} \item{skip}{[\code{logical(1)}]\cr Skip creation of a new registry if a registry is found in \code{file.dir}. Defaults to \code{TRUE}.} } \value{ [\code{\link{Registry}}] } \description{ Note that if you don't want links in your paths (\code{file.dir}, \code{work.dir}) to get resolved and have complete control over the way the path is used internally, pass an absolute path which begins with \dQuote{/}. } \details{ Every object is a list that contains the passed arguments of the constructor. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) print(reg) } BatchJobs/man/getErrorMessages.Rd0000644000176000001440000000147612465167456016514 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/getErrorMessages.R \name{getErrorMessages} \alias{getErrorMessages} \title{Get error messages of jobs.} \usage{ getErrorMessages(reg, ids) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs with errors.} } \value{ [\code{character}]. Error messages for jobs as character vector\cr \code{NA} if job has terminated successfully. } \description{ Get error messages of jobs. } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{debugSSH}}; \code{\link{getJobInfo}}; \code{\link{getLogFiles}}; \code{\link{grepLogs}}; \code{\link{killJobs}}; \code{\link{resetJobs}}; \code{\link{setJobFunction}}; \code{\link{showLog}}; \code{\link{testJob}} } BatchJobs/man/configuration.Rd0000644000176000001440000000067212465167456016077 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/conf.R \name{configuration} \alias{.BatchJobs.R} \alias{configuration} \title{BatchJobs configuration.} \description{ In order to understand how the package should be configured please read \url{https://github.com/tudo-r/BatchJobs/wiki/Configuration}. } \seealso{ Other conf: \code{\link{getConfig}}; \code{\link{loadConfig}}; \code{\link{setConfig}} } BatchJobs/man/makeJob.Rd0000644000176000001440000000153312465167456014575 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Job.R \name{makeJob} \alias{Job} \alias{makeJob} \title{Creates a job description.} \usage{ makeJob(id = NA_integer_, fun, fun.id = digest(fun), pars, name, seed) } \arguments{ \item{id}{[\code{integer(1)}]\cr Job id, determined by DB autoincrement. Default is \code{NA}.} \item{fun}{[\code{function}]\cr Job function to apply on parameters.} \item{fun.id}{[\code{character(1)}]\cr Id used to store function on disk. Default is \code{digest(fun)}.} \item{pars}{[\code{list}]\cr Parameter list for job function.} \item{name}{[\code{character(1)}]\cr Alias name for this job.} \item{seed}{[\code{integer(1)}]\cr Random seed for job.} } \description{ Usually you will not do this manually. Every object is a list that contains the passed arguments of the constructor. } BatchJobs/man/makeClusterFunctionsSGE.Rd0000644000176000001440000000345612465167456017742 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsSGE.R \name{makeClusterFunctionsSGE} \alias{makeClusterFunctionsSGE} \title{Create cluster functions for Sun Grid Engine systems.} \usage{ makeClusterFunctionsSGE(template.file, list.jobs.cmd = c("qstat", "-u $USER")) } \arguments{ \item{template.file}{[\code{character(1)}]\cr Path to a brew template file that is used for the PBS job file.} \item{list.jobs.cmd}{[\code{character}]\cr Change default system command / options to list jobs. The first entry is the command, the following the options. See \code{\link[BBmisc]{system3}}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Job files are created based on the brew template \code{template.file}. This file is processed with brew and then submitted to the queue using the \code{qsub} command. Jobs are killed using the \code{qdel} command and the list of running jobs is retrieved using \code{qselect}. The user must have the appropriate privileges to submit, delete and list jobs on the cluster (this is usually the case). The template file can access all arguments passed to the \code{submitJob} function, see here \code{\link{ClusterFunctions}}. It is the template file's job to choose a queue for the job and handle the desired resource allocations. Examples can be found on \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfSGE}. } \seealso{ Other clusterFunctions: \code{\link{ClusterFunctions}}, \code{\link{makeClusterFunctions}}; \code{\link{makeClusterFunctionsInteractive}}; \code{\link{makeClusterFunctionsLSF}}; \code{\link{makeClusterFunctionsLocal}}; \code{\link{makeClusterFunctionsMulticore}}; \code{\link{makeClusterFunctionsSLURM}}; \code{\link{makeClusterFunctionsSSH}}; \code{\link{makeClusterFunctionsTorque}} } BatchJobs/man/batchExport.Rd0000644000176000001440000000232312465167456015506 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Exports.R \name{batchExport} \alias{batchExport} \title{Export R object to be available on the slaves.} \usage{ batchExport(reg, ..., li = list(), overwrite = FALSE) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{...}{[any]\cr Objects to export. You must provide a valid name.} \item{li}{[\code{list}]\cr More objects to export provided as a named list.} \item{overwrite}{[\code{logical(1)}]\cr If set to \code{FALSE} (default), exported objects are protected from being overwritten by multiple calls of this function. Setting this to \code{TRUE} disables this check.} } \value{ [\code{character}]. Invisibly returns a character vector of exported objects. } \description{ Saves objects as \code{RData} files in the \dQuote{exports} subdirectory of your \code{file.dir} to be later loaded on the slaves. } \seealso{ Other exports: \code{\link{addRegistryPackages}}; \code{\link{addRegistrySourceDirs}}; \code{\link{addRegistrySourceFiles}}; \code{\link{batchUnexport}}; \code{\link{loadExports}}; \code{\link{removeRegistryPackages}}; \code{\link{removeRegistrySourceDirs}}; \code{\link{removeRegistrySourceFiles}} } BatchJobs/man/makeClusterFunctionsSLURM.Rd0000644000176000001440000000351112465167456020216 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsSLURM.R \name{makeClusterFunctionsSLURM} \alias{makeClusterFunctionsSLURM} \title{Create cluster functions for SLURM-based systems.} \usage{ makeClusterFunctionsSLURM(template.file, list.jobs.cmd = c("squeue", "-h", "-o \%i", "-u $USER")) } \arguments{ \item{template.file}{[\code{character(1)}]\cr Path to a brew template file that is used for the PBS job file.} \item{list.jobs.cmd}{[\code{character}]\cr Change default system command / options to list jobs. The first entry is the command, the following the options. See \code{\link[BBmisc]{system3}}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Job files are created based on the brew template \code{template.file}. This file is processed with brew and then submitted to the queue using the \code{sbatch} command. Jobs are killed using the \code{scancel} command and the list of running jobs is retrieved using \code{squeue}. The user must have the appropriate privileges to submit, delete and list jobs on the cluster (this is usually the case). The template file can access all arguments passed to the \code{submitJob} function, see here \code{\link{ClusterFunctions}}. It is the template file's job to choose a queue for the job and handle the desired resource allocations. Examples can be found on \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfSLURM}. } \seealso{ Other clusterFunctions: \code{\link{ClusterFunctions}}, \code{\link{makeClusterFunctions}}; \code{\link{makeClusterFunctionsInteractive}}; \code{\link{makeClusterFunctionsLSF}}; \code{\link{makeClusterFunctionsLocal}}; \code{\link{makeClusterFunctionsMulticore}}; \code{\link{makeClusterFunctionsSGE}}; \code{\link{makeClusterFunctionsSSH}}; \code{\link{makeClusterFunctionsTorque}} } BatchJobs/man/batchMapResults.Rd0000644000176000001440000000407412465167456016331 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/batchMapResults.R \name{batchMapResults} \alias{batchMapResults} \title{Maps a function over the results of a registry by using batchMap.} \usage{ batchMapResults(reg, reg2, fun, ..., ids, part = NA_character_, more.args = list()) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry whose results should be mapped by \code{fun}.} \item{reg2}{[\code{\link{Registry}}]\cr Empty registry that should store the job for the mapping.} \item{fun}{[\code{function(job, res, ...)}]\cr Function to map over results of \code{reg}. Further arguments come from ... of \code{batchMapResults} and \code{more.args}.} \item{...}{[any]\cr Furher arguments to vectorize over (list or vector). Must all be the same length as number of results in \code{reg}.} \item{ids}{[\code{integer}]\cr Ids of jobs whose results should be mapped with \code{fun}. Default is all jobs.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} } \value{ Vector of type \code{integer} with job ids. } \description{ Maps a function over the results of a registry by using batchMap. } \examples{ reg1 = makeRegistry(id = "BatchJobsExample1", file.dir = tempfile(), seed = 123) # square some numbers f = function(x) x^2 batchMap(reg1, f, 1:10) # submit jobs and wait for the jobs to finish submitJobs(reg1) waitForJobs(reg1) # look at results reduceResults(reg1, fun = function(aggr,job,res) c(aggr, res)) reg2 = makeRegistry(id = "BatchJobsExample2", file.dir = tempfile(), seed = 123) # define function to tranform results, we simply do the inverse of the squaring g = function(job, res) sqrt(res) batchMapResults(reg1, reg2, fun = g) # submit jobs and wait for the jobs to finish submitJobs(reg2) waitForJobs(reg2) # check results reduceResults(reg2, fun = function(aggr,job,res) c(aggr, res)) } BatchJobs/man/cfHandleUnknownSubmitError.Rd0000644000176000001440000000200312465167456020500 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsHelpers.R \name{cfHandleUnknownSubmitError} \alias{cfHandleUnknownSubmitError} \title{Cluster functions helper: Handle an unknown error during job submission.} \usage{ cfHandleUnknownSubmitError(cmd, exit.code, output) } \arguments{ \item{cmd}{[\code{character(1)}]\cr OS command used to submit the job, e.g. qsub.} \item{exit.code}{[\code{integer(1)}]\cr Exit code of the OS command, should not be 0.} \item{output}{[\code{character}]\cr Output of the OS command, hopefully an informative error message. If these are mutiple lines in a vector, they are automatically pasted together.} } \value{ [\code{\link{SubmitJobResult}}]. } \description{ This function is only intended for use in your own cluster functions implementation. Simply constructs a \code{\link{SubmitJobResult}} object with status code 101, NA as batch job id and an informative error message containing the output of the OS command in \code{output}. } BatchJobs/man/loadResult.Rd0000644000176000001440000000147612465167456015351 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/loadResult.R \name{loadResult} \alias{loadResult} \title{Loads a specific result file.} \usage{ loadResult(reg, id, part = NA_character_, missing.ok = FALSE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{id}{[\code{integer(1)}]\cr Id of job.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{missing.ok}{[\code{logical(1)}]\cr If \code{FALSE} an error is thrown if no result file is found. Otherwise \code{NULL} is returned. Default is \code{FALSE}.} } \value{ [any]. Result of job. } \description{ Loads a specific result file. } \seealso{ \code{\link{reduceResults}} } BatchJobs/man/batchMap.Rd0000644000176000001440000000204712465167456014745 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/batchMap.R \name{batchMap} \alias{batchMap} \title{Maps a function over lists or vectors, adding jobs to a registry.} \usage{ batchMap(reg, fun, ..., more.args = list(), use.names = FALSE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Empty Registry that will store jobs for the mapping.} \item{fun}{[\code{function}]\cr Function to map over \code{...}.} \item{...}{[any]\cr Arguments to vectorize over (list or vector).} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} \item{use.names}{[\code{logical(1)}]\cr Store parameter names to enable named results in \code{\link{loadResults}} and some other functions. Default is \code{FALSE}.} } \value{ Vector of type \code{integer} with job ids. } \description{ You can then submit these jobs to the batch system. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:10) print(reg) } BatchJobs/man/showClusterStatus.Rd0000644000176000001440000000137412465167456016756 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/showClusterStatus.R \name{showClusterStatus} \alias{showClusterStatus} \title{Show information about available computational resources on cluster.} \usage{ showClusterStatus(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry. Must not be passed and this is the default.} } \value{ [\code{data.frame}]. Returns the displayed table invisibly. } \description{ Currently only supported for multicore and SSH mode. Displays: Name of node, current load, number of running R processes, number of R processes with more than 50% load, number of BatchJobs jobs running. The latter counts either jobs belonging to \code{reg} or all BatchJobs jobs if reg was not passed. } BatchJobs/man/sanitizePath.Rd0000644000176000001440000000130612465167456015666 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/sanitizePath.R \name{sanitizePath} \alias{sanitizePath} \title{Sanitize a path} \usage{ sanitizePath(path, make.absolute = TRUE, normalize.absolute = FALSE) } \arguments{ \item{path}{[\code{character}]\cr Vector of paths to sanitize.} \item{make.absolute}{[\code{logical}]\cr If \code{TRUE} convert to an absolute path.} \item{normalize.absolute}{[\code{logical}]\cr Also call \code{\link[base]{normalizePath}} on absolute paths? This will immediately resolve symlinks.} } \value{ \code{character} with sanitized paths. } \description{ Replaces backward slashes with forward slashes and optionally normalizes the path. } BatchJobs/man/setConfig.Rd0000644000176000001440000000135212465167456015145 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/conf.R \name{setConfig} \alias{setConfig} \title{Set and overwrite configuration settings} \usage{ setConfig(conf = list(), ...) } \arguments{ \item{conf}{[\code{Config} or \code{list}]\cr List of configuration parameters as returned by \code{\link{loadConfig}} or \code{\link{getConfig}}.} \item{...}{[\code{ANY}]\cr Named configuration parameters. Overwrites parameters in \code{conf}, if provided.} } \value{ Invisibly returns a list of configuration settings. } \description{ Set and overwrite configuration settings } \seealso{ Other conf: \code{\link{.BatchJobs.R}}, \code{\link{configuration}}; \code{\link{getConfig}}; \code{\link{loadConfig}} } BatchJobs/man/waitForJobs.Rd0000644000176000001440000000333312465167456015456 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/waitForJobs.R \name{waitForJobs} \alias{waitForJobs} \title{Wait for termination of jobs on the batch system.} \usage{ waitForJobs(reg, ids, sleep = 10, timeout = 604800, stop.on.error = FALSE, progressbar = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Vector of job ids. Default is all submitted jobs not yet terminated.} \item{sleep}{[\code{numeric(1)}]\cr Seconds to sleep between status updates. Default is \code{10}.} \item{timeout}{[\code{numeric(1)}]\cr After waiting \code{timeout} seconds, show a message and return \code{FALSE}. This argument may be required on some systems where, e.g., expired jobs or jobs on hold are problematic to detect. If you don't want a timeout, set this to \code{Inf}. Default is \code{604800} (one week).} \item{stop.on.error}{[\code{logical(1)}]\cr Immediately return if a job terminates with an error? Default is \code{FALSE}.} \item{progressbar}{[\code{logical(1)}]\cr Set to \code{FALSE} to disable the progress bar. To disable all progress bars, see \code{\link[BBmisc]{makeProgressBar}}.} } \value{ [\code{logical(1)}]. Returns \code{TRUE} if all jobs terminated successfully and \code{FALSE} if either an error occurred or the timeout is reached. } \description{ Waits for termination of jobs while displaying a progress bar containing summarizing informations of the jobs. The following abbreviations are used in the progress bar: \dQuote{S} for number of jobs on system, \dQuote{D} for number of jobs successfully terminated, \dQuote{E} for number ofjobs terminated with an R exception and \dQuote{R} for number of jobs currently running on the system. } BatchJobs/man/addRegistrySourceFiles.Rd0000644000176000001440000000171112465167456017650 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{addRegistrySourceFiles} \alias{addRegistrySourceFiles} \title{Add source files to registry.} \usage{ addRegistrySourceFiles(reg, src.files, src.now = TRUE) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{src.files}{[\code{character}]\cr Paths to add to registry. See \code{\link{makeRegistry}}.} \item{src.now}{[\code{logical(1)}] Source files now on master? Default is \code{TRUE}.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{src.files} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}; \code{\link{addRegistrySourceDirs}}; \code{\link{batchExport}}; \code{\link{batchUnexport}}; \code{\link{loadExports}}; \code{\link{removeRegistryPackages}}; \code{\link{removeRegistrySourceDirs}}; \code{\link{removeRegistrySourceFiles}} } BatchJobs/man/copyRequiredJobFiles.Rd0000644000176000001440000000075512465167456017323 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/testJob.R \name{copyRequiredJobFiles} \alias{copyRequiredJobFiles} \title{ONLY FOR INTERNAL USAGE.} \usage{ copyRequiredJobFiles(reg1, reg2, id) } \arguments{ \item{reg1}{[\code{\link{Registry}}]\cr Source registry.} \item{reg2}{[\code{\link{Registry}}]\cr Detination registry.} \item{id}{[\code{character(1)}]\cr Job id.} } \value{ Nothing. } \description{ ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/makeSubmitJobResult.Rd0000644000176000001440000000227712465167456017166 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctions.R \name{makeSubmitJobResult} \alias{SubmitJobResult} \alias{makeSubmitJobResult} \title{Create a SubmitJobResult object.} \usage{ makeSubmitJobResult(status, batch.job.id, msg, ...) } \arguments{ \item{status}{[\code{integer(1)}]\cr Launch status of job. 0 means success, codes bewteen 1 and 100 are temporary errors and any error greater than 100 is a permanent failure.} \item{batch.job.id}{[\code{character(1)}]\cr Unique id of this job on batch system. Note that this is not the usual job id used in BatchJobs! Must be globally unique so that the job can be terminated using just this information.} \item{msg}{[\code{character(1)}]\cr Optional error message in case \code{status} is not equal to 0. Default is \dQuote{OK}, \dQuote{TEMPERR}, \dQuote{ERROR}, depending on \code{status}.} \item{...}{[\code{any}]\cr Currently unused.} } \value{ [\code{\link{SubmitJobResult}}]. A list, containing \code{status}, \code{batch.job.id} and \code{msg}. } \description{ Use this function in your implementation of \code{\link{makeClusterFunctions}} to create a return value for the \code{submitJob} function. } BatchJobs/man/showStatus.Rd0000644000176000001440000000252612471624612015401 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/showStatus.R \name{showStatus} \alias{getStatus} \alias{showStatus} \title{Retrieve or show status information about jobs.} \usage{ showStatus(reg, ids, run.and.exp = TRUE, errors = 10L) getStatus(reg, ids, run.and.exp = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of selected jobs. Default is all jobs.} \item{run.and.exp}{[\code{logical(1)}]\cr Show running and expired jobs? Requires to list the job on the batch system. If not possible, because that cluster function is not avilable, this option is ignored anyway. Default is \code{TRUE}.} \item{errors}{[\code{integer(1)}]\cr How many of the error messages should be displayed if any errors occured in the jobs? Default is 10.} } \value{ [\code{list}]. List of absolute job numbers. \code{showStatus} returns them invisibly. } \description{ E.g.: How many there are, how many are done, any errors, etc. \code{showStatus} displays on the console, \code{getStatus} returns an informative result without console output. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:10) submitJobs(reg) waitForJobs(reg) # should show 10 submitted jobs, which are all done. showStatus(reg) } BatchJobs/man/batchMapQuick.Rd0000644000176000001440000000404112465167456015736 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/batchMapQuick.R \name{batchMapQuick} \alias{batchMapQuick} \title{Combination of makeRegistry, batchMap and submitJobs.} \usage{ batchMapQuick(fun, ..., more.args = list(), file.dir = NULL, packages = character(0L), chunk.size, n.chunks, chunks.as.arrayjobs = FALSE, inds, resources = list()) } \arguments{ \item{fun}{[\code{function}]\cr Function to map over \code{...}.} \item{...}{[any]\cr Arguments to vectorize over (list or vector).} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} \item{file.dir}{[\code{character}]\cr See \code{\link{makeRegistry}}. Default is \code{NULL}, which means that it is created in the current directory under the name \dQuote{bmq_[random alphanumerics]}.} \item{packages}{[\code{character}]\cr See \code{\link{makeRegistry}}.} \item{chunk.size}{[\code{integer(1)}]\cr Preferred number of jobs in each chunk. Can not be used in combination with \code{n.chunks}. Note that the ids will get shuffled to balance out possible run time differences. Default is not to use chunking.} \item{n.chunks}{[\code{integer(1)}]\cr Preferred number chunks. Can not be used in combination with \code{chunk.size}. Note that the ids will get shuffled to balance out possible run time differences. Default is not to use chunking.} \item{chunks.as.arrayjobs}{[\code{logical(1)}]\cr Submit chunks as array jobs? Default is \code{FALSE}.} \item{inds}{[\code{integer}]\cr Indices of ids / chunks to submit. Default is all. If ids get chunked, this subsets the list of shuffled ids.} \item{resources}{[\code{list}]\cr Required resources for all batch jobs. Default is empty list.} } \value{ [\code{\link{Registry}}] } \description{ Combination of \code{\link{makeRegistry}}, \code{\link{batchMap}} and \code{\link{submitJobs}} for quick computations on the cluster. Should only be used by skilled users who know what they are doing. Creates the file.dir, maps function, potentially chunks jobs and submits them. } BatchJobs/man/loadRegistry.Rd0000644000176000001440000000134312471624612015661 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Registry.R \name{loadRegistry} \alias{loadRegistry} \title{Load a previously saved registry.} \usage{ loadRegistry(file.dir, work.dir) } \arguments{ \item{file.dir}{[\code{character(1)}]\cr Location of the file.dir to load the registry from.} \item{work.dir}{[\code{character(1)}]\cr Location of the work. Unchanged if missing.} } \value{ [\code{\link{Registry}}]. } \description{ Load a previously saved registry. } \details{ Loads a previously created registry from the file system. The \code{file.dir} is automatically updated upon load, so be careful if you use the registry on multiple machines simultaneously, e.g. via sshfs or a samba share. } BatchJobs/man/reduceResults.Rd0000644000176000001440000001173612465167456016064 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/reduceResults.R \name{reduceResults} \alias{reduceResults} \alias{reduceResultsDataFrame} \alias{reduceResultsList} \alias{reduceResultsMatrix} \alias{reduceResultsVector} \title{Reduce results from result directory.} \usage{ reduceResults(reg, ids, part = NA_character_, fun, init, impute.val, progressbar = TRUE, ...) reduceResultsList(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val, progressbar = TRUE) reduceResultsVector(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val) reduceResultsMatrix(reg, ids, part = NA_character_, fun, ..., rows = TRUE, use.names = "ids", impute.val) reduceResultsDataFrame(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val, strings.as.factors = default.stringsAsFactors()) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of selected jobs. Default is all jobs for which results are available.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{fun}{[\code{function}]\cr For \code{reduceResults}, a function \code{function(aggr, job, res, ...)} to reduce things, for all others, a function \code{function(job, res, ...)} to select stuff. Here, \code{job} is the current job descriptor (see \code{\link{Job}}), \code{result} is the current result object and \code{aggr} are the so far aggregated results. When using \code{reduceResults}, your function should add the stuff you want to have from \code{job} and \code{result} to \code{aggr} and return that. When using the other reductions, you should select the stuff you want to have from \code{job} and \code{result} and return something that can be coerced to an element of the selected return data structure (reasonable conversion is tried internally). Default behavior for this argument is to return \code{res}, except for \code{reduceResults} where no default is available.} \item{init}{[\code{ANY}]\cr Initial element, as used in \code{\link{Reduce}}. Default is first result.} \item{impute.val}{[any]\cr For \code{reduceResults}: If not missing, the value of \code{impute.val} is passed to function \code{fun} as argument \code{res} for jobs with missing results.\cr For the specialized reduction functions \code{reduceResults[Type]}: If not missing, \code{impute.val} is used as a replacement for the return value of \code{fun} on missing results.} \item{progressbar}{[\code{logical(1)}]\cr Set to \code{FALSE} to disable the progress bar. To disable all progress bars, see \code{\link[BBmisc]{makeProgressBar}}.} \item{...}{[any]\cr Additional arguments to \code{fun}.} \item{use.names}{[\code{character(1)}]\cr Name the results with job ids (\dQuote{ids}), stored job names (\dQuote{names}) or return a unnamed result (\dQuote{none}). Default is \code{ids}.} \item{rows}{[\code{logical(1)}]\cr Should the selected vectors be used as rows (or columns) in the result matrix? Default is \code{TRUE}.} \item{strings.as.factors}{[\code{logical(1)}] Should all character columns in result be converted to factors? Default is \code{default.stringsAsFactors()}.} } \value{ Aggregated results, return type depends on function. If \code{ids} is empty: \code{reduceResults} returns \code{init} (if available) or \code{NULL}, \code{reduceResultsVector} returns \code{c()}, \code{reduceResultsList} returns \code{list()}, \code{reduceResultsMatrix} returns \code{matrix(0,0,0)}, \code{reduceResultsDataFrame} returns \code{data.frame()}. } \description{ The following functions provide ways to reduce result files into either specific R objects (like vectors, lists, matrices or data.frames) or to arbitrarily aggregate them, which is a more general operation. } \examples{ # generate results: reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:5) submitJobs(reg) waitForJobs(reg) # reduce results to a vector reduceResultsVector(reg) # reduce results to sum reduceResults(reg, fun = function(aggr, job, res) aggr+res) reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) list(a = x, b = as.character(2*x), c = x^2) batchMap(reg, f, 1:5) submitJobs(reg) waitForJobs(reg) # reduce results to a vector reduceResultsVector(reg, fun = function(job, res) res$a) reduceResultsVector(reg, fun = function(job, res) res$b) # reduce results to a list reduceResultsList(reg) # reduce results to a matrix reduceResultsMatrix(reg, fun = function(job, res) res[c(1,3)]) reduceResultsMatrix(reg, fun = function(job, res) c(foo = res$a, bar = res$c), rows = TRUE) reduceResultsMatrix(reg, fun = function(job, res) c(foo = res$a, bar = res$c), rows = FALSE) # reduce results to a data.frame print(str(reduceResultsDataFrame(reg))) # reduce results to a sum reduceResults(reg, fun = function(aggr, job, res) aggr+res$a, init = 0) } BatchJobs/man/batchReduceResults.Rd0000644000176000001440000000406112465167456017017 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/batchReduceResults.R \name{batchReduceResults} \alias{batchReduceResults} \title{Reduces results via a binary function and adds jobs for this to a registry.} \usage{ batchReduceResults(reg, reg2, fun, ids, part = NA_character_, init, block.size, more.args = list()) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry whose results should be reduced by \code{fun}.} \item{reg2}{[\code{\link{Registry}}]\cr Empty registry that should store the job for the mapping.} \item{fun}{[\code{function(aggr, job, res, ...)}]\cr Function to reduce results with.} \item{ids}{[\code{integer}]\cr Ids of jobs whose results should be reduced with \code{fun}. Default is all jobs.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{init}{[any]\cr Initial object for reducing.} \item{block.size}{[\code{integer(1)}]\cr Number of results reduced in one job.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} } \value{ Vector of type \code{integer} with job ids. } \description{ Each jobs reduces a certain number of results on one slave. You can then submit these jobs to the batch system. Later, you can do a final reduction with \code{\link{reduceResults}} on the master. } \examples{ # generating example results: reg1 = makeRegistry(id = "BatchJobsExample1", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg1, f, 1:20) submitJobs(reg1) waitForJobs(reg1) # define function to reduce on slave, we want to sum the squares myreduce = function(aggr, job, res) aggr + res # sum 5 results on each slave process, i.e. 4 jobs reg2 = makeRegistry(id = "BatchJobsExample2", file.dir = tempfile(), seed = 123) batchReduceResults(reg1, reg2, fun = myreduce, init = 0, block.size = 5) submitJobs(reg2) waitForJobs(reg2) # now reduce one final time on master reduceResults(reg2, fun = myreduce) } BatchJobs/man/filterResults.Rd0000644000176000001440000000202412465167456016070 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/filterResults.R \name{filterResults} \alias{filterResults} \title{Find all results where a specific condition is true.} \usage{ filterResults(reg, ids, fun, ...) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs whose results you want to test for the condition. Default is all jobs for which results are available.} \item{fun}{[\code{fun(job, res)}]\cr Predicate function that returns \code{TRUE} or \code{FALSE}.} \item{...}{[any]\cr Additional arguments to \code{fun}.} } \value{ [\code{integer}]. Ids of jobs where \code{fun(job, result)} returns \code{TRUE}. } \description{ Find all results where a specific condition is true. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:10) submitJobs(reg) waitForJobs(reg) # which square numbers are even: filterResults(reg, fun = function(job, res) res \%\% 2 == 0) } BatchJobs/man/loadConfig.Rd0000644000176000001440000000120112465167456015262 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/conf.R \name{loadConfig} \alias{loadConfig} \title{Load a specific configuration file.} \usage{ loadConfig(conffile = ".BatchJobs.R") } \arguments{ \item{conffile}{[\code{character(1)}]\cr Location of the configuration file to load. Default is \dQuote{.BatchJobs.conf} in the current working directory.} } \value{ Invisibly returns a list of configuration settings. } \description{ Load a specific configuration file. } \seealso{ Other conf: \code{\link{.BatchJobs.R}}, \code{\link{configuration}}; \code{\link{getConfig}}; \code{\link{setConfig}} } BatchJobs/man/getJobInfo.Rd0000644000176000001440000000362012465167456015252 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/getJobInfo.R \name{getJobInfo} \alias{getJobInfo} \title{Get computational information of jobs.} \usage{ getJobInfo(reg, ids, pars = FALSE, prefix.pars = FALSE, select, unit = "seconds") } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} \item{pars}{[\code{logical(1)}]\cr Include job parameters in the output? Default is \code{FALSE}.} \item{prefix.pars}{[\code{logical(1)}]\cr Should a prefix be added to job parameter names (column names) to avoid name clashes? Default is \code{FALSE}.} \item{select}{[\code{character}]\cr Select only a subset of columns. Usually this is not required and you can subset yourself, but in some rare cases it may be advantageous to not query all information. Note that the column \dQuote{id} (job id) is always selected. If not provided, all columns are queried and returned.} \item{unit}{[\code{character(1)}]\cr Unit to convert execution and queing times to. Possible values: \dQuote{seconds}, \dQuote{minutes}, \dQuote{hours}, \dQuote{days} and \dQuote{weeks}. Default is \dQuote{seconds}.} } \value{ [\code{data.frame}]. } \description{ error messages (shortened, see \code{\link{showLog}} for detailed error messages), Returns time stamps (submitted, started, done), time running, approximate memory usage (in Mb, see note) time in queue, hostname of the host the job was executed, assigned batch ID, the R PID and the seed of the job. } \note{ To estimate memory usage the sum of the last column of \code{\link[base]{gc}} is used. } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{debugSSH}}; \code{\link{getErrorMessages}}; \code{\link{getLogFiles}}; \code{\link{grepLogs}}; \code{\link{killJobs}}; \code{\link{resetJobs}}; \code{\link{setJobFunction}}; \code{\link{showLog}}; \code{\link{testJob}} } BatchJobs/man/grepLogs.Rd0000644000176000001440000000253712465167456015014 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/grepLogs.R \name{grepLogs} \alias{grepLogs} \title{Grep log files for a pattern.} \usage{ grepLogs(reg, ids, pattern = "warn", ignore.case = TRUE, verbose = FALSE, range = 2L) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs to grep. Default is all terminated jobs (done + errors).} \item{pattern}{[\code{character(1)}]\cr Pattern to search for. See \code{\link{grep}}. Default is \code{"warn"}.} \item{ignore.case}{[\code{logical(1)}]\cr Ignore case. See \code{\link{grep}}. Default is \code{TRUE}.} \item{verbose}{[\code{logical(1)}]\cr Print matches. Default is \code{FALSE}.} \item{range}{[\code{integer(1)}]\cr If \code{verbose} is set to \code{TRUE}, print \code{range} leading and trailing lines for contextual information about the warning. Default is \code{2}.} } \value{ [\code{integer}]. Ids of jobs where pattern was found in the log file. } \description{ Searches for occurence of \code{pattern} in log files. } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{debugSSH}}; \code{\link{getErrorMessages}}; \code{\link{getJobInfo}}; \code{\link{getLogFiles}}; \code{\link{killJobs}}; \code{\link{resetJobs}}; \code{\link{setJobFunction}}; \code{\link{showLog}}; \code{\link{testJob}} } BatchJobs/man/makeClusterFunctions.Rd0000644000176000001440000000633612465167456017403 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctions.R \name{makeClusterFunctions} \alias{ClusterFunctions} \alias{makeClusterFunctions} \title{Create a ClusterFuntions object.} \usage{ makeClusterFunctions(name, submitJob, killJob, listJobs, getArrayEnvirName, class = NULL, ...) } \arguments{ \item{name}{[\code{character(1)}]\cr Name of cluster functions.} \item{submitJob}{[\code{function(conf, reg, job.name, rscript, log.file, job.dir, resources, ...)}]\cr Function to submit a new job. Must return a \code{\link{SubmitJobResult}} object.\cr The arguments are:\cr conf [\code{environment}]: The user configuration.\cr reg [\code{\link{Registry}}]: The registry.\cr job.name [\code{character(1)}]: Name of job, used if the job is displayed on the batch system. This is just for display and not an id!\cr rscript [\code{character(1)}]: File path to R script that is used to execute the job.\cr log.file [\code{character(1)}]: File path where log file (.Rout) has to be placed.\cr job.dir [\code{character(1)}]: Directory where all files relating to this job are placed.\cr resources [\code{list}]: Freely definable list of required resources for this job, e.g. walltime or memory.} \item{killJob}{[\code{function(conf, reg, batch.job.id)}]\cr Function to kill a job on the batch system. Make sure that you definately kill the job! Return value is currently ignored.\cr The arguments are:\cr conf [\code{environment}]: The user configuration.\cr reg [\code{\link{Registry}}]: The registry.\cr batch.job.id [\code{character(1)}]: Batch job id, as produced by \code{submitJob}.\cr Set \code{killJob} to \code{NULL} if killing jobs cannot be supported.} \item{listJobs}{[\code{function(conf, reg)}]\cr List all jobs on the batch system for the current user / registry. This includes queued, running, held, idle, etc. jobs. Must return an integer vector of batch job ids, same format as they are produced by \code{submitJob}. It does not matter if you return a few job ids too many (e.g. all for the current user instead of all for the current registry), but you have to include all relevant ones. The arguments are:\cr conf [\code{environment}]: The user configuration.\cr reg [\code{\link{Registry}}]: The registry.\cr Set \code{listJobs} to \code{NULL} if listing jobs cannot be supported.} \item{getArrayEnvirName}{[\code{function()}]\cr Returns the name of the environment variable specifying the array ID. Should return \code{NA} if not supported.} \item{class}{[\code{character(1)}]\cr Optional class name for cluster functions object. Useful to provide a nice print method which might show additional information about the workers. Default is \code{NULL}.} \item{...}{[\code{any}]\cr Currently ignored.} } \description{ Use this funtion when you implement a backend for a batch system. You must define the functions specified in the arguments. } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}; \code{\link{makeClusterFunctionsLSF}}; \code{\link{makeClusterFunctionsLocal}}; \code{\link{makeClusterFunctionsMulticore}}; \code{\link{makeClusterFunctionsSGE}}; \code{\link{makeClusterFunctionsSLURM}}; \code{\link{makeClusterFunctionsSSH}}; \code{\link{makeClusterFunctionsTorque}} } BatchJobs/man/submitJobs.Rd0000644000176000001440000000707612465167456015356 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/submitJobs.R \name{submitJobs} \alias{submitJobs} \title{Submit jobs or chunks of jobs to batch system via cluster function.} \usage{ submitJobs(reg, ids, resources = list(), wait, max.retries = 10L, chunks.as.arrayjobs = FALSE, job.delay = FALSE, progressbar = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Vector for job id or list of vectors of chunked job ids. Only corresponding jobs are submitted. Chunked jobs will get executed sequentially as a single job for the scheduler. Default is all jobs which were not yet submitted to the batch system.} \item{resources}{[\code{list}]\cr Required resources for all batch jobs. The elements of this list (e.g. something like \dQuote{walltime} or \dQuote{nodes} are defined by your template job file. Defaults can be specified in your config file. Default is empty list.} \item{wait}{[\code{function(retries)}]\cr Function that defines how many seconds should be waited in case of a temporary error. Default is exponential back-off with \code{10*2^retries}.} \item{max.retries}{[\code{integer(1)}]\cr Number of times to submit one job again in case of a temporary error (like filled queues). Each time \code{wait} is called to wait a certain number of seconds. Default is 10 times.} \item{chunks.as.arrayjobs}{[\code{logical(1)}]\cr If ids are passed as a list of chunked job ids, execute jobs in a chunk as array jobs. Note that your scheduler and your template must be adjusted to use this option. Default is \code{FALSE}.} \item{job.delay}{[\code{function(n, i)} or \code{logical(1)}]\cr Function that defines how many seconds a job should be delayed before it starts. This is an expert option and only necessary to change when you want submit extremely many jobs. We then delay the jobs a bit to write the submit messages as early as possible to avoid writer starvation. \code{n} is the number of jobs and \code{i} the number of the ith job. The default function used with \code{job.delay} set to \code{TRUE} is no delay for 100 jobs or less and otherwise \code{runif(1, 0.1*n, 0.2*n)}. If set to \code{FALSE} (the default) delaying jobs is disabled.} \item{progressbar}{[\code{logical(1)}]\cr Set to \code{FALSE} to disable the progress bar. To disable all progress bars, see \code{\link[BBmisc]{makeProgressBar}}.} } \value{ [\code{integer}]. Vector of submitted job ids. } \description{ If the internal submit cluster function completes successfully, the \code{retries} counter is set back to 0 and the next job or chunk is submitted. If the internal submit cluster function returns a fatal error, the submit process is completely stopped and an exception is thrown. If the internal submit cluster function returns a temporary error, the submit process waits for a certain time, which is determined by calling the user-defined \code{wait}-function with the current \code{retries} counter, the counter is increased by 1 and the same job is submitted again. If \code{max.retries} is reached the function simply terminates. Potential temporary submit warnings and errors are logged inside your file directory in the file \dQuote{submit.log}. To keep track you can use \code{tail -f [file.dir]/submit.log} in another terminal. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:10) submitJobs(reg) waitForJobs(reg) # Submit the 10 jobs again, now randomized into 2 chunks: chunked = chunk(getJobIds(reg), n.chunks = 2, shuffle = TRUE) submitJobs(reg, chunked) } BatchJobs/man/makeClusterFunctionsInteractive.Rd0000644000176000001440000000267112465167456021577 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsInteractive.R \name{makeClusterFunctionsInteractive} \alias{makeClusterFunctionsInteractive} \title{Create cluster functions for sequential execution in same session.} \usage{ makeClusterFunctionsInteractive(write.logs = TRUE) } \arguments{ \item{write.logs}{[\code{logical(1)}]\cr Sink the output to log files. Turning logging off can increase the speed of calculations but makes it next to impossible to debug. Default is \code{TRUE}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ All jobs executed under these cluster functions are executed sequentially, in the same interactive R process that you currently are. That is, \code{submitJob} does not return until the job has finished. The main use of this \code{ClusterFunctions} implementation is to test and debug programs on a local computer. Listing jobs returns an empty vector (as no jobs can be running when you call this) and \code{killJob} returns at once (for the same reason). } \seealso{ Other clusterFunctions: \code{\link{ClusterFunctions}}, \code{\link{makeClusterFunctions}}; \code{\link{makeClusterFunctionsLSF}}; \code{\link{makeClusterFunctionsLocal}}; \code{\link{makeClusterFunctionsMulticore}}; \code{\link{makeClusterFunctionsSGE}}; \code{\link{makeClusterFunctionsSLURM}}; \code{\link{makeClusterFunctionsSSH}}; \code{\link{makeClusterFunctionsTorque}} } BatchJobs/man/findState.Rd0000644000176000001440000000605312502316642015131 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/findStatus.R \name{findDone} \alias{findDisappeared} \alias{findDone} \alias{findErrors} \alias{findExpired} \alias{findMissingResults} \alias{findNotDone} \alias{findNotErrors} \alias{findNotOnSystem} \alias{findNotRunning} \alias{findNotStarted} \alias{findNotSubmitted} \alias{findNotTerminated} \alias{findOnSystem} \alias{findRunning} \alias{findStarted} \alias{findSubmitted} \alias{findTerminated} \title{Find jobs depending on computional state.} \usage{ findDone(reg, ids, limit = NULL) findNotDone(reg, ids, limit = NULL) findMissingResults(reg, ids, limit = NULL) findErrors(reg, ids, limit = NULL) findNotErrors(reg, ids, limit = NULL) findTerminated(reg, ids, limit = NULL) findNotTerminated(reg, ids, limit = NULL) findSubmitted(reg, ids, limit = NULL) findNotSubmitted(reg, ids, limit = NULL) findOnSystem(reg, ids, limit = NULL) findNotOnSystem(reg, ids, limit = NULL) findRunning(reg, ids, limit = NULL) findNotRunning(reg, ids, limit = NULL) findStarted(reg, ids, limit = NULL) findNotStarted(reg, ids, limit = NULL) findExpired(reg, ids, limit = NULL) findDisappeared(reg, ids, limit = NULL) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Subset of job ids to restrict the result to. Default is all jobs.} \item{limit}{[\code{integer(1)}]\cr Limit the number of returned ids. Default is all ids.} } \value{ [\code{integer}]. Ids of jobs. } \description{ \code{findDone}: Find jobs which succesfully terminated. \code{findNotDone}: Find jobs for which results are still missing. \code{findMissingResults}: Deprecated. Alias for findNotDone. \code{findErrors}: Find jobs where errors occured. \code{findNotErrors}: Find jobs where no errors occured. \code{findTerminated}: Find jobs which have terminated (done / error). \code{findNotTerminated}: Find jobs which have not terminated (not done / no error). \code{findSubmitted}: Find jobs which have been submitted. \code{findNotSubmitted}: Find jobs which have not been submitted. \code{findOnSystem}: Find jobs which are present on the batch system at the moment. \code{findNotOnSystem}: Find jobs which are not present on the batch system at the moment. \code{findRunning}: Find jobs which are running. \code{findNotRunning}: Find jobs which are not running. \code{findStarted}: Find jobs which have been started on the batch system. \code{findStarted}: Find jobs which have not been started on the batch system. \code{findNotRunning}: Find jobs which are not running. \code{findExpired}: Find jobs where walltime was probably hit. Right now the heuristic is as follows: Find all jobs that have started, did not abort with an error, did not complete with a result and are not submitted or running anymore. Note that this heuristic does not include jobs the scheduler looses before starting. \code{findDisappeared}: Find jobs which disappeared from the system. Right now the heuristic is as follows: Find all jobs that are submitted but not started nor on the system anymore. } BatchJobs/man/cfKillBatchJob.Rd0000644000176000001440000000172512465167456016031 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsHelpers.R \name{cfKillBatchJob} \alias{cfKillBatchJob} \title{Cluster functions helper: Kill a batch job via OS command} \usage{ cfKillBatchJob(cmd, batch.job.id, max.tries = 3L) } \arguments{ \item{cmd}{[\code{character(1)}]\cr OS command, e.g. \dQuote{qdel}.} \item{batch.job.id}{[\code{character(1)}]\cr Id of the batch job on the batch system.} \item{max.tries}{[\code{integer(1)}]\cr Number of total times to try execute the OS command in cases of failures. Default is \code{3}.} } \value{ Nothing. } \description{ This function is only intended for use in your own cluster functions implementation. Calls the OS command to kill a job via \code{system} like this: \dQuote{cmd batch.job.id}. If the command returns an exit code > 0, the command is repeated after a 1 second sleep \code{max.tries-1} times. If the command failed in all tries, an exception is generated. } BatchJobs/man/findJobs.Rd0000644000176000001440000000206112465167456014760 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/findJobs.R \name{findJobs} \alias{findJobs} \title{Finds ids of jobs that match a query.} \usage{ findJobs(reg, ids, pars, jobnames) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Subset of job ids to restrict the result to. Default is all jobs.} \item{pars}{[R expression]\cr All jobs whose parameters match the given expression are selected. This implies that you have named the parameters when you passed the vectors. If you forgot to do this you can use \code{.arg1}, \code{.arg2}, etc., to refer to the the unnamed ones.} \item{jobnames}{[\code{character}]\cr Restrict to jobs with stored names. Exact matching is used.} } \value{ [\code{integer}]. Ids for jobs which match the query. } \description{ Finds ids of jobs that match a query. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x, y) x * y batchExpandGrid(reg, f, x = 1:2, y = 1:3) findJobs(reg, pars = (y > 2)) } BatchJobs/man/testJob.Rd0000644000176000001440000000306612465167456014642 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/testJob.R \name{testJob} \alias{testJob} \title{Tests a job by running it with Rscript in a new process.} \usage{ testJob(reg, id, resources = list(), external = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{id}{[\code{integer(1)}]\cr Id of job to test. Default is first job id of registry.} \item{resources}{[\code{list}]\cr Usually not needed, unless you call the function \code{\link{getResources}} in your job. See \code{\link{submitJobs}}. Default is empty list.} \item{external}{[\code{logical(1)}]\cr Run test in an independent external R session instead of current. The former allows to uncover missing variable definitions (which may accidentially be defined in the current global environment) and the latter is useful to get traceable execeptions. Default is \code{TRUE}.} } \value{ [any]. Result of job. If the job did not complete because of an error, NULL is returned. } \description{ Useful for debugging. Note that neither the registry, database or file directory are changed. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) if (x==1) stop("oops") else x batchMap(reg, f, 1:2) testJob(reg, 2) } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{debugSSH}}; \code{\link{getErrorMessages}}; \code{\link{getJobInfo}}; \code{\link{getLogFiles}}; \code{\link{grepLogs}}; \code{\link{killJobs}}; \code{\link{resetJobs}}; \code{\link{setJobFunction}}; \code{\link{showLog}} } BatchJobs/man/showLog.Rd0000644000176000001440000000303512465167456014646 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/showLog.R \name{showLog} \alias{showLog} \title{Display the contents of a log file.} \usage{ showLog(reg, id, pager = getOption("pager")) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{id}{[\code{integer(1)}]\cr Id of selected job. Default is first id in registry.} \item{pager}{[\code{any}]\cr Pager to use to display the log. Defaults to \code{getOption("pager")}. This option is passed to \code{file.show} and is highly OS dependant and GUI dependant. If either R's pager or the environment variable \dQuote{PAGER} is set to \dQuote{less} or \dQuote{vim}, the correct part of the log file will be shown. Otherwise you find information about the correct part in the beginning of the displayed file.} } \value{ [\code{character(1)}]. Invisibly returns path to log file. } \description{ Display the contents of a log file, useful in case of errors. Note this rare special case: When you use chunking, submit some jobs, some jobs fail, then you resubmit these jobs again in different chunks, the log files will contain the log of the old, failed job as well. \code{showLog} tries to jump to the correct part of the new log file with a supported pager. } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{debugSSH}}; \code{\link{getErrorMessages}}; \code{\link{getJobInfo}}; \code{\link{getLogFiles}}; \code{\link{grepLogs}}; \code{\link{killJobs}}; \code{\link{resetJobs}}; \code{\link{setJobFunction}}; \code{\link{testJob}} } BatchJobs/man/debugSSH.Rd0000644000176000001440000000315312465167456014671 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/debugSSH.R \name{debugSSH} \alias{debugSSH} \title{Helper function to debug SSH mode.} \usage{ debugSSH(nodename, rhome = "", r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), dir = getwd()) } \arguments{ \item{nodename}{[\code{character(1)}]\cr Node on which worker should be constructed for the test.} \item{rhome}{[\code{character(1)}]\cr Path to R installation on the worker. \dQuote{} means R installation on the PATH is used, of course this implies that it must be on the PATH (also for non-interactive shells)! Default is \dQuote{}.} \item{r.options}{[\code{list}] Options for R and Rscript, one option per element of the vector, a la \dQuote{--vanilla}. Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}.} \item{dir}{[\code{character(1)}]\cr Path where internally used test registries can be created. Note that this must be shared for the worker. Default is current working directory.} } \value{ Nothing. } \description{ Useful in case of configuration problems. Tries different operations of increasing difficulty and provides debug output on the console. Note that this function does not access nor use information specified for your cluster functions in your configuration. } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{getErrorMessages}}; \code{\link{getJobInfo}}; \code{\link{getLogFiles}}; \code{\link{grepLogs}}; \code{\link{killJobs}}; \code{\link{resetJobs}}; \code{\link{setJobFunction}}; \code{\link{showLog}}; \code{\link{testJob}} } BatchJobs/man/removeRegistrySourceDirs.Rd0000644000176000001440000000150712465167456020257 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{removeRegistrySourceDirs} \alias{removeRegistrySourceDirs} \title{Remove packages from registry.} \usage{ removeRegistrySourceDirs(reg, src.dirs) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{src.dirs}{[\code{character}]\cr Paths to remove from registry.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{src.dirs} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}; \code{\link{addRegistrySourceDirs}}; \code{\link{addRegistrySourceFiles}}; \code{\link{batchExport}}; \code{\link{batchUnexport}}; \code{\link{loadExports}}; \code{\link{removeRegistryPackages}}; \code{\link{removeRegistrySourceFiles}} } BatchJobs/man/getJobIds.Rd0000644000176000001440000000051112465167456015072 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Job.R \name{getJobIds} \alias{getJobIds} \title{Get ids of jobs in registry.} \usage{ getJobIds(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} } \value{ [\code{character}]. } \description{ Get ids of jobs in registry. } BatchJobs/man/getResources.Rd0000644000176000001440000000064212465167456015677 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Resources.R \name{getResources} \alias{getResources} \title{Function to get job resources in job function.} \usage{ getResources() } \value{ [\code{list}]. } \description{ Return the list passed to \code{\link{submitJobs}}, e.g. nodes, walltime, etc. } \details{ Can only be called in job function during job execution on slave. } BatchJobs/man/batchReduce.Rd0000644000176000001440000000252112465167456015434 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/batchReduce.R \name{batchReduce} \alias{batchReduce} \title{Reduces via a binary function over a list adding jobs to a registry.} \usage{ batchReduce(reg, fun, xs, init, block.size, more.args = list()) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Empty Registry.} \item{fun}{[\code{function(aggr, x, ...)}]\cr Function to reduce \code{xs} with.} \item{xs}{[\code{vector}]\cr Vector to reduce.} \item{init}{[any]\cr Initial object for reducing.} \item{block.size}{[\code{integer(1)}]\cr Number of elements of \code{xs} reduced in one job.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} } \value{ Vector of type \code{integer} with job ids. } \description{ Each jobs reduces a certain number of elements on one slave. You can then submit these jobs to the batch system. } \examples{ # define function to reduce on slave, we want to sum a vector f = function(aggr, x) aggr + x reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) # sum 20 numbers on each slave process, i.e. 5 jobs batchReduce(reg, fun = f, 1:100, init = 0, block.size = 5) submitJobs(reg) waitForJobs(reg) # now reduce one final time on master reduceResults(reg, fun = function(aggr,job,res) f(aggr, res)) } BatchJobs/man/makeClusterFunctionsTorque.Rd0000644000176000001440000000351012465167456020572 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsTorque.R \name{makeClusterFunctionsTorque} \alias{makeClusterFunctionsTorque} \title{Create cluster functions for torque-based systems.} \usage{ makeClusterFunctionsTorque(template.file, list.jobs.cmd = c("qselect", "-u $USER", "-s EHQRTW")) } \arguments{ \item{template.file}{[\code{character(1)}]\cr Path to a brew template file that is used for the PBS job file.} \item{list.jobs.cmd}{[\code{character}]\cr Change default system command / options to list jobs. The first entry is the command, the following the options. See \code{\link[BBmisc]{system3}}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Job files are created based on the brew template \code{template.file}. This file is processed with brew and then submitted to the queue using the \code{qsub} command. Jobs are killed using the \code{qdel} command and the list of running jobs is retrieved using \code{qselect}. The user must have the appropriate privileges to submit, delete and list jobs on the cluster (this is usually the case). The template file can access all arguments passed to the \code{submitJob} function, see here \code{\link{ClusterFunctions}}. It is the template file's job to choose a queue for the job and handle the desired resource allocations. Examples can be found on \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfTorque}. } \seealso{ Other clusterFunctions: \code{\link{ClusterFunctions}}, \code{\link{makeClusterFunctions}}; \code{\link{makeClusterFunctionsInteractive}}; \code{\link{makeClusterFunctionsLSF}}; \code{\link{makeClusterFunctionsLocal}}; \code{\link{makeClusterFunctionsMulticore}}; \code{\link{makeClusterFunctionsSGE}}; \code{\link{makeClusterFunctionsSLURM}}; \code{\link{makeClusterFunctionsSSH}} } BatchJobs/man/addRegistryPackages.Rd0000644000176000001440000000145312465167456017146 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Packages.R \name{addRegistryPackages} \alias{addRegistryPackages} \title{Add packages to registry.} \usage{ addRegistryPackages(reg, packages) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{packages}{[\code{character}]\cr Packages to add to registry.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{packages} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistrySourceDirs}}; \code{\link{addRegistrySourceFiles}}; \code{\link{batchExport}}; \code{\link{batchUnexport}}; \code{\link{loadExports}}; \code{\link{removeRegistryPackages}}; \code{\link{removeRegistrySourceDirs}}; \code{\link{removeRegistrySourceFiles}} } BatchJobs/man/getLogFiles.Rd0000644000176000001440000000135112465167456015427 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/getLogFiles.R \name{getLogFiles} \alias{getLogFiles} \title{Get log file paths for jobs.} \usage{ getLogFiles(reg, ids) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} } \value{ [\code{character}]. Vector of file paths to log files. } \description{ Get log file paths for jobs. } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{debugSSH}}; \code{\link{getErrorMessages}}; \code{\link{getJobInfo}}; \code{\link{grepLogs}}; \code{\link{killJobs}}; \code{\link{resetJobs}}; \code{\link{setJobFunction}}; \code{\link{showLog}}; \code{\link{testJob}} } BatchJobs/man/debugMulticore.Rd0000644000176000001440000000161512465167456016200 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/debugMulticore.R \name{debugMulticore} \alias{debugMulticore} \title{Helper function to debug multicore mode.} \usage{ debugMulticore(r.options) } \arguments{ \item{r.options}{[\code{list}] Options for R and Rscript, one option per element of the vector, a la \dQuote{--vanilla}. Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}.} } \value{ Nothing. } \description{ Useful in case of severe errors. Tries different operations of increasing difficulty and provides debug output on the console } \seealso{ Other debug: \code{\link{debugSSH}}; \code{\link{getErrorMessages}}; \code{\link{getJobInfo}}; \code{\link{getLogFiles}}; \code{\link{grepLogs}}; \code{\link{killJobs}}; \code{\link{resetJobs}}; \code{\link{setJobFunction}}; \code{\link{showLog}}; \code{\link{testJob}} } BatchJobs/man/removeRegistryPackages.Rd0000644000176000001440000000147312465167456017715 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Packages.R \name{removeRegistryPackages} \alias{removeRegistryPackages} \title{Remove packages from registry.} \usage{ removeRegistryPackages(reg, packages) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{packages}{[\code{character}]\cr Packages to remove from registry.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{packages} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}; \code{\link{addRegistrySourceDirs}}; \code{\link{addRegistrySourceFiles}}; \code{\link{batchExport}}; \code{\link{batchUnexport}}; \code{\link{loadExports}}; \code{\link{removeRegistrySourceDirs}}; \code{\link{removeRegistrySourceFiles}} } BatchJobs/man/getJobParamDf.Rd0000644000176000001440000000104112465167456015664 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/getJobParamDf.R \name{getJobParamDf} \alias{getJobParamDf} \title{Returns parameters for all jobs as the rows of a data.frame.} \usage{ getJobParamDf(reg, ids) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} } \value{ [\code{data.frame}]. Rows are named with job ids. } \description{ Returns parameters for all jobs as the rows of a data.frame. } \examples{ # see batchExpandGrid } BatchJobs/man/getJobResources.Rd0000644000176000001440000000130712465167456016331 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Resources.R \name{getJobResources} \alias{getJobResources} \title{Function to get the resources that were submitted for some jobs.} \usage{ getJobResources(reg, ids, as.list = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all submitted jobs.} \item{as.list}{[\code{integer(1)}]\cr If \code{FALSE} a data.frame will be returned. Default is \code{TRUE}.} } \value{ [\code{list} | \code{data.frame}]. List (or data.frame) of resource lists as passed to \code{\link{submitJobs}}. } \description{ Throws an error if call it for unsubmitted jobs. } BatchJobs/man/batchUnexport.Rd0000644000176000001440000000157312465167456016057 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Exports.R \name{batchUnexport} \alias{batchUnexport} \title{Unload exported R objects.} \usage{ batchUnexport(reg, what) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{what}{[\code{character}]\cr Names of objects to remove.} } \value{ [\code{character}]. Invisibly returns a character vector of unexported objects. } \description{ Removes \code{RData} files from the \dQuote{exports} subdirectory of your \code{file.dir} and thereby prevents loading on the slave. } \seealso{ Other exports: \code{\link{addRegistryPackages}}; \code{\link{addRegistrySourceDirs}}; \code{\link{addRegistrySourceFiles}}; \code{\link{batchExport}}; \code{\link{loadExports}}; \code{\link{removeRegistryPackages}}; \code{\link{removeRegistrySourceDirs}}; \code{\link{removeRegistrySourceFiles}} } BatchJobs/man/getSSHWorkersInfo.Rd0000644000176000001440000000141412465167456016551 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/getSSHWorkersInfo.R \name{getSSHWorkersInfo} \alias{getSSHWorkersInfo} \title{Print and return R installation and other information for SSH workers.} \usage{ getSSHWorkersInfo(nodenames) } \arguments{ \item{nodenames}{[\code{character}]\cr Nodenames of workers.} } \value{ [\code{list}]. Displayed information as a list named by nodenames. } \description{ Workers are queried in parallel via \code{\link{callFunctionOnSSHWorkers}}. The function will display a warning if the first lib path on the worker is not writable as this indicates potential problems in the configuration and \code{\link{installPackagesOnSSHWorkers}} will not work. } \seealso{ \code{\link{callFunctionOnSSHWorkers}} } BatchJobs/man/sweepRegistry.Rd0000644000176000001440000000316412465167456016103 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/sweepRegistry.R \name{sweepRegistry} \alias{sweepRegistry} \title{Sweep obsolete files from the file system.} \usage{ sweepRegistry(reg, sweep = c("scripts", "conf")) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{sweep}{[\code{character}]\cr Possible choices: Temporary R scripts of jobs, really not needed for anything else then execution (\dQuote{scripts}), log file of jobs, think about whether you later want to inspect them (\dQuote{logs}), BatchJobs configuration files which are temporarily stored on submit, really not needed for anything else then execution (\dQuote{conf}), resource lists of \code{\link{submitJobs}} which are temporarily stored on submit, think about whether you later want to inspect them (\dQuote{resources}), Default is \code{c("scripts", "conf")}.} } \value{ [\code{logical}]. Invisibly returns \code{TRUE} on success and \code{FALSE} if some files could not be removed. } \description{ Removes R scripts, log files, resource informations and temporarily stored configuration files from the registry's file directory. Assuming all your jobs completed successfully, none of these are needed for further work. This operation potentially releases quite a lot of disk space, depending on the number of your jobs. BUT A HUGE WORD OF WARNING: IF you later notice something strange and need to determine the reason for it, you are at a huge disadvantage. Only do this at your own risk and when you are sure that you have successfully completed a project and only want to archive your produced experiments and results. } BatchJobs/man/batchExpandGrid.Rd0000644000176000001440000000260112465167456016251 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/batchExpandGrid.R \name{batchExpandGrid} \alias{batchExpandGrid} \title{Map function over all combinations.} \usage{ batchExpandGrid(reg, fun, ..., more.args = list()) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Empty Registry that will store jobs for the mapping.} \item{fun}{[\code{function}]\cr Function to map over the combinations.} \item{...}{[any]\cr Vectors that are used to compute all combinations. If the arguments are named, these names are used to bind to arguments of \code{fun}.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} } \value{ [\code{data.frame}]. Expanded grid of combinations produced by \code{\link{expand.grid}}. } \description{ Maps an n-ary-function over a list of all combinations which are given by some vectors. Internally \code{\link{expand.grid}} is used to compute the combinations, then \code{\link{batchMap}} is called. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x, y, z) x * y + z # lets store the param grid grid = batchExpandGrid(reg, f, x = 1:2, y = 1:3, more.args = list(z = 10)) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg) # later, we can always access the param grid like this grid = getJobParamDf(reg) cbind(grid, y = y) } BatchJobs/man/BatchJobs.Rd0000644000176000001440000000322112465167456015060 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/zzz.R \docType{package} \name{BatchJobs} \alias{BatchJobs} \alias{BatchJobs-package} \title{The BatchJobs package} \description{ The BatchJobs package } \section{Additional information}{ \describe{ \item{Homepage:}{\url{https://github.com/tudo-r/BatchJobs}} \item{Wiki:}{\url{https://github.com/tudo-r/BatchJobs/wiki}} \item{FAQ:}{\url{https://github.com/tudo-r/BatchJobs/wiki/FAQ}} \item{Configuration:}{\url{https://github.com/tudo-r/BatchJobs/wiki/Configuration}} } The package currently support the following further R options, which you can set either in your R profile file or a script via \code{\link{options}}: \describe{ \item{BatchJobs.verbose}{This boolean flag can be set to \code{FALSE} to reduce the console output of the package operations. Usually you want to see this output in interactive work, but when you use the package in e.g. knitr documents, it clutters the resulting document too much.} \item{BatchJobs.check.posix}{If this boolean flag is enabled, the package checks your registry file dir (and related user-defined directories) quite strictly to be POSIX compliant. Usually this is a good idea, you do not want to have strange chars in your file paths, as this might results in problems when these paths get passed to the scheduler or other command-line tools that the package interoperates with. But on some OS this check might be too strict and cause problems. Setting the flag to \code{FALSE} allows to disable the check entirely. The default is \code{FALSE} on Windows systems and \code{TRUE} else.} } } BatchJobs/man/getJobNr.Rd0000644000176000001440000000051512465167456014736 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Job.R \name{getJobNr} \alias{getJobNr} \title{Get number of jobs in registry.} \usage{ getJobNr(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} } \value{ [\code{integer(1)}]. } \description{ Get number of jobs in registry. } BatchJobs/man/makeClusterFunctionsSSH.Rd0000644000176000001440000000360512465167456017755 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsSSH.R \name{makeClusterFunctionsSSH} \alias{makeClusterFunctionsSSH} \title{Create an SSH cluster to execute jobs.} \usage{ makeClusterFunctionsSSH(..., workers) } \arguments{ \item{...}{[\code{\link{SSHWorker}}]\cr Worker objects, all created with \code{\link{makeSSHWorker}}.} \item{workers}{[list of \code{\link{SSHWorker}}]\cr Alternative way to pass workers.} } \value{ [\code{ClusterFunctions}]. } \description{ Worker nodes must share the same file system and be accessible by ssh without manually entering passwords (e.g. by ssh-agent or passwordless pubkey). Note that you can also use this function to parallelize on multiple cores on your local machine. But you still have to run an ssh server and provide passwordless access to localhost. } \examples{ \dontrun{ # Assume you have three nodes larry, curley and moe. All have 6 # cpu cores. On curley and moe R is installed under # "/opt/R/R-current" and on larry R is installed under # "/usr/local/R/". larry should not be used extensively because # somebody else wants to compute there as well. # Then a call to 'makeClusterFunctionsSSH' # might look like this: cluster.functions = makeClusterFunctionsSSH( makeSSHWorker(nodename = "larry", rhome = "/usr/local/R", max.jobs = 2), makeSSHWorker(nodename = "curley", rhome = "/opt/R/R-current"), makeSSHWorker(nodename = "moe", rhome = "/opt/R/R-current")) } } \seealso{ \code{\link{makeSSHWorker}} Other clusterFunctions: \code{\link{ClusterFunctions}}, \code{\link{makeClusterFunctions}}; \code{\link{makeClusterFunctionsInteractive}}; \code{\link{makeClusterFunctionsLSF}}; \code{\link{makeClusterFunctionsLocal}}; \code{\link{makeClusterFunctionsMulticore}}; \code{\link{makeClusterFunctionsSGE}}; \code{\link{makeClusterFunctionsSLURM}}; \code{\link{makeClusterFunctionsTorque}} } BatchJobs/man/getJob.Rd0000644000176000001440000000073212465167456014437 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/getJob.R \name{getJob} \alias{getJob} \title{Get job from registry by id.} \usage{ getJob(reg, id, check.id = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{id}{[\code{integer(1)}]\cr Id of job.} \item{check.id}{[\code{logical(1)}]\cr Check the job id? Default is \code{TRUE}.} } \value{ [\code{\link{Job}}]. } \description{ Get job from registry by id. } BatchJobs/man/resetJobs.Rd0000644000176000001440000000305512465167456015166 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/resetJobs.R \name{resetJobs} \alias{resetJobs} \title{Reset computational state of jobs.} \usage{ resetJobs(reg, ids, force = FALSE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs to kill. Default is none.} \item{force}{[\code{logical(1)}]\cr Reset jobs without checking whether they are currently running. READ THE DETAILS SECTION! Default is \code{FALSE}.} } \value{ Vector of reseted job ids. } \description{ Reset state of jobs in the database. Useful under two circumstances: Either to re-submit them because of changes in e.g. external data or to resolve rare issues when jobs are killed in an unfortunate state and therefore blocking your registry. The function internally lists all jobs on the batch system and if those include some of the jobs you want to reset, it informs you to kill them first by raising an exception. If you really know what you are doing, you may set \code{force} to \code{TRUE} to omit this sanity check. Note that this is a dangerous operation to perform which may harm the database integrity. In this case you HAVE to make externally sure that none of the jobs you want to reset are still running. } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{debugSSH}}; \code{\link{getErrorMessages}}; \code{\link{getJobInfo}}; \code{\link{getLogFiles}}; \code{\link{grepLogs}}; \code{\link{killJobs}}; \code{\link{setJobFunction}}; \code{\link{showLog}}; \code{\link{testJob}} } BatchJobs/man/cfBrewTemplate.Rd0000644000176000001440000000223012465167456016124 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsHelpers.R \name{cfBrewTemplate} \alias{cfBrewTemplate} \title{Cluster functions helper: Brew your template into a job description file.} \usage{ cfBrewTemplate(conf, template, rscript, extension) } \arguments{ \item{conf}{[\code{environment}]\cr BatchJobs configuration.} \item{template}{[\code{character(1)}]\cr Job desfription template as a char vecrtor, possibly read in via \code{\link{cfReadBrewTemplate}}.} \item{rscript}{[\code{character(1)}]\cr File path to you your corresponding R script for the job.} \item{extension}{[\code{character(1)}]\cr Extension for the job description file, e.g. \dQuote{pbs}.} } \value{ [\code{character(1)}]. File path of result. } \description{ This function is only intended for use in your own cluster functions implementation. Calls brew silently on your template, any error will lead to an exception. If debug mode is turned on in the configuration, the file is stored at the same place as the corresponding R script in the \dQuote{jobs}-subdir of your files directory, otherwise in the temp dir via \code{\link{tempfile}}. } BatchJobs/man/sourceRegistryFiles.Rd0000644000176000001440000000103312465167456017234 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{sourceRegistryFiles} \alias{sourceRegistryFiles} \title{Source registry files} \usage{ sourceRegistryFiles(reg, envir = .GlobalEnv) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{envir}{[\code{environment}]\cr Environment to source the files into. Default is the global environment.} } \value{ Nothing. } \description{ Sources all files found in \code{src.dirs} and specified via \code{src.files}. } BatchJobs/man/loadExports.Rd0000644000176000001440000000175212465170744015526 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Exports.R \name{loadExports} \alias{loadExports} \title{Load exported R data objects.} \usage{ loadExports(reg, what = NULL) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{what}{[\code{character}]\cr Names of objects and corresponding \dQuote{RData} files which should be loaded. Default \code{NULL} loads all files.} } \value{ [\code{character}]. Invisibly returns a character vector of loaded objects. } \description{ Loads exported \code{RData} object files in the \dQuote{exports} subdirectory of your \code{file.dir} and assigns the objects to the global environment. } \seealso{ Other exports: \code{\link{addRegistryPackages}}; \code{\link{addRegistrySourceDirs}}; \code{\link{addRegistrySourceFiles}}; \code{\link{batchExport}}; \code{\link{batchUnexport}}; \code{\link{removeRegistryPackages}}; \code{\link{removeRegistrySourceDirs}}; \code{\link{removeRegistrySourceFiles}} } BatchJobs/man/setJobFunction.Rd0000644000176000001440000000266112465167456016164 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/setJobFunction.R \name{setJobFunction} \alias{setJobFunction} \title{Sets the job function for already existing jobs.} \usage{ setJobFunction(reg, ids, fun, more.args = list(), reset = TRUE, force = FALSE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} \item{fun}{[\code{function}]\cr Replacement function.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} \item{reset}{[\code{logical(1)}]\cr Reset job status via \code{\link{resetJobs}}. Default is \code{TRUE}.} \item{force}{[\code{logical(1)}]\cr See \code{\link{resetJobs}}. Default is \code{FALSE}.} } \value{ Nothing. } \description{ Use this function only as last measure when there is a bug in a part of your job function and you have already computed a large number of (unaffected) results. This function allows you to fix the error and to associate the jobs with the corrected function. Note that by default the computational state of the affected jobs is also reset. } \seealso{ Other debug: \code{\link{debugMulticore}}; \code{\link{debugSSH}}; \code{\link{getErrorMessages}}; \code{\link{getJobInfo}}; \code{\link{getLogFiles}}; \code{\link{grepLogs}}; \code{\link{killJobs}}; \code{\link{resetJobs}}; \code{\link{showLog}}; \code{\link{testJob}} } BatchJobs/man/callFunctionOnSSHWorkers.Rd0000644000176000001440000000432712465167456020102 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/callFunctionOnSSHWorkers.R \name{callFunctionOnSSHWorkers} \alias{callFunctionOnSSHWorkers} \title{Call an arbitrary function on specified SSH workers.} \usage{ callFunctionOnSSHWorkers(nodenames, fun, ..., consecutive = FALSE, show.output = consecutive, simplify = TRUE, use.names = TRUE, dir = getwd()) } \arguments{ \item{nodenames}{[\code{character}]\cr Nodenames of workers to call function on. Only workers which were specified in your \code{\link{makeClusterFunctionsSSH}} configuration can be used.} \item{fun}{[\code{function}]\cr Function to call on workers.} \item{...}{[any]\cr Arguments for \code{fun}.} \item{consecutive}{[\code{logical(1)}]\cr Do calls consecutively and always wait until each worker is done. Default is \code{FALSE}.} \item{show.output}{[\code{logical(1)}]\cr Show output of workers on master during call. Can be useful to see what is happening. Can only be used in consecutive mode. Default is \code{consecutive}.} \item{simplify}{[\code{logical(1)}]\cr Should the result be simplified? See \code{\link{sapply}}. Default is \code{TRUE}.} \item{use.names}{[\code{logical(1)}]\cr Name results by \code{nodenames}. Default is \code{TRUE}.} \item{dir}{[\code{character(1)}]\cr Directory under which a temporary registry will be created in a subdirectory for communication. This has to be somewhere on the shared filesystem. The created subdirectory will be cleaned up on exit. Default is current working directory.} } \value{ Results of function calls, either a list or simplified. } \description{ Calls can be made in parallel or consecutively, the function waits until all calls have finished and returns call results. In consecutive mode the output on the workers can also be shown on the master during computation. Please read and understand the comments for argument \code{dir}. Note that this function should only be used for short administrative tasks or information gathering on the workers, the true work horse for real computation is \code{\link{submitJobs}}. In \code{\link{makeSSHWorker}} various options for load management are possible. Note that these will be ignored for the current call to execute it immediatly. } BatchJobs/man/dbGetJobs.Rd0000644000176000001440000000066612465167456015076 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/database.R \name{dbGetJobs} \alias{dbGetJobs} \title{ONLY FOR INTERNAL USAGE.} \usage{ dbGetJobs(reg, ids) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of selected jobs.} } \value{ [list of \code{\link{Job}}]. Retrieved jobs from DB. } \description{ ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/getConfig.Rd0000644000176000001440000000076212465167456015135 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/conf.R \name{getConfig} \alias{getConfig} \title{Returns a list of BatchJobs configuration settings} \usage{ getConfig() } \value{ \code{list} of current configuration variables with classs \dQuote{Config}. } \description{ Returns a list of BatchJobs configuration settings } \seealso{ Other conf: \code{\link{.BatchJobs.R}}, \code{\link{configuration}}; \code{\link{loadConfig}}; \code{\link{setConfig}} } BatchJobs/man/makeSSHWorker.Rd0000644000176000001440000000423712465167456015716 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsSSH.R \name{makeSSHWorker} \alias{SSHWorker} \alias{makeSSHWorker} \title{Create SSH worker for SSH cluster functions.} \usage{ makeSSHWorker(nodename, rhome = "", ncpus, max.jobs, max.load, nice, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script) } \arguments{ \item{nodename}{[\code{character(1)}]\cr Host name of node.} \item{rhome}{[\code{character(1)}]\cr Path to R installation on worker. \dQuote{} means R installation on the PATH is used, of course this implies that it must be on the PATH (also for non-interactive shells)! Default is \dQuote{}.} \item{ncpus}{[\code{integers(1)}]\cr Number of VPUs of worker. Default means to query the worker via \dQuote{/proc/cpuinfo}.} \item{max.jobs}{[\code{integer(1)}]\cr Maximal number of jobs that can run concurrently for the current registry. Default is \code{ncpus}.} \item{max.load}{[\code{numeric(1)}]\cr Load average (of the last 5 min) at which the worker is considered occupied, so that no job can be submitted. Default is \code{ncpus-1}.} \item{nice}{[\code{integer(1)}]\cr Process priority to run R with set via nice. Integers between -20 and 19 are allowed. If missing, processes are not nice'd and the system default applies (usually 0). Default is no niceing.} \item{r.options}{[\code{character}] Options for R and Rscript, one option per element of the vector, a la \dQuote{--vanilla}. Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}.} \item{script}{[\code{character(1)}]\cr Path to helper bash script which interacts with the worker. You really should not have to touch this, as this would imply that we have screwed up and published an incompatible version for your system. This option is only provided as a last resort for very experienced hackers. Note that the path has to be absolute. This is what is done in the package: \url{https://github.com/tudo-r/BatchJobs/blob/master/inst/bin/linux-helper} Default means to take it from package directory.} } \value{ [\code{\link{SSHWorker}}]. } \description{ Create SSH worker for SSH cluster functions. } BatchJobs/man/removeRegistrySourceFiles.Rd0000644000176000001440000000152012465167456020413 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{removeRegistrySourceFiles} \alias{removeRegistrySourceFiles} \title{Remove source files from registry.} \usage{ removeRegistrySourceFiles(reg, src.files) } \arguments{ \item{reg}{[\code{Registry}]\cr Registry.} \item{src.files}{[\code{character}]\cr Paths to remove from registry.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{src.files} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}; \code{\link{addRegistrySourceDirs}}; \code{\link{addRegistrySourceFiles}}; \code{\link{batchExport}}; \code{\link{batchUnexport}}; \code{\link{loadExports}}; \code{\link{removeRegistryPackages}}; \code{\link{removeRegistrySourceDirs}} } BatchJobs/man/cfReadBrewTemplate.Rd0000644000176000001440000000122612465167456016724 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsHelpers.R \name{cfReadBrewTemplate} \alias{cfReadBrewTemplate} \title{Cluster functions helper: Read in your brew template file.} \usage{ cfReadBrewTemplate(template.file) } \arguments{ \item{template.file}{[\code{character(1)}]\cr File path.} } \value{ [\code{character}]. } \description{ This function is only intended for use in your own cluster functions implementation. Simply reads your template and returns it as a character vector. If you do this in the constructor of your cluster functions once, you can avoid this repeated file access later on. } BatchJobs/man/removeRegistry.Rd0000644000176000001440000000154712471624612016245 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/Registry.R \name{removeRegistry} \alias{removeRegistry} \title{Remove a registry object.} \usage{ removeRegistry(reg, ask = c("yes", "no")) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ask}{[\code{character(1)}]\cr If \code{"yes"} the user is prompted to confirm the action. If trying to prompt the user this way in a non-interactive session, then an informative error is generated. If \code{"no"}, the registry will be removed without further confirmation.} } \value{ [\code{logical[1]}] } \description{ If there are no live/running jobs, the registry will be closed and all of its files will be removed from the file system. If there are live/running jobs, an informative error is generated. The default is to prompt the user for confirmation. } BatchJobs/man/makeClusterFunctionsLocal.Rd0000644000176000001440000000225512465167456020352 0ustar ripleyusers% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/clusterFunctionsLocal.R \name{makeClusterFunctionsLocal} \alias{makeClusterFunctionsLocal} \title{Create cluster functions for sequential execution on local host.} \usage{ makeClusterFunctionsLocal() } \value{ [\code{\link{ClusterFunctions}}]. } \description{ All jobs executed under these cluster functions are executed sequentially, but in an independent, new R session. That is, \code{submitJob} does not return until the job has finished. The main use of this \code{ClusterFunctions} implementation is to test and debug programs on a local computer. Listing jobs returns an empty vector (as no jobs can be running when you call this) and \code{killJob} returns at once (for the same reason). } \seealso{ Other clusterFunctions: \code{\link{ClusterFunctions}}, \code{\link{makeClusterFunctions}}; \code{\link{makeClusterFunctionsInteractive}}; \code{\link{makeClusterFunctionsLSF}}; \code{\link{makeClusterFunctionsMulticore}}; \code{\link{makeClusterFunctionsSGE}}; \code{\link{makeClusterFunctionsSLURM}}; \code{\link{makeClusterFunctionsSSH}}; \code{\link{makeClusterFunctionsTorque}} } BatchJobs/LICENSE0000644000176000001440000000007412415510734013151 0ustar ripleyusersYEAR: 2013-2014 COPYRIGHT HOLDER: Bernd Bischl, Michel Lang