diff --git a/.Rbuildignore b/.Rbuildignore index 77a17cc..090355b 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -4,11 +4,11 @@ ^\.github$ ^CODE_OF_CONDUCT\.md$ ^LICENSE\.md$ -^man/commits_range\.Rd$ -^man/blob_to_file\.Rd$ -^man/install_minio\.Rd$ -^man/sys_tempdir\.Rd$ -^man/sys_tempfile\.Rd$ -^man/as_repo\.Rd$ -^man/as_commit\.Rd$ ^CITATION\.cff$ +^codemeta.json$ +^.lintr$ +^_pkgdown\.yml$ +^docs$ +^pkgdown$ +^inst/scratch$ +^inst/mc$ diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index ed7650c..438046b 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -3,8 +3,6 @@ on: push: branches: [main, master] - pull_request: - branches: [main, master] release: types: [published] workflow_dispatch: diff --git a/.gitignore b/.gitignore index 72e042c..f8d4146 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,111 @@ -.Rproj.user +inst/doc +inst/scratch +inst/mc +docs +tests/testthat/*.log +### R ### +# History files +*Rhistory .Rhistory -.Rdata +.Rapp.history +.Rproj.user +.RData +.Ruserdata +# Session Data files +# User-specific files +# Example code in package build process +*-Ex.R +# Output files from R CMD build +/*.tar.gz +# Output files from R CMD check +/*.Rcheck/ +# RStudio files +.Rproj.user/ +# produced vignettes +vignettes/*.html +vignettes/*.pdf +# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 .httr-oauth -.DS_Store +# knitr and R markdown default cache directories +*_cache/ +/cache/ +# Temporary files created by R markdown +*.utf8.md +*.knit.md .quarto -inst/doc +### R.Bookdown Stack ### +# R package: bookdown caching files +/*_files/ +# End of https://www.toptal.com/developers/gitignore/api/r,c++,macos,microsoftoffice +### C++ ### +# Prerequisites +*.d +# Compiled Object files +*.slo +*.lo +*.o +*.obj +# Precompiled Headers +*.gch +*.pch +# Linker files +*.ilk +# Debugger Files +*.pdb +# Compiled Dynamic libraries +*.so +*.dylib +*.dll +# Fortran module files +*.mod +*.smod +# Compiled Static libraries +*.lai +*.la +*.a +*.lib +# Executables +*.exe +*.out +*.app +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride +# Icon must end with two \r +Icon +# Thumbnails +._* +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk +### MicrosoftOffice ### +*.tmp +# Word temporary +~$*.doc* +# Word Auto Backup File +Backup of *.doc* +# Excel temporary +~$*.xls* +# Excel Backup File +*.xlk +# PowerPoint temporary +~$*.ppt* +# Visio autosave temporary files +*.~vsd* + + +# Created by https://www.toptal.com/developers/gitignore/api/r,c++,macos,microsoftoffice +# Edit at https://www.toptal.com/developers/gitignore?templates=r,c++,macos,microsoftoffice diff --git a/DESCRIPTION b/DESCRIPTION index 3419c3b..dabcf1b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -5,37 +5,42 @@ Authors@R: c( person("Noam", "Ross", , "ross@ecohealthalliance.org", role = c("aut", "cre"), comment = c(ORCID = "0000-0002-2136-0000")), person("EcoHealth Alliance", role = c("cph", "fnd")) - ) -Description: Tools working with the history of version-controlled projects, + ) +Description: The 'relic' package provides tools for extracting files and + objects from the history of a git repository. It is a high-level + interface designed to enable comparison of objects in reproducible + research workflows, especially pipelines that use the 'targets' + package. License: MIT + file LICENSE -Encoding: UTF-8 -Language: en-US -Roxygen: list(markdown = TRUE, roclets = c("collate", "rd", "namespace", "devtag::dev_roclet")) -RoxygenNote: 7.2.3 +URL: https://ecohealthalliance.github.io/relic, + https://ecohealthalliance.r-universe.dev/relic +BugReports: https://github.com/ecohealthalliance/relic/issues Imports: - git2r, fs, + git2r, rlang Suggests: - igraph, + callr, + devtag, + glue, knitr, + lintr, + minioclient, + pkgcheck, rmarkdown, + spelling, targets, testthat (>= 3.0.0), - devtag, - pkgcheck, - lintr, - processx, - paws, - httr, - autotest, - callr, - spelling + withr +VignetteBuilder: + knitr Remotes: - ropensci/git2r, - moodymudskipper/devtag, - ropensci-review-tools/pkgcheck + cboettig/minioclient#14, + ropensci-review-tools/pkgcheck, + ropensci/git2r Config/testthat/edition: 3 -URL: https://ecohealthalliance.github.io/relic, https://ecohealthalliance.r-universe.dev/relic -BugReports: https://github.com/ecohealthalliance/relic/issues -VignetteBuilder: knitr +Encoding: UTF-8 +Language: en-US +Roxygen: list(markdown = TRUE, roclets = c("collate", "rd", "namespace", + "devtag::dev_roclet")) +RoxygenNote: 7.2.3 diff --git a/NAMESPACE b/NAMESPACE index 7af6569..3b35d53 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,7 +1,5 @@ # Generated by roxygen2: do not edit by hand -S3method(relic,git_blob) -S3method(relic,git_tree) export(commits_between) export(dir_ls_version) export(dir_ls_versions) @@ -9,10 +7,15 @@ export(file_copy_version) export(file_copy_versions) export(file_read_version) export(file_read_versions) -export(graph_from_commits) -export(graph_from_git_repository) export(relic) -export(tar_load_version) +export(relic_cache) +export(relic_cache_clear) +export(tar_exists_version) +export(tar_exists_version_raw) +export(tar_meta_version) +export(tar_read_raw_version) +export(tar_read_raw_versions) +export(tar_read_version) export(tar_read_versions) import(fs) import(git2r) diff --git a/NEWS.md b/NEWS.md index 5b06b6a..a6cb6f5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ # relic (development version) +* Started adding tests + +# relic 0.0.1 + * Initial proof of concept diff --git a/R/commits_between.R b/R/commits_between.R index cf315c7..3dc145d 100644 --- a/R/commits_between.R +++ b/R/commits_between.R @@ -13,7 +13,8 @@ #' #' @param from A commit object or revision string. A string of the form #' 'from...to' may also be passed, in which case the commits between the two -#' revision strings are used and the `to` argument is ignored. +#' revision strings are used and the `to` argument is ignored. If a list of commits +#' is passed, these commits are used rather than calculating the commits between. #' @param to A commit object or reference #' @param filter_file The path to a file relative to the git directory. If not NULL, #' only commits modifying this file will be returned. Note that modifying @@ -24,6 +25,9 @@ #' @export commits_between <- function(from, to = NULL, filter_file = NULL, repo = ".") { repo <- as_repo(repo) + if (is.list(from) && all(vapply(from, is_commit, logical(1)))) { + return(commits) + } range <- commits_range(from, to, repo) if (is.null(range[[2]])) { return(list(range[[1]])) @@ -66,7 +70,7 @@ commits_between <- function(from, to = NULL, filter_file = NULL, repo = ".") { } #' Parse 1 or 2 commits or revision strings into two commits -#' @dev +#' @noRd commits_range <- function(from, to, repo) { if (is.character(from)) { refs <- strsplit(from, "...", fixed = TRUE)[[1]] diff --git a/R/dir_ls_version.R b/R/dir_ls_version.R index 645d7ba..ce4e64c 100644 --- a/R/dir_ls_version.R +++ b/R/dir_ls_version.R @@ -1,16 +1,17 @@ #' List files and folders in a git repository #' #' @inheritParams file_read_version -#' @param param all If TRUE hidden files are also returned +#' @param all If TRUE hidden files are also returned #' @param recurse If TRUE recurse fully, if a positive number the number of levels to recurse #' @param type One or more of "any", "file", "directory", "symlink", or "submodule" -#' @param regexp A regular expression (e.g. ⁠[.]csv$⁠) passed on to grep() to filter paths. -#' @param glob A wildcard aka globbing pattern (e.g. ⁠*.csv⁠) passed on to grep() to filter paths +#' @param regexp A regular expression (e.g. '\\.csv$'⁠) passed on to grep() to filter paths. +#' @param glob A wildcard aka globbing pattern (e.g. '*.csv'⁠) passed on to grep() to filter paths #' @param invert If TRUE, return paths that do not match the pattern or glob #' @return A character vector of paths #' @export -dir_ls_version <- function(path = ".", ref = "HEAD", all = FALSE, recurse = FALSE, type = "any", regexp = NULL, glob = NULL, repo = ".") { - tree <- get_obj_at_commit(path, ref, repo) +dir_ls_version <- function(path = ".", ref = "HEAD", all = FALSE, recurse = FALSE, type = "any", regexp = NULL, glob = NULL, invert = FALSE, repo = ".") { + commit <- as_commit(ref, repo) + tree <- get_obj_at_commit(path, commit) if (!is_tree(tree)) abort("Path is not a directory") if (!is.numeric(recurse)) recurse <- if (recurse) Inf else 0 @@ -27,7 +28,7 @@ dir_ls_version <- function(path = ".", ref = "HEAD", all = FALSE, recurse = FALS if (!all) { paths <- path_filter(paths, regexp = "^[^\\.].*$") } - paths <- path_filter(paths, glob = glob, regexp = regexp) + paths <- path_filter(paths, glob = glob, regexp = regexp, invert = invert) paths } diff --git a/R/example_repo.R b/R/example_repo.R new file mode 100644 index 0000000..76b25eb --- /dev/null +++ b/R/example_repo.R @@ -0,0 +1,62 @@ +#' Set up a testing and example repository +#' +#' This function generates a repository with a commit history that can be used +#' for testing and examples. +#' @param dir Path to the directory where the repository should be created +#' @param reporter the reporter to use when building targets with [targets::tar_make()]. Defaults to "silent". +#' @param s3 Whether the repository should use S3 storage for targets. Note that +#' the S3 endpoint and bucket must already be available. +create_example_repo <- function(dir = fs::file_temp("relic_example_"), reporter = "silent", s3 = TRUE) { + check_installed(c("targets", "glue")) + if (dir_exists(dir)) dir_delete(dir) + dir_create(dir) + od <- setwd(dir) + on.exit(setwd(od)) + repo <- git2r::init() + # Configure the repository to use a generic user name and email address + git2r::config(repo, user.name = "relic-bot", user.email = "relic@relic.r.pacakge") + + # Create a minimal _targets.R file + write_to_file( + "_targets.R", + 'library(targets) + list( + tar_target(cars, mtcars, repository = "local"), + tar_target(cars_csv, {name <- "cars.csv"; write.csv(cars, name); name}, format = "file", repository = "local") + ) + ' + ) + + stamp("initial-target-file") + + targets::tar_make(reporter = reporter) + append_to_file("_targets/.gitignore", c("!objects/\n!objects/*")) + stamp("first-targets-run") + + overwrite_at_line( + "_targets.R", 3, + "tar_target(cars, rbind(mtcars, mtcars))," + ) + targets::tar_make(reporter = reporter) + stamp("longer-cars") + + if(s3) { + insert_lines_at( + "_targets.R", 2, + 'Sys.setenv(AWS_ACCESS_KEY_ID="minioadmin", AWS_SECRET_ACCESS_KEY="minioadmin") + tar_option_set( + resources = tar_resources( + aws = tar_resources_aws( + bucket = "relic-test", + prefix = "_targets", + endpoint = "http://localhost:9000" + )), + repository = "aws" + ) + ') + targets::tar_make(reporter = reporter) + stamp("setup-s3") + } + + dir +} diff --git a/R/example_utils_git.R b/R/example_utils_git.R new file mode 100644 index 0000000..ebb0c48 --- /dev/null +++ b/R/example_utils_git.R @@ -0,0 +1,42 @@ +#' Convenience functions for making and viewing a git repository +#' +#' @noRd +#' @param msg Text to use for both the commit message and tag +#' @return The path to the repository +#' @rdname example_utils_git +stamp <- function(msg, repo = ".") { + git2r::add(repo = repo, path = ".") + git2r::commit(repo = repo, message = msg, all = TRUE) + git2r::tag(object = repo, name = msg) +} + +#' @noRd +#' @param nlines the number of lines to print from each file +#' @rdname example_utils_git +print_dir <- function(repo, nlines = 10) { + files <- dir_ls(dir, all = TRUE, recurse = TRUE, type = "file", regexp = ".*\\.git/.*", invert = TRUE) + for (f in files) { + print(path(f)) + text_content <- tryCatch( + { + readLines(f, n = nlines) + }, + warning = function(w) { + return("\n") + }, + error = function(e) { + stop(e) + } + ) + # Check if file is binary + + cat(c(text_content, "\n"), sep = "\n") + } + if (dir_exists(path(dir, ".git"))) { + repo <- git2r::repository(dir) + cat("Git commits:\n") + print(as.data.frame(repo)) + cat(c("TAGS:", paste(names(tags(repo)), collapse = ", "), "\n")) + invisible(NULL) + } +} diff --git a/R/example_utils_txt.R b/R/example_utils_txt.R new file mode 100644 index 0000000..6c4ccc0 --- /dev/null +++ b/R/example_utils_txt.R @@ -0,0 +1,50 @@ +#' Tools for generating an example repository +#' @noRd +#' @param path path to the file to write or modify +#' @param text text to write to the file +#' @rdname example_utils_txt +write_to_file <- function(path, text) { + dir_create(path_dir(path)) + text <- glue::glue(text, .open = "<<", .close = ">>") + text <- strsplit(text, "\n")[[1]] + cat(text, file = path, sep = "\n") +} + +#' @noRd +#' @rdname example_utils_txt +append_to_file <- function(path, text) { + dir_create(path_dir(path)) + text <- strsplit(glue::glue(text, .open = "<<", .close = ">>", ), "\n")[[1]] + cat(text, file = path, sep = "\n", append = TRUE) +} + +#' @noRd +#' @param line the line number to start overwriting or inserting at +#' @rdname example_utils_txt +overwrite_at_line <- function(path, line, text) { + file_lines <- readLines(path) + text <- strsplit(glue::glue(text, .open = "<<", .close = ">>", ), "\n")[[1]] + # text_lines <- seq_along(text) + line - 1 + for (i in seq_along(text)) { + file_lines[line + i - 1] <- text[i] + } + writeLines(file_lines, path) +} + +#' @noRd +#' @param lines the lines to delete +#' @rdname example_utils_txt +delete_lines <- function(path, lines) { + file_lines <- readLines(path) + file_lines <- file_lines[-lines] + writeLines(file_lines, path) +} + +#' @noRd +#' @rdname example_utils_txt +insert_lines_at <- function(path, line, text) { + file_lines <- readLines(path) + text <- strsplit(glue::glue(text, .open = "<<", .close = ">>", ), "\n")[[1]] + file_lines <- c(file_lines[1:(line-1)], text, file_lines[(line):length(file_lines)]) + writeLines(file_lines, path) +} diff --git a/R/extract.R b/R/extract.R index 534ec4d..d1993f8 100644 --- a/R/extract.R +++ b/R/extract.R @@ -6,33 +6,33 @@ get_obj_at_commit <- function(path, commit) { return(obj) } - for (i in seq_len(splits)) { + for (i in seq_along(splits)) { parent <- obj obj <- obj[splits[i]] - if (is_empty(obj)) { + if (rlang::is_empty(obj)) { return(NULL) } } if (is_blob(obj)) { - return(relic(obj, mode = parent$mode[parent$id == obj$sha])) + return(relic(obj, mode = parent$filemode[parent$id == obj$sha])) } else { return(obj) } } read_blob <- function(blob) { - content(blob, split = FALSE, raw = is_binary(obj)) + content(blob, split = FALSE, raw = git2r::is_binary(blob)) } #' Write out the contents of a blob to a file #' Will always create the path and overwrite existing files -#' @dev -blob_to_file <- function(relic, path, mode = relic$mode) { - contents <- read_blob(obj) +#' @noRd +blob_to_file <- function(relic, path, mode = attr(relic, "mode")) { + contents <- read_blob(relic) dir_create(path_dir(path)) if (mode == "120000") { - file_symlink(contents, path) + link_create(contents, path) } else if (mode %in% c("100644", "100755")) { if (is.raw(contents)) { writeBin(contents, path) @@ -49,7 +49,7 @@ blob_to_file <- function(relic, path, mode = relic$mode) { path } -tree_to_dir <- function(tree, path, recurse = TRUE) { +tree_to_dir <- function(obj, path, recurse = TRUE) { dir_create(path) if (!is.numeric(recurse)) recurse <- (if (recurse) Inf else 0) + 1 while (recurse > 0) { @@ -60,8 +60,8 @@ tree_to_dir <- function(tree, path, recurse = TRUE) { path_i <- path(dir, name_i) if (is_tree(obj_i)) { dir_create(path_i) - tree_to_dir(obj_i, name = name_i, dir = path_i, recurse = recurse) - } else if (if_blob(obj_i)) { + tree_to_dir(obj_i, path_i, recurse) + } else if (is_blob(obj_i)) { blob_to_file(obj_i, path_i, sprintf("%06o", tree$filemode[i])) } else { abort("Object is not a git blob or tree") diff --git a/R/file_copy_version.R b/R/file_copy_version.R index acd8b80..d358a9b 100644 --- a/R/file_copy_version.R +++ b/R/file_copy_version.R @@ -2,6 +2,11 @@ #' #' Copy files from git history to disk. Directories are copied recursively. #' +#' The default behavior is to copy the file to a cache directory named after +#' both the commit and and the file path. These paths are universally unique, +#' so if `use_cache = TRUE`, the file will not be re-copied if it already exists +#' at that path. +#' #' @inheritParams file_read_version #' @param name the name to give the file when it is copied to disk. Defaults to #' the original name of the file or directory. @@ -13,34 +18,35 @@ #' both `name` and `dir` are NULL, FALSE otherwise. #' @param recurse If `path` is a directory, should should it be copied recursively? If #' numeric, how many levels deep should it be copied? Defaults to TRUE. +#' @param use_cache Should the cache be used? Defaults to TRUE. #' @return #' - For `file_copy_version()`, A character vector of paths to the copied files or directories. #' - For `file_copy_versions()`, a list of character vectors of paths. #' Where files do not exist in a commit, `NA` values are returned. #' @export -file_copy_version <- function(path, ref = "HEAD", name = NULL, dir = NULL, full_name = NULL, recurse = TRUE, repo = ".") { - path(vapply(path, \(x) file_copy_version_single(x, ref, repo, name, dir, full_name, recurse, repo), character(1))) +file_copy_version <- function(path, ref = "HEAD", name = NULL, dir = NULL, full_name = NULL, recurse = TRUE, repo = ".", use_cache = TRUE) { + path(vapply(path, \(x) file_copy_version_single(x, ref, name, dir, full_name, recurse, repo, use_cache), character(1))) } #' @inheritParams commits_between #' @export #' @rdname file_copy_version -file_copy_versions <- function(path, from = "HEAD", to = NULL, name = NULL, dir = NULL, file_filter = NULL, repo = ".") { +file_copy_versions <- function(path, from = "HEAD", to = NULL, name = NULL, dir = NULL, full_name = NULL, recurse = TRUE, repo = ".", use_cache = TRUE) { commits <- commits_between(from = from, to = to, repo = repo) - versions <- lapply(commits, \(x) file_copy_version(path, x, name, dir, fir_full, recurse, repo)) + versions <- lapply(commits, \(x) file_copy_version(path, x, name, dir, full_name, recurse, repo, use_cache)) names(versions) <- vapply(commits, sha, character(1)) versions } -file_copy_version_single <- function(path, ref, name, dir, full_name, recurse, repo) { +file_copy_version_single <- function(path, ref, name, dir, full_name, recurse, repo, use_cache) { commit <- as_commit(ref, repo) if (is.null(full_name)) full_name <- is.null(name) && is.null(dir) if (is.null(name)) name <- path_file(path) - if (is.null(dir)) dir <- path(sys_tempdir(), sha(commit)) + if (is.null(dir)) dir <- path(relic_cache(), sha(commit)) if (full_name) dir <- path(dir, path_dir(path)) - out_path <- path(dir, name) + out_path <- path_norm(path(dir, name)) obj <- get_obj_at_commit(path, commit) if (is_none(obj)) { @@ -49,9 +55,11 @@ file_copy_version_single <- function(path, ref, name, dir, full_name, recurse, r if (is_tree(obj)) { dir_create(out_path) - return(file_copy_recursive(obj, dir = out_path, recurse = recurse, repo = repo)) + return(file_copy_recursive(obj, out_path, recurse, repo, use_cache)) } else if (is_blob(obj)) { - blob_to_file(obj, out_path, obj$mode) + if (!file_exists(out_path) || !use_cache) { + blob_to_file(obj, out_path, attr(obj, "mode")) + } } else { abort("Path is not a file or directory") } @@ -59,7 +67,7 @@ file_copy_version_single <- function(path, ref, name, dir, full_name, recurse, r out_path } -file_copy_recursive <- function(tree, dir, recurse, repo) { +file_copy_recursive <- function(obj, dir, recurse, repo, use_cache) { if (!is.numeric(recurse)) recurse <- if (recurse) Inf else 0 if (recurse > 0) { @@ -69,9 +77,11 @@ file_copy_recursive <- function(tree, dir, recurse, repo) { path_i <- path(dir, name_i) if (inherits(obj_i, "git_tree")) { dir_create(path_i) - file_copy_recursive(obj_i, name = name_i, dir = path_i, recurse = recurse - 1) + file_copy_recursive(obj_i, dir = path_i, recurse = recurse - 1) } else if (inherits(obj_i, "git_blob")) { + if (!file_exists(path_i) || !use_cache) { blob_to_file(obj_i, path_i) + } } else { abort("Object is not a git blob or tree") } diff --git a/R/file_read_version.R b/R/file_read_version.R index d8c0ae8..6f0eead 100644 --- a/R/file_read_version.R +++ b/R/file_read_version.R @@ -30,14 +30,12 @@ file_read_versions <- function(path, from = "HEAD", to = NULL, repo = ".") { versions } -file_read_version_single <- function(path, ref) { +file_read_version_single <- function(path, ref, repo) { commit <- as_commit(ref, repo) obj <- get_obj_at_commit(path, commit) - if (is_tree(obj) || is_empty(obj)) { + if (is_tree(obj) || rlang::is_empty(obj)) { return(NULL) } else if (is_blob(obj)) { return(read_blob(obj)) - } else { - abort("Path is not a file or directory") } } diff --git a/R/relic.R b/R/relic.R index 2423d23..3fc8763 100644 --- a/R/relic.R +++ b/R/relic.R @@ -1,28 +1,27 @@ #' An internal S3 class of git objects with additional metadata - +#' @param x an object is_relic <- function(x) inherits(x, "relic") && !is.null(attr(x, "mode")) #' @export +#' @noRd relic <- function(x, ...) { UseMethod("relic") } -#' @export -relic.git_tree <- function(tree, i) { - blob <- tree[i] +relic.git_tree <- function(x, i, ...) { + blob <- x[i] if (!is_blob(blob)) abort("Object is not a git blob") structure( blob, - mode = sprintf("%06o", tree$filemode[i]), + mode = if (is.integer(mode)) sprintf("%06o", mode) else mode, class = c("relic", "git_blob") ) } -#' @export -relic.git_blob <- function(blob, mode) { +relic.git_blob <- function(x, mode, ...) { structure( - blob, - mode = mode, + x, + mode = if (is.integer(mode)) sprintf("%06o", mode) else mode, class = c("relic", "git_blob") ) } diff --git a/R/tar_exists_version.R b/R/tar_exists_version.R new file mode 100644 index 0000000..3714616 --- /dev/null +++ b/R/tar_exists_version.R @@ -0,0 +1,24 @@ +#' Check if a target exists in a version of the pipeline +#' +#' @inheritParams tar_read_version +#' @export +tar_exists_version <- function(name, ref = "HEAD", branches = NULL, repo = ".", store = targets::tar_path_store()) { + check_installed("targets") + name <- targets::tar_deparse_language(substitute(name)) + tar_exists_version_raw(name, ref, branches, repo, store) +} + +#' @export +#' @rdname tar_exists_version +tar_exists_version_raw <- function(name, ref = "HEAD", branches = NULL, repo = ".", store = targets::tar_path_store()) { + check_installed("targets") + repo <- as_repo(repo) + ref <- as_commit(ref, repo) + path_store <- file_copy_version(store, ref, repo = repo, recurse = FALSE) + meta <- tar_meta_version(ref = ref, store = path_store) + target_meta <- meta[meta$name == name, ] + if (!nrow(target_meta)) { + return(FALSE) + } + TRUE +} diff --git a/R/tar_meta_version.R b/R/tar_meta_version.R new file mode 100644 index 0000000..6aebd9c --- /dev/null +++ b/R/tar_meta_version.R @@ -0,0 +1,15 @@ +#' Read a target project's metadata from git history +#' +#' @export +#' @param ref A git commit SHA, tag, branch or other [revision +#' string](https://git-scm.com/docs/git-rev-parse.html#_specifying_revisions): +#' such as "HEAD~1", or a [git2r::commit()] object. Defaults to "HEAD". +#' @param store Path to the targets store within the project. Defaults to the current project's store. +#' @param repo The path to the git repository or a [git2r::repository()] object. +#' @param ... Arguments passed to [targets::tar_meta()] +tar_meta_version <- function(ref = "HEAD", ..., store = targets::tar_path_store(), repo = ".") { + check_installed("targets") + path_meta <- file_copy_version(path(store, "meta", "meta"), ref, repo = repo) + meta <- targets::tar_meta(..., store = store) + meta +} diff --git a/R/tar_read_version.R b/R/tar_read_version.R new file mode 100644 index 0000000..333beb4 --- /dev/null +++ b/R/tar_read_version.R @@ -0,0 +1,80 @@ +#' Read a target's value from a git repository +#' +#' Reads the content of targets from a git repository. Target metadata and +#' local target files are extracted into a temporary directory before being +#' read in by `tar_read()`. For targets of type "file", the files are also +#' extracted and the paths to the extracted files are returned. +#' +#' For cloud targets, the target metadata is read from git history and then +#' this metadata is used to download the target from the cloud. For this to work, +#' cloud storage must be set up with versioning. Note that the cloud configuration +#' will use same bucket/endpoint/credentials set in the _current_ environment. +#' +#' @param name Name of the target. `tar_read_version()` can take a symbol, `tar_read_raw_version()` requires a character. +#' @param branches Integer of indices of the (targets) branches to load if the target is a pattern. +#' @param store Path to the targets store within the project. Defaults to the current project's store. +#' @inheritParams file_read_version +#' @export +#' @return The target's return values, loaded files in the git file/⁠, or the paths to the custom files and directories if format = "file" was set. +#' If the target is not found at the commit, NULL is returned. See [tar_exists_version()] to check for the presence of a target. +tar_read_version <- function(name, ref = "HEAD", branches = NULL, extract_files = TRUE, repo = ".", store = targets::tar_path_store()) { + check_installed("targets") + name <- targets::tar_deparse_language(substitute(name)) + tar_read_raw_version(name, branches, ref, extract_files, repo, store) +} + +#' @export +#' @inheritParams commits_between +#' @rdname tar_read_version +tar_read_versions <- function(name, from = "HEAD", to = NULL, branches = NULL, extract_files = TRUE, repo = ".", store = targets::tar_path_store()) { + check_installed("targets") + name <- targets::tar_deparse_language(substitute(name)) + tar_read_raw_versions(name, from, to, branches, extract_files, repo, store) +} + +#' @rdname tar_read_version +#' @param extract_files If TRUE, targets of type "file" will be extracted to a temporary directory and these paths will be returned. If FALSE, the paths as stored in the target will be returned unmodified +#' @export +tar_read_raw_version <- function(name, ref = "HEAD", branches = NULL, extract_files = TRUE, repo = ".", store = targets::tar_path_store()) { + check_installed("targets") + repo <- as_repo(repo) + ref <- as_commit(ref, repo) + path_store <- file_copy_version(store, ref, repo = repo, recurse = FALSE) + meta <- tar_meta_version(ref = ref, store = path_store) + target_meta <- meta[meta$name == name, ] + if (!nrow(target_meta)) { + return(NULL) + } + # For local targets + if (target_meta$repository == "local") { + if (!target_meta$format == "file") { + if (target_meta$type == "pattern") { + file_copy_version(path(targets::tar_path_objects_dir(path_store), target_meta$children[[1]]), ref, repo = repo) + } else { + file_copy_version(path(targets::tar_path_objects_dir(path_store), name), ref, repo = repo) + } + target <- targets::tar_read_raw(name, meta = meta, store = path_store) + } else if (target_meta$format == "file") { + file_path <- targets::tar_read_raw(name, meta = meta, store = path_store) + if (extract_files) { + target <- file_copy_version(file_path, ref, repo = repo) + } else { + target <- file_path + } + } + } else { + targets <- targets::tar_read_raw(name, meta = meta, store = path_store) + } + target +} + +#' @rdname tar_read_version +#' @export +tar_read_raw_versions <- function(name, from = "HEAD", to = NULL, branches = NULL, extract_files = TRUE, repo = ".", store = targets::tar_path_store()) { + check_installed("targets") + repo <- as_repo(repo) + commits <- commits_between(from, to, repo) + versions <- lapply(commits, \(x) tar_read_raw_version(name, x, branches, extract_files, repo, store)) + names(versions) <- vapply(commits, sha, character(1)) + versions +} diff --git a/R/targets.R b/R/targets.R deleted file mode 100644 index 9b33474..0000000 --- a/R/targets.R +++ /dev/null @@ -1,35 +0,0 @@ -#' Get a target from a given git reference -#' -#' @param name The name of the target -#' @param ref The git reference: tag, branch, SHA, or revision like HEAD~1 (see https://git-scm.com/docs/git-rev-parse.html#_specifying_revisions) -#' @param ... Additional arguments passed to \code{\link{tar_load}} or \code{\link{tar_read}} -#' @export -tar_load_version <- function(name, ref = "HEAD", envir = parent.frame(), ..., repo = ".") { - name <- targets::tar_deparse_language(substitute(name)) - tmp_store <- make_temporary_store(ref, repo) - tar_load(name, store = tmp_store, envir = envir, ...) -} - -#' @export -#' @rdname tar_load_version -tar_read_version <- function(name, ref = "HEAD", repo = ".", ...) { - name <- targets::tar_deparse_language(substitute(name)) - tmp_store <- make_temporary_store(ref, repo) - tar_read_raw(name, store = tmp_store, ...) -} - -make_temporary_store <- function(ref = "HEAD", repo = ".", store = targets::tar_path_store()) { - - current_store <- store - store_git_obj <- get_git_obj_at_reference(path = fs::path(current_store), ref = ref, repo = repo) - tmp_store_path <- fs::path(tar_tempdir(), sha(store_git_obj)) - - # Don't bother copying if the store is already in the tempdir - if(fs::dir_exists(tmp_store_path)) { - return(tmp_store_path) - } else { - tmp_store <- copy_git_tree(store_git_obj, dirname = sha(store_git_obj), path = tar_tempdir()) - } - - tmp_store -} diff --git a/R/utils.R b/R/utils.R index c8dde96..27edde4 100644 --- a/R/utils.R +++ b/R/utils.R @@ -1,24 +1,31 @@ -#' Get system-level temporary directories and files +#' The relic cache directory #' -#' Useful for files that should survive restarting R, but not resetting the -#' computer. For instance, [`targets`](https://docs.ropensci.org/targets/) uses -#' a new session for each target built, so these paths will persist across the -#' entire build process. -#' @dev -sys_tempdir <- function() { - path_dir(path_temp()) +#' Get the relic cache directory, which can be specified as either the R option +#' `relic.cache.dir` or the environment variable `RELIC_CACHE_DIR`. (The +#' environment variable has higher priority). If neither +#' is set, the default is set by tools::R_user_dir("relic", "cache"). +#' @export +#' @return The path to the relic cache directory +#' @examples +#' relic_cache() +relic_cache <- function() { + Sys.getenv( + "RELIC_CACHE_DIR", + getOption("relic.cache.dir", + default = tools::R_user_dir("relic", "cache") + ) + ) } -#' @dev -#' @param pattern A pattern to use for the temporary file name -#' @param ext An extension to use for the temporary file name -#' @rdname sys_tempdir -sys_tempfile <- function(pattern, ext = "") { - file_temp(pattern = "file", tmp_dir = sys_tempdir(), ext = ext) +#' @export +#' @rdname relic_cache +relic_cache_clear <- function() { + dir_delete(relic_cache()) } + #' Wrappers to convert objects to git2r objects but allow git2r objects to pass though -#' @dev +#' @noRd as_repo <- function(x) { if (inherits(x, "git_repository")) { x @@ -28,7 +35,7 @@ as_repo <- function(x) { } #' @rdname as_repo -#' @dev +#' @noRd as_commit <- function(x, repo = ".") { if (is_commit(x)) { x diff --git a/README.Rmd b/README.Rmd index 2f1f97a..237720b 100644 --- a/README.Rmd +++ b/README.Rmd @@ -26,7 +26,7 @@ knitr::opts_chunk$set( > _And deeper than oblivion we do bury_
> _The incensing relics of it_
-> William Shakespeare, [All's Well That Ends Well V.3](https://internetshakespeare.uvic.ca/doc/AWW_M/scene/5.3/index.html) +> William Shakespeare, [All's Well That Ends Well, V.3](https://internetshakespeare.uvic.ca/doc/AWW_M/scene/5.3/index.html) The `relic` package provides tools for working with version-controlled workflows, in git repositories. It enables extracting and comparing files and objects from project history. diff --git a/README.md b/README.md index 61d525d..142ea88 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ status](https://www.r-pkg.org/badges/version/relic)](https://CRAN.R-project.org/ > *And deeper than oblivion we do bury*
*The incensing relics of -> it*
William Shakespeare, [All’s Well That Ends Well +> it*
William Shakespeare, [All’s Well That Ends Well, > V.3](https://internetshakespeare.uvic.ca/doc/AWW_M/scene/5.3/index.html) The `relic` package provides tools for working with version-controlled diff --git a/_pkgdown.yml b/_pkgdown.yml new file mode 100644 index 0000000..d71acfb --- /dev/null +++ b/_pkgdown.yml @@ -0,0 +1,4 @@ +url: ~ +template: + bootstrap: 5 + diff --git a/inst/WORDLIST b/inst/WORDLIST index 86c5eb8..58e7392 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -1,9 +1,21 @@ CMD -ORCID -SHA codecov +csv +dir +doltr +EcoHealth +funder +globbing https +LFS +Lifecycle +MinIO +ORCID pkgcheck +recurse retarting revwalk scm +SHA +submodule +symlink diff --git a/inst/example-repo/_targets.R b/inst/example-repo/_targets.R new file mode 100644 index 0000000..324fb68 --- /dev/null +++ b/inst/example-repo/_targets.R @@ -0,0 +1,16 @@ +library(targets) +list( + tar_target(cars, mtcars), + tar_target(cars_csv, { + filename <- "cars.csv" + write.csv(cars, filename) + filename + }, format = "file"), + tar_target(cars_plot, { + filename <- "cars.png" + png(filename) + plot(cars) + dev.off() + filename + }, format = "file") +) diff --git a/inst/paper.bib b/inst/paper/paper.bib similarity index 100% rename from inst/paper.bib rename to inst/paper/paper.bib diff --git a/inst/paper/paper.md b/inst/paper/paper.md new file mode 100644 index 0000000..e69de29 diff --git a/man/commits_between.Rd b/man/commits_between.Rd index bf60b8f..3250d67 100644 --- a/man/commits_between.Rd +++ b/man/commits_between.Rd @@ -9,7 +9,8 @@ commits_between(from, to = NULL, filter_file = NULL, repo = ".") \arguments{ \item{from}{A commit object or revision string. A string of the form 'from...to' may also be passed, in which case the commits between the two -revision strings are used and the \code{to} argument is ignored.} +revision strings are used and the \code{to} argument is ignored. If a list of commits +is passed, these commits are used rather than calculating the commits between.} \item{to}{A commit object or reference} diff --git a/man/create_example_repo.Rd b/man/create_example_repo.Rd new file mode 100644 index 0000000..4f74a21 --- /dev/null +++ b/man/create_example_repo.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/example_repo.R +\name{create_example_repo} +\alias{create_example_repo} +\title{Set up a testing and example repository} +\usage{ +create_example_repo( + dir = fs::file_temp("relic_example_"), + reporter = "silent", + s3 = TRUE +) +} +\arguments{ +\item{dir}{Path to the directory where the repository should be created} + +\item{reporter}{the reporter to use when building targets with \code{\link[targets:tar_make]{targets::tar_make()}}. Defaults to "silent".} + +\item{s3}{Whether the repository should use S3 storage for targets. Note that +the S3 endpoint and bucket must already be available.} +} +\description{ +This function generates a repository with a commit history that can be used +for testing and examples. +} diff --git a/man/dir_ls_version.Rd b/man/dir_ls_version.Rd index 9ef7063..0002078 100644 --- a/man/dir_ls_version.Rd +++ b/man/dir_ls_version.Rd @@ -13,6 +13,7 @@ dir_ls_version( type = "any", regexp = NULL, glob = NULL, + invert = FALSE, repo = "." ) @@ -35,25 +36,26 @@ relative to the git directory} \item{ref}{A git commit SHA, tag, branch or other \href{https://git-scm.com/docs/git-rev-parse.html#_specifying_revisions}{revision string}: such as "HEAD~1", or a \code{\link[git2r:commit]{git2r::commit()}} object. Defaults to "HEAD".} +\item{all}{If TRUE hidden files are also returned} + \item{recurse}{If TRUE recurse fully, if a positive number the number of levels to recurse} \item{type}{One or more of "any", "file", "directory", "symlink", or "submodule"} -\item{regexp}{A regular expression (e.g. ⁠\link{.}csv$⁠) passed on to grep() to filter paths.} +\item{regexp}{A regular expression (e.g. '\\.csv$'⁠) passed on to grep() to filter paths.} + +\item{glob}{A wildcard aka globbing pattern (e.g. '*.csv'⁠) passed on to grep() to filter paths} -\item{glob}{A wildcard aka globbing pattern (e.g. ⁠*.csv⁠) passed on to grep() to filter paths} +\item{invert}{If TRUE, return paths that do not match the pattern or glob} \item{repo}{The path to the git repository or a \code{\link[git2r:repository]{git2r::repository()}} object.} \item{from}{A commit object or revision string. A string of the form 'from...to' may also be passed, in which case the commits between the two -revision strings are used and the \code{to} argument is ignored.} +revision strings are used and the \code{to} argument is ignored. If a list of commits +is passed, these commits are used rather than calculating the commits between.} \item{to}{A commit object or reference} - -\item{param}{all If TRUE hidden files are also returned} - -\item{invert}{If TRUE, return paths that do not match the pattern or glob} } \value{ A character vector of paths diff --git a/man/file_copy_version.Rd b/man/file_copy_version.Rd index 0941a7a..92efce6 100644 --- a/man/file_copy_version.Rd +++ b/man/file_copy_version.Rd @@ -12,7 +12,8 @@ file_copy_version( dir = NULL, full_name = NULL, recurse = TRUE, - repo = "." + repo = ".", + use_cache = TRUE ) file_copy_versions( @@ -21,8 +22,10 @@ file_copy_versions( to = NULL, name = NULL, dir = NULL, - file_filter = NULL, - repo = "." + full_name = NULL, + recurse = TRUE, + repo = ".", + use_cache = TRUE ) } \arguments{ @@ -48,9 +51,12 @@ numeric, how many levels deep should it be copied? Defaults to TRUE.} \item{repo}{The path to the git repository or a \code{\link[git2r:repository]{git2r::repository()}} object.} +\item{use_cache}{Should the cache be used? Defaults to TRUE.} + \item{from}{A commit object or revision string. A string of the form 'from...to' may also be passed, in which case the commits between the two -revision strings are used and the \code{to} argument is ignored.} +revision strings are used and the \code{to} argument is ignored. If a list of commits +is passed, these commits are used rather than calculating the commits between.} \item{to}{A commit object or reference} } @@ -64,3 +70,9 @@ Where files do not exist in a commit, \code{NA} values are returned. \description{ Copy files from git history to disk. Directories are copied recursively. } +\details{ +The default behavior is to copy the file to a cache directory named after +both the commit and and the file path. These paths are universally unique, +so if \code{use_cache = TRUE}, the file will not be re-copied if it already exists +at that path. +} diff --git a/man/file_read_version.Rd b/man/file_read_version.Rd index 5659888..a91c3df 100644 --- a/man/file_read_version.Rd +++ b/man/file_read_version.Rd @@ -20,7 +20,8 @@ such as "HEAD~1", or a \code{\link[git2r:commit]{git2r::commit()}} object. Defau \item{from}{A commit object or revision string. A string of the form 'from...to' may also be passed, in which case the commits between the two -revision strings are used and the \code{to} argument is ignored.} +revision strings are used and the \code{to} argument is ignored. If a list of commits +is passed, these commits are used rather than calculating the commits between.} \item{to}{A commit object or reference} } diff --git a/man/is_relic.Rd b/man/is_relic.Rd new file mode 100644 index 0000000..707f9cd --- /dev/null +++ b/man/is_relic.Rd @@ -0,0 +1,14 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/relic.R +\name{is_relic} +\alias{is_relic} +\title{An internal S3 class of git objects with additional metadata} +\usage{ +is_relic(x) +} +\arguments{ +\item{x}{an object} +} +\description{ +An internal S3 class of git objects with additional metadata +} diff --git a/man/relic-package.Rd b/man/relic-package.Rd new file mode 100644 index 0000000..9861f9d --- /dev/null +++ b/man/relic-package.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/relic-package.R +\docType{package} +\name{relic-package} +\alias{relic} +\alias{relic-package} +\title{Objects from a while back} +\description{ +\if{html}{\figure{logo.png}{options: style='float: right' alt='logo' width='120'}} + +The 'relic' package provides tools for extracting files and objects from the history of a git repository. It is a high-level interface designed to enable comparison of objects in reproducible research workflows, especially pipelines that use the 'targets' package. +} +\seealso{ +Useful links: +\itemize{ + \item \url{https://ecohealthalliance.github.io/relic} + \item \url{https://ecohealthalliance.r-universe.dev/relic} + \item Report bugs at \url{https://github.com/ecohealthalliance/relic/issues} +} + +} +\author{ +\strong{Maintainer}: Noam Ross \email{ross@ecohealthalliance.org} (\href{https://orcid.org/0000-0002-2136-0000}{ORCID}) + +Other contributors: +\itemize{ + \item EcoHealth Alliance [copyright holder, funder] +} + +} +\keyword{internal} diff --git a/man/relic_cache.Rd b/man/relic_cache.Rd new file mode 100644 index 0000000..a115da1 --- /dev/null +++ b/man/relic_cache.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils.R +\name{relic_cache} +\alias{relic_cache} +\alias{relic_cache_clear} +\title{The relic cache directory} +\usage{ +relic_cache() + +relic_cache_clear() +} +\value{ +The path to the relic cache directory +} +\description{ +Get the relic cache directory, which can be specified as either the R option +\code{relic.cache.dir} or the environment variable \code{RELIC_CACHE_DIR}. (The +environment variable has higher priority). If neither +is set, the default is set by tools::R_user_dir("relic", "cache"). +} +\examples{ +relic_cache() +} diff --git a/man/tar_exists_version.Rd b/man/tar_exists_version.Rd new file mode 100644 index 0000000..8348e34 --- /dev/null +++ b/man/tar_exists_version.Rd @@ -0,0 +1,38 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/tar_exists_version.R +\name{tar_exists_version} +\alias{tar_exists_version} +\alias{tar_exists_version_raw} +\title{Check if a target exists in a version of the pipeline} +\usage{ +tar_exists_version( + name, + ref = "HEAD", + branches = NULL, + repo = ".", + store = targets::tar_path_store() +) + +tar_exists_version_raw( + name, + ref = "HEAD", + branches = NULL, + repo = ".", + store = targets::tar_path_store() +) +} +\arguments{ +\item{name}{Name of the target. \code{tar_read_version()} can take a symbol, \code{tar_read_raw_version()} requires a character.} + +\item{ref}{A git commit SHA, tag, branch or other \href{https://git-scm.com/docs/git-rev-parse.html#_specifying_revisions}{revision string}: +such as "HEAD~1", or a \code{\link[git2r:commit]{git2r::commit()}} object. Defaults to "HEAD".} + +\item{branches}{Integer of indices of the (targets) branches to load if the target is a pattern.} + +\item{repo}{The path to the git repository or a \code{\link[git2r:repository]{git2r::repository()}} object.} + +\item{store}{Path to the targets store within the project. Defaults to the current project's store.} +} +\description{ +Check if a target exists in a version of the pipeline +} diff --git a/man/tar_meta_version.Rd b/man/tar_meta_version.Rd new file mode 100644 index 0000000..70da2a5 --- /dev/null +++ b/man/tar_meta_version.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/tar_meta_version.R +\name{tar_meta_version} +\alias{tar_meta_version} +\title{Read a target project's metadata from git history} +\usage{ +tar_meta_version( + ref = "HEAD", + ..., + store = targets::tar_path_store(), + repo = "." +) +} +\arguments{ +\item{ref}{A git commit SHA, tag, branch or other \href{https://git-scm.com/docs/git-rev-parse.html#_specifying_revisions}{revision string}: +such as "HEAD~1", or a \code{\link[git2r:commit]{git2r::commit()}} object. Defaults to "HEAD".} + +\item{...}{Arguments passed to \code{\link[targets:tar_meta]{targets::tar_meta()}}} + +\item{store}{Path to the targets store within the project. Defaults to the current project's store.} + +\item{repo}{The path to the git repository or a \code{\link[git2r:repository]{git2r::repository()}} object.} +} +\description{ +Read a target project's metadata from git history +} diff --git a/man/tar_read_version.Rd b/man/tar_read_version.Rd new file mode 100644 index 0000000..9ac1da1 --- /dev/null +++ b/man/tar_read_version.Rd @@ -0,0 +1,84 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/tar_read_version.R +\name{tar_read_version} +\alias{tar_read_version} +\alias{tar_read_versions} +\alias{tar_read_raw_version} +\alias{tar_read_raw_versions} +\title{Read a target's value from a git repository} +\usage{ +tar_read_version( + name, + ref = "HEAD", + branches = NULL, + extract_files = TRUE, + repo = ".", + store = targets::tar_path_store() +) + +tar_read_versions( + name, + from = "HEAD", + to = NULL, + branches = NULL, + extract_files = TRUE, + repo = ".", + store = targets::tar_path_store() +) + +tar_read_raw_version( + name, + ref = "HEAD", + branches = NULL, + extract_files = TRUE, + repo = ".", + store = targets::tar_path_store() +) + +tar_read_raw_versions( + name, + from = "HEAD", + to = NULL, + branches = NULL, + extract_files = TRUE, + repo = ".", + store = targets::tar_path_store() +) +} +\arguments{ +\item{name}{Name of the target. \code{tar_read_version()} can take a symbol, \code{tar_read_raw_version()} requires a character.} + +\item{ref}{A git commit SHA, tag, branch or other \href{https://git-scm.com/docs/git-rev-parse.html#_specifying_revisions}{revision string}: +such as "HEAD~1", or a \code{\link[git2r:commit]{git2r::commit()}} object. Defaults to "HEAD".} + +\item{branches}{Integer of indices of the (targets) branches to load if the target is a pattern.} + +\item{extract_files}{If TRUE, targets of type "file" will be extracted to a temporary directory and these paths will be returned. If FALSE, the paths as stored in the target will be returned unmodified} + +\item{repo}{The path to the git repository or a \code{\link[git2r:repository]{git2r::repository()}} object.} + +\item{store}{Path to the targets store within the project. Defaults to the current project's store.} + +\item{from}{A commit object or revision string. A string of the form +'from...to' may also be passed, in which case the commits between the two +revision strings are used and the \code{to} argument is ignored. If a list of commits +is passed, these commits are used rather than calculating the commits between.} + +\item{to}{A commit object or reference} +} +\value{ +The target's return values, loaded files in the git file/⁠, or the paths to the custom files and directories if format = "file" was set. +If the target is not found at the commit, NULL is returned. See \code{\link[=tar_exists_version]{tar_exists_version()}} to check for the presence of a target. +} +\description{ +Reads the content of targets from a git repository. Target metadata and +local target files are extracted into a temporary directory before being +read in by \code{tar_read()}. For targets of type "file", the files are also +extracted and the paths to the extracted files are returned. +} +\details{ +For cloud targets, the target metadata is read from git history and then +this metadata is used to download the target from the cloud. For this to work, +cloud storage must be set up with versioning. Note that the cloud configuration +will use same bucket/endpoint/credentials set in the \emph{current} environment. +} diff --git a/pkgdown/favicon/apple-touch-icon-120x120.png b/pkgdown/favicon/apple-touch-icon-120x120.png new file mode 100644 index 0000000..76810b2 Binary files /dev/null and b/pkgdown/favicon/apple-touch-icon-120x120.png differ diff --git a/pkgdown/favicon/apple-touch-icon-152x152.png b/pkgdown/favicon/apple-touch-icon-152x152.png new file mode 100644 index 0000000..e73cd8b Binary files /dev/null and b/pkgdown/favicon/apple-touch-icon-152x152.png differ diff --git a/pkgdown/favicon/apple-touch-icon-180x180.png b/pkgdown/favicon/apple-touch-icon-180x180.png new file mode 100644 index 0000000..eeeacf4 Binary files /dev/null and b/pkgdown/favicon/apple-touch-icon-180x180.png differ diff --git a/pkgdown/favicon/apple-touch-icon-60x60.png b/pkgdown/favicon/apple-touch-icon-60x60.png new file mode 100644 index 0000000..7e5436c Binary files /dev/null and b/pkgdown/favicon/apple-touch-icon-60x60.png differ diff --git a/pkgdown/favicon/apple-touch-icon-76x76.png b/pkgdown/favicon/apple-touch-icon-76x76.png new file mode 100644 index 0000000..40e8e56 Binary files /dev/null and b/pkgdown/favicon/apple-touch-icon-76x76.png differ diff --git a/pkgdown/favicon/apple-touch-icon.png b/pkgdown/favicon/apple-touch-icon.png new file mode 100644 index 0000000..e38e2de Binary files /dev/null and b/pkgdown/favicon/apple-touch-icon.png differ diff --git a/pkgdown/favicon/favicon-16x16.png b/pkgdown/favicon/favicon-16x16.png new file mode 100644 index 0000000..4a4329e Binary files /dev/null and b/pkgdown/favicon/favicon-16x16.png differ diff --git a/pkgdown/favicon/favicon-32x32.png b/pkgdown/favicon/favicon-32x32.png new file mode 100644 index 0000000..3f12153 Binary files /dev/null and b/pkgdown/favicon/favicon-32x32.png differ diff --git a/pkgdown/favicon/favicon.ico b/pkgdown/favicon/favicon.ico new file mode 100644 index 0000000..b532bbd Binary files /dev/null and b/pkgdown/favicon/favicon.ico differ diff --git a/relic.Rproj b/relic.Rproj index ab7355d..ed19eb2 100644 --- a/relic.Rproj +++ b/relic.Rproj @@ -18,5 +18,6 @@ LineEndingConversion: Posix BuildType: Package PackageUseDevtools: Yes -PackageInstallArgs: --no-multiarch --with-keep.source +PackageInstallArgs: --no-multiarch --with-keep.source --install-tests +PackageCheckArgs: --no-tests PackageRoxygenize: rd,collate,namespace,devtag::dev_roclet diff --git a/tests/testthat/autotest.yaml b/tests/testthat/autotest.yaml deleted file mode 100644 index 772ee78..0000000 --- a/tests/testthat/autotest.yaml +++ /dev/null @@ -1,13 +0,0 @@ -package: -functions: - - : - - preprocess: - - '' - - '' - - '' - - parameters: - - : - - : - - :: - - parameters: - - : diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R new file mode 100644 index 0000000..6fe5c35 --- /dev/null +++ b/tests/testthat/setup.R @@ -0,0 +1,46 @@ +#Sys.setenv("RELIC_TEST_S3"="true") + +withr::local_envvar( + "R_USER_CACHE_DIR" = tempdir()) + +## Run a MinIO server in the background to test S3 object storage with `targets` +if(nzchar(Sys.getenv("RELIC_TEST_S3"))) { + s3_dir <- file_temp("s3_cache") + dir_create(s3_dir) + + mc_dir <- tools::R_user_dir("relic-test", "data") + dir_create(mc_dir) + withr::local_options(list("minioclient.dir"=mc_dir)) + + minioclient::install_mc() + minioclient::install_minio_server() + message("Installed MinIO server and client") + s3_srv <- minioclient::minio_server(dir = s3_dir, process_args = list(stdout = "minio.log", stderr = "2>&1")) + Sys.sleep(2) + stopifnot(s3_srv$is_alive()) + message("Started background MinIO server") + ## Create a bucket for testing + minioclient::mc_alias_set("relic", "localhost:9000", "minioadmin", "minioadmin", scheme = "http") + minioclient::mc_mb("relic/relic-test") + + s3_repo <- create_example_repo(s3 = TRUE) + + if(rlang::is_interactive()) { + s3_srv$kill() + dir_delete(s3_dir) + } else { + withr::defer(s3_srv$kill(), testthat::teardown_env()) + withr::defer(dir_delete(s3_dir), testthat::teardown_env()) + withr::defer(dir_delete(s3_repo), testthat::teardown_env()) + } + +} + +## Create an example repository for testing +ex_repo <- create_example_repo(s3 = FALSE) +withr::defer(dir_delete(ex_repo), testthat::teardown_env()) + + + + + diff --git a/tests/testthat/test-autotest.R b/tests/testthat/test-autotest.R deleted file mode 100644 index 33ee612..0000000 --- a/tests/testthat/test-autotest.R +++ /dev/null @@ -1,9 +0,0 @@ -library(autotest) - -Sys.setenv("AUTOTEST" = "1") -# Skip this test unless environment variable AUTOTEST is set to 1 -test_that("autotest tests pass", { - skip_if(!Sys.getenv("AUTOTEST") == "1", "Skipping autotest, set `AUTOTEST=1` to test") - tests <- autotest::autotest_package(here::here(), test = TRUE) - autotest::expect_autotest_no_err(tests) -}) diff --git a/tests/testthat/test-cache.R b/tests/testthat/test-cache.R new file mode 100644 index 0000000..df41bbb --- /dev/null +++ b/tests/testthat/test-cache.R @@ -0,0 +1,8 @@ +test_that("A relic cache dir set by environent variable overrides one set by options", { + env_dir <- tempfile() + options_dir <- tempfile() + withr::local_envvar(RELIC_CACHE_DIR = env_dir) + withr::local_options(relic.cache.dir = options_dir) + expect_equal(relic_cache(), env_dir) +}) + diff --git a/tests/testthat/test-file_read_version.R b/tests/testthat/test-file_read_version.R new file mode 100644 index 0000000..93e6555 --- /dev/null +++ b/tests/testthat/test-file_read_version.R @@ -0,0 +1,11 @@ +test_that("files are read in properly", { + + withr::local_dir(ex_repo) + + bin_file <- file_read_version("_targets/objects/cars", "first-targets-run", repo = ex_repo)[[1]] + text_file <- file_read_version("_targets.R", "first-targets-run", repo = ex_repo)[[1]] + no_file <- file_read_version("_targets/objects/cars", "initial-target-file", repo = ex_repo)[[1]] + expect_type(bin_file, "raw") + expect_type(text_file, "character") + expect_null(no_file) +}) diff --git a/vignettes/intro-to-relic.Rmd b/vignettes/intro.Rmd similarity index 79% rename from vignettes/intro-to-relic.Rmd rename to vignettes/intro.Rmd index da2c647..b0b422d 100644 --- a/vignettes/intro-to-relic.Rmd +++ b/vignettes/intro.Rmd @@ -1,8 +1,8 @@ --- -title: "intro-to-relic" +title: "intro" output: rmarkdown::html_vignette vignette: > - %\VignetteIndexEntry{intro-to-relic} + %\VignetteIndexEntry{intro} %\VignetteEngine{knitr::rmarkdown} %\VignetteEncoding{UTF-8} --- diff --git a/vignettes/relic-history.Rmd b/vignettes/relic-history.Rmd deleted file mode 100644 index b530f1b..0000000 --- a/vignettes/relic-history.Rmd +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: "Analyzing relic Git History with relic" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{relic-history} - %\VignetteEngine{knitr::rmarkdown} - %\VignetteEncoding{UTF-8} ---- - -```{r, include = FALSE} -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>" -) -``` - -```{r setup} -library(relic) -library(igraph) -``` - - diff --git a/vignettes/targets.Rmd b/vignettes/targets.Rmd index b2cbdb7..e4e6c42 100644 --- a/vignettes/targets.Rmd +++ b/vignettes/targets.Rmd @@ -1,5 +1,5 @@ --- -title: "Using `relic` with `targets`" +title: "targets" output: rmarkdown::html_vignette vignette: > %\VignetteIndexEntry{targets} @@ -17,23 +17,3 @@ knitr::opts_chunk$set( ```{r setup} library(relic) ``` - -`relic` is _most_ useful with `targets` when you are version-controlling your -targets objects and data. This means committing your target objects and files -directly to your git repository, using git LFS to store target objects, and/or -using [versioned cloud storage](https://books.ropensci.org/targets/cloud-storage.html) -to store target objects. For all of these cases, you generally want to commit -your `_targets/meta/meta` file to your git repository. This file contains -information th state of the pipeline. - -Even if you are not version-controlling your targets objects and data, you can -still make use of `relic` to inspect your pipeline history and also re-build -targets from previous versions. This vignette shows you how. - -# Targets and cloud storage - -## Setup - -If you want to follow along with this part of the tutorial, you'll need to install the MinIO server and client programs. These are used to set up a local server that emulates cloud storage. - -