diff --git a/README.md b/README.md index 7dc40a7..27113e3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ `locksmith` is a utility to generate `renv.lock` file containing all dependencies of given set of R packages. -Given the input list of R packages or git repositories containing the R packages, as well as a list of R package repositories (e.g. in a package manager, CRAN, BioConductor etc.), `locksmith` will try to determine the list of all dependencies and their versions required to make the input list of packages work. It will then save the result in an `renv.lock`-compatible file. +Given the input list of git repositories containing the R packages, as well as a list of R package repositories (e.g. in a package manager, CRAN, BioConductor etc.), `locksmith` will try to determine the list of all dependencies and their versions required to make the input list of packages work. It will then save the result in an `renv.lock`-compatible file. ## Installing @@ -27,9 +27,16 @@ locksmith --logLevel debug --exampleParameter 'exampleValue' Real-life example with multiple input packages and repositories. ```bash -locksmith --inputPackageList https://raw.githubusercontent.com/insightsengineering/formatters/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rtables/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda.2022/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/nestcolor/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rlistings/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/citril/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda.test/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/citril.metadata/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/chevron/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/dunlin/main/DESCRIPTION --inputRepositoryList https://bioconductor.org/packages/release/bioc,https://cran.rstudio.com/ +locksmith --inputPackageList https://raw.githubusercontent.com/insightsengineering/formatters/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rtables/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda.2022/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/nestcolor/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rlistings/main/DESCRIPTION --inputRepositoryList BioC=https://bioconductor.org/packages/release/bioc,CRAN=https://cran.rstudio.com/ ``` +In order to download the packages from GitHub or GitLab repositories, please set the environment variables containing the Personal Access Tokens. + +* For GitHub, set the `LOCKSMITH_GITHUBTOKEN` environment variable. +* For GitLab, set the `LOCKSMITH_GITLABTOKEN` environment variable. + +By default `locksmith` will save the resulting output file to `renv.lock`. + ## Configuration file If you'd like to set the above options in a configuration file, by default `locksmith` checks `~/.locksmith`, `~/.locksmith.yaml` and `~/.locksmith.yml` files. diff --git a/cmd/construct.go b/cmd/construct.go index d4fe8d0..0b09943 100644 --- a/cmd/construct.go +++ b/cmd/construct.go @@ -20,9 +20,13 @@ import ( "strings" ) -func constructOutputPackageList(packages []PackageDescription, packagesFiles map[string]PackagesFile, +// ConstructOutputPackageList generates a list of all packages and their dependencies +// which should be included in the output renv.lock file, +// based on the list of package descriptions, and information contained in the PACKAGES files. +func ConstructOutputPackageList(packages []PackageDescription, packagesFiles map[string]PackagesFile, repositoryList []string) []PackageDescription { var outputPackageList []PackageDescription + var fatalErrors string // Add all input packages to output list, as the packages should be downloaded from git repositories. for _, p := range packages { outputPackageList = append(outputPackageList, PackageDescription{ @@ -33,28 +37,32 @@ func constructOutputPackageList(packages []PackageDescription, packagesFiles map } for _, p := range packages { for _, d := range p.Dependencies { - skipDependency := false if d.DependencyType == "Depends" || d.DependencyType == "Imports" || d.DependencyType == "Suggests" || d.DependencyType == "LinkingTo" { - if checkIfSkipDependency("", p.Package, d.DependencyName, + if !CheckIfSkipDependency("", p.Package, d.DependencyName, d.VersionOperator, d.VersionValue, &outputPackageList) { - skipDependency = true - } - if !skipDependency { - log.Info(p.Package, " → ", d.DependencyName) - resolveDependenciesRecursively( + log.Info(p.Package, " → ", d.DependencyName, " (", d.DependencyType, ")") + ResolveDependenciesRecursively( &outputPackageList, d.DependencyName, d.VersionOperator, - d.VersionValue, repositoryList, packagesFiles, 1, + d.VersionValue, repositoryList, packagesFiles, 1, &fatalErrors, ) } } } } + if fatalErrors != "" { + log.Fatal(fatalErrors) + } return outputPackageList } -func resolveDependenciesRecursively(outputList *[]PackageDescription, name string, versionOperator string, - versionValue string, repositoryList []string, packagesFiles map[string]PackagesFile, recursionLevel int) { +// ResolveDependenciesRecursively checks dependencies of the package, and their required versions. +// Checks if the required version is already included in the output package list +// (later used to generate the renv.lock), or if the dependency should be downloaded from a package repository. +// Repeats the process recursively for all dependencies not yet processed. +func ResolveDependenciesRecursively(outputList *[]PackageDescription, name string, versionOperator string, + versionValue string, repositoryList []string, packagesFiles map[string]PackagesFile, recursionLevel int, + fatalErrors *string) { var indentation string for i := 0; i < recursionLevel; i++ { indentation += " " @@ -67,14 +75,14 @@ func resolveDependenciesRecursively(outputList *[]PackageDescription, name strin log.Warn(indentation, name, " not found in top repository.") } // Check if package in the repository is available in sufficient version. - if !checkIfVersionSufficient(p.Version, versionOperator, versionValue) { - // Try to retrieve the package from the next repository. + if !CheckIfVersionSufficient(p.Version, versionOperator, versionValue) { log.Warn( indentation, p.Package, " in repository ", r, " is available in version ", p.Version, " which is insufficient according to requirement ", versionOperator, " ", versionValue, ) + // Try to retrieve the package from the next repository. continue } // Add package to the output list. @@ -87,12 +95,15 @@ func resolveDependenciesRecursively(outputList *[]PackageDescription, name strin for _, d := range p.Dependencies { if d.DependencyType == "Depends" || d.DependencyType == "Imports" || d.DependencyType == "LinkingTo" { - if !checkIfSkipDependency(indentation, p.Package, d.DependencyName, + if !CheckIfSkipDependency(indentation, p.Package, d.DependencyName, d.VersionOperator, d.VersionValue, outputList) { - log.Info(indentation, p.Package, " → ", d.DependencyName) - resolveDependenciesRecursively( - outputList, d.DependencyName, d.VersionOperator, - d.VersionValue, repositoryList, packagesFiles, recursionLevel+1, + log.Info( + indentation, p.Package, " → ", d.DependencyName, + " (", d.DependencyType, ")", + ) + ResolveDependenciesRecursively( + outputList, d.DependencyName, d.VersionOperator, d.VersionValue, + repositoryList, packagesFiles, recursionLevel+1, fatalErrors, ) } } @@ -106,13 +117,13 @@ func resolveDependenciesRecursively(outputList *[]PackageDescription, name strin if versionOperator != "" && versionValue != "" { versionConstraint = " in version " + versionOperator + " " + versionValue } - log.Fatal( - indentation, "Could not find package ", name, versionConstraint, - " in any of the repositories.", - ) + *fatalErrors += "Could not find package " + name + versionConstraint + " in any of the repositories.\n" } -func checkIfBasePackage(name string) bool { +// CheckIfBasePackage checks whether the package should be treated as a base R package +// (included in every R installation) or if it should be treated as a dependency +// to be downloaded from a package repository. +func CheckIfBasePackage(name string) bool { var basePackages = []string{ "base", "compiler", "datasets", "graphics", "grDevices", "grid", "methods", "parallel", "splines", "stats", "stats4", "tcltk", "tools", @@ -121,10 +132,13 @@ func checkIfBasePackage(name string) bool { return stringInSlice(name, basePackages) } -func checkIfSkipDependency(indentation string, packageName string, dependencyName string, +// CheckIfSkipDependency checks if processing of the package (dependency) should be skipped. +// Dependency should be skipped if it is a base R package, or has already been added to output +// package list (later used to generate the renv.lock). +func CheckIfSkipDependency(indentation string, packageName string, dependencyName string, versionOperator string, versionValue string, outputList *[]PackageDescription) bool { - if checkIfBasePackage(dependencyName) { - log.Debug(indentation, "Skipping package ", dependencyName, " as it is a base R package.") + if CheckIfBasePackage(dependencyName) { + log.Trace(indentation, "Skipping package ", dependencyName, " as it is a base R package.") return true } // Go through the list of dependencies added to the output list previously, to check @@ -133,7 +147,7 @@ func checkIfSkipDependency(indentation string, packageName string, dependencyNam for i := 0; i < len(*outputList); i++ { if dependencyName == (*outputList)[i].Package { // Dependency found on the output list. - if checkIfVersionSufficient((*outputList)[i].Version, versionOperator, versionValue) { + if CheckIfVersionSufficient((*outputList)[i].Version, versionOperator, versionValue) { var requirementMessage string if versionOperator != "" && versionValue != "" { requirementMessage = " according to the requirement " + versionOperator + " " + versionValue @@ -141,7 +155,7 @@ func checkIfSkipDependency(indentation string, packageName string, dependencyNam requirementMessage = " since no required version has been specified." } log.Debug( - indentation, "Output list already contains dependency ", dependencyName, " version ", + indentation, "Output list already contains ", dependencyName, " version ", (*outputList)[i].Version, " which is sufficient for ", packageName, requirementMessage, ) @@ -149,8 +163,8 @@ func checkIfSkipDependency(indentation string, packageName string, dependencyNam } log.Warn( indentation, - "Output list already contains dependency ", dependencyName, " version ", - (*outputList)[i].Version, " but it is insufficient as ", packageName, + "Output list already contains ", dependencyName, " but the version ", + (*outputList)[i].Version, " is insufficient as ", packageName, " requires ", dependencyName, " ", versionOperator, " ", versionValue, ) // Overwrite the information about the previous version of the dependency on the output list. @@ -172,7 +186,9 @@ func splitVersion(r rune) bool { return r == '.' || r == '-' } -func checkIfVersionSufficient(availableVersionValue string, versionOperator string, +// CheckIfVersionSufficient checks if availableVersionValue fulfills the requirement +// expressed by versionOperator ('>=' or '>') and requiredVersionValue. +func CheckIfVersionSufficient(availableVersionValue string, versionOperator string, requiredVersionValue string) bool { // Check if there are any version requirements at all. if versionOperator == "" && requiredVersionValue == "" { diff --git a/cmd/construct_test.go b/cmd/construct_test.go index 1e196cb..d5df152 100644 --- a/cmd/construct_test.go +++ b/cmd/construct_test.go @@ -21,78 +21,78 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_checkIfVersionSufficient(t *testing.T) { - assert.True(t, checkIfVersionSufficient("2", ">=", "1")) - assert.True(t, checkIfVersionSufficient("2", ">", "1")) - assert.False(t, checkIfVersionSufficient("1", ">=", "2")) - assert.False(t, checkIfVersionSufficient("1", ">", "2")) - assert.False(t, checkIfVersionSufficient("2", ">", "2")) - assert.True(t, checkIfVersionSufficient("2", ">=", "2")) - assert.True(t, checkIfVersionSufficient("1.2", ">=", "1.2")) - assert.False(t, checkIfVersionSufficient("1.2", ">", "1.2")) - assert.True(t, checkIfVersionSufficient("1.3", ">=", "1.2")) - assert.True(t, checkIfVersionSufficient("1.3", ">", "1.2")) - assert.False(t, checkIfVersionSufficient("1.2", ">=", "1.3")) - assert.False(t, checkIfVersionSufficient("1.2", ">", "1.3")) - assert.False(t, checkIfVersionSufficient("1", ">=", "1.2")) - assert.False(t, checkIfVersionSufficient("1", ">", "1.2")) - assert.True(t, checkIfVersionSufficient("1.2", ">=", "1")) - assert.True(t, checkIfVersionSufficient("1.2", ">", "1")) - assert.False(t, checkIfVersionSufficient("1.2.3", ">=", "1.2.4")) - assert.False(t, checkIfVersionSufficient("1.2.3", ">", "1.2.4")) - assert.True(t, checkIfVersionSufficient("1.2.3", ">=", "1.2.3")) - assert.False(t, checkIfVersionSufficient("1.2.3", ">", "1.2.3")) - assert.True(t, checkIfVersionSufficient("1.2.4", ">=", "1.2.3")) - assert.True(t, checkIfVersionSufficient("1.2.4", ">", "1.2.3")) - assert.False(t, checkIfVersionSufficient("1.2", ">=", "1.2.3")) - assert.False(t, checkIfVersionSufficient("1.2", ">", "1.2.3")) - assert.True(t, checkIfVersionSufficient("1.2.3", ">=", "1.2")) - assert.True(t, checkIfVersionSufficient("1.2.3", ">", "1.2")) - assert.True(t, checkIfVersionSufficient("1.3", ">=", "1.2.3")) - assert.True(t, checkIfVersionSufficient("1.3", ">", "1.2.3")) - assert.False(t, checkIfVersionSufficient("1.2.3", ">=", "1.3")) - assert.False(t, checkIfVersionSufficient("1.2.3", ">", "1.3")) - assert.False(t, checkIfVersionSufficient("1", ">=", "1.2.3")) - assert.False(t, checkIfVersionSufficient("1", ">", "1.2.3")) - assert.True(t, checkIfVersionSufficient("1.2.3", ">=", "1")) - assert.True(t, checkIfVersionSufficient("1.2.3", ">", "1")) - assert.True(t, checkIfVersionSufficient("2", ">=", "1.2.3")) - assert.True(t, checkIfVersionSufficient("2", ">", "1.2.3")) - assert.False(t, checkIfVersionSufficient("1.2.3", ">=", "2")) - assert.False(t, checkIfVersionSufficient("1.2.3", ">", "2")) - assert.False(t, checkIfVersionSufficient("1.2.3.4", ">=", "1.2.3.5")) - assert.False(t, checkIfVersionSufficient("1.2.3.4", ">", "1.2.3.5")) - assert.True(t, checkIfVersionSufficient("1.2.3.5", ">=", "1.2.3.4")) - assert.True(t, checkIfVersionSufficient("1.2.3.5", ">", "1.2.3.4")) - assert.True(t, checkIfVersionSufficient("1.2.3.4", ">=", "1.2.3.4")) - assert.False(t, checkIfVersionSufficient("1.2.3.4", ">", "1.2.3.4")) - assert.False(t, checkIfVersionSufficient("1.2.3", ">=", "1.2.3.4")) - assert.False(t, checkIfVersionSufficient("1.2.3", ">", "1.2.3.4")) - assert.True(t, checkIfVersionSufficient("1.2.3.4", ">=", "1.2.3")) - assert.True(t, checkIfVersionSufficient("1.2.3.4", ">", "1.2.3")) - assert.True(t, checkIfVersionSufficient("1.2.4", ">=", "1.2.3.4")) - assert.True(t, checkIfVersionSufficient("1.2.4", ">", "1.2.3.4")) - assert.False(t, checkIfVersionSufficient("1.2.3.4", ">=", "1.2.4")) - assert.False(t, checkIfVersionSufficient("1.2.3.4", ">", "1.2.4")) - assert.False(t, checkIfVersionSufficient("1.2", ">=", "1.2.3.4")) - assert.False(t, checkIfVersionSufficient("1.2", ">", "1.2.3.4")) - assert.True(t, checkIfVersionSufficient("1.2.3.4", ">=", "1.2")) - assert.True(t, checkIfVersionSufficient("1.2.3.4", ">", "1.2")) - assert.True(t, checkIfVersionSufficient("1.3", ">=", "1.2.3.4")) - assert.True(t, checkIfVersionSufficient("1.3", ">", "1.2.3.4")) - assert.False(t, checkIfVersionSufficient("1.2.3.4", ">=", "1.3")) - assert.False(t, checkIfVersionSufficient("1.2.3.4", ">", "1.3")) - assert.False(t, checkIfVersionSufficient("1", ">=", "1.2.3.4")) - assert.False(t, checkIfVersionSufficient("1", ">", "1.2.3.4")) - assert.True(t, checkIfVersionSufficient("2", ">=", "1.2.3.4")) - assert.True(t, checkIfVersionSufficient("2", ">", "1.2.3.4")) - assert.True(t, checkIfVersionSufficient("1.2.3.4", ">=", "1")) - assert.True(t, checkIfVersionSufficient("1.2.3.4", ">", "1")) - assert.False(t, checkIfVersionSufficient("1.2.3.4", ">=", "2")) - assert.False(t, checkIfVersionSufficient("1.2.3.4", ">", "2")) +func Test_CheckIfVersionSufficient(t *testing.T) { + assert.True(t, CheckIfVersionSufficient("2", ">=", "1")) + assert.True(t, CheckIfVersionSufficient("2", ">", "1")) + assert.False(t, CheckIfVersionSufficient("1", ">=", "2")) + assert.False(t, CheckIfVersionSufficient("1", ">", "2")) + assert.False(t, CheckIfVersionSufficient("2", ">", "2")) + assert.True(t, CheckIfVersionSufficient("2", ">=", "2")) + assert.True(t, CheckIfVersionSufficient("1.2", ">=", "1.2")) + assert.False(t, CheckIfVersionSufficient("1.2", ">", "1.2")) + assert.True(t, CheckIfVersionSufficient("1.3", ">=", "1.2")) + assert.True(t, CheckIfVersionSufficient("1.3", ">", "1.2")) + assert.False(t, CheckIfVersionSufficient("1.2", ">=", "1.3")) + assert.False(t, CheckIfVersionSufficient("1.2", ">", "1.3")) + assert.False(t, CheckIfVersionSufficient("1", ">=", "1.2")) + assert.False(t, CheckIfVersionSufficient("1", ">", "1.2")) + assert.True(t, CheckIfVersionSufficient("1.2", ">=", "1")) + assert.True(t, CheckIfVersionSufficient("1.2", ">", "1")) + assert.False(t, CheckIfVersionSufficient("1.2.3", ">=", "1.2.4")) + assert.False(t, CheckIfVersionSufficient("1.2.3", ">", "1.2.4")) + assert.True(t, CheckIfVersionSufficient("1.2.3", ">=", "1.2.3")) + assert.False(t, CheckIfVersionSufficient("1.2.3", ">", "1.2.3")) + assert.True(t, CheckIfVersionSufficient("1.2.4", ">=", "1.2.3")) + assert.True(t, CheckIfVersionSufficient("1.2.4", ">", "1.2.3")) + assert.False(t, CheckIfVersionSufficient("1.2", ">=", "1.2.3")) + assert.False(t, CheckIfVersionSufficient("1.2", ">", "1.2.3")) + assert.True(t, CheckIfVersionSufficient("1.2.3", ">=", "1.2")) + assert.True(t, CheckIfVersionSufficient("1.2.3", ">", "1.2")) + assert.True(t, CheckIfVersionSufficient("1.3", ">=", "1.2.3")) + assert.True(t, CheckIfVersionSufficient("1.3", ">", "1.2.3")) + assert.False(t, CheckIfVersionSufficient("1.2.3", ">=", "1.3")) + assert.False(t, CheckIfVersionSufficient("1.2.3", ">", "1.3")) + assert.False(t, CheckIfVersionSufficient("1", ">=", "1.2.3")) + assert.False(t, CheckIfVersionSufficient("1", ">", "1.2.3")) + assert.True(t, CheckIfVersionSufficient("1.2.3", ">=", "1")) + assert.True(t, CheckIfVersionSufficient("1.2.3", ">", "1")) + assert.True(t, CheckIfVersionSufficient("2", ">=", "1.2.3")) + assert.True(t, CheckIfVersionSufficient("2", ">", "1.2.3")) + assert.False(t, CheckIfVersionSufficient("1.2.3", ">=", "2")) + assert.False(t, CheckIfVersionSufficient("1.2.3", ">", "2")) + assert.False(t, CheckIfVersionSufficient("1.2.3.4", ">=", "1.2.3.5")) + assert.False(t, CheckIfVersionSufficient("1.2.3.4", ">", "1.2.3.5")) + assert.True(t, CheckIfVersionSufficient("1.2.3.5", ">=", "1.2.3.4")) + assert.True(t, CheckIfVersionSufficient("1.2.3.5", ">", "1.2.3.4")) + assert.True(t, CheckIfVersionSufficient("1.2.3.4", ">=", "1.2.3.4")) + assert.False(t, CheckIfVersionSufficient("1.2.3.4", ">", "1.2.3.4")) + assert.False(t, CheckIfVersionSufficient("1.2.3", ">=", "1.2.3.4")) + assert.False(t, CheckIfVersionSufficient("1.2.3", ">", "1.2.3.4")) + assert.True(t, CheckIfVersionSufficient("1.2.3.4", ">=", "1.2.3")) + assert.True(t, CheckIfVersionSufficient("1.2.3.4", ">", "1.2.3")) + assert.True(t, CheckIfVersionSufficient("1.2.4", ">=", "1.2.3.4")) + assert.True(t, CheckIfVersionSufficient("1.2.4", ">", "1.2.3.4")) + assert.False(t, CheckIfVersionSufficient("1.2.3.4", ">=", "1.2.4")) + assert.False(t, CheckIfVersionSufficient("1.2.3.4", ">", "1.2.4")) + assert.False(t, CheckIfVersionSufficient("1.2", ">=", "1.2.3.4")) + assert.False(t, CheckIfVersionSufficient("1.2", ">", "1.2.3.4")) + assert.True(t, CheckIfVersionSufficient("1.2.3.4", ">=", "1.2")) + assert.True(t, CheckIfVersionSufficient("1.2.3.4", ">", "1.2")) + assert.True(t, CheckIfVersionSufficient("1.3", ">=", "1.2.3.4")) + assert.True(t, CheckIfVersionSufficient("1.3", ">", "1.2.3.4")) + assert.False(t, CheckIfVersionSufficient("1.2.3.4", ">=", "1.3")) + assert.False(t, CheckIfVersionSufficient("1.2.3.4", ">", "1.3")) + assert.False(t, CheckIfVersionSufficient("1", ">=", "1.2.3.4")) + assert.False(t, CheckIfVersionSufficient("1", ">", "1.2.3.4")) + assert.True(t, CheckIfVersionSufficient("2", ">=", "1.2.3.4")) + assert.True(t, CheckIfVersionSufficient("2", ">", "1.2.3.4")) + assert.True(t, CheckIfVersionSufficient("1.2.3.4", ">=", "1")) + assert.True(t, CheckIfVersionSufficient("1.2.3.4", ">", "1")) + assert.False(t, CheckIfVersionSufficient("1.2.3.4", ">=", "2")) + assert.False(t, CheckIfVersionSufficient("1.2.3.4", ">", "2")) } -func Test_constructOutputPackageList(t *testing.T) { +func Test_ConstructOutputPackageList(t *testing.T) { var repositoryList = []string{ "https://repo1.example.com/ExampleRepo1", "https://repo2.example.com/ExampleRepo2", @@ -287,7 +287,7 @@ func Test_constructOutputPackageList(t *testing.T) { }, }, } - outputPackageList := constructOutputPackageList( + outputPackageList := ConstructOutputPackageList( []PackageDescription{ { "package1", diff --git a/cmd/download.go b/cmd/download.go index 1dbd2f2..c98e6db 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -44,9 +44,9 @@ type GitHubObject struct { Sha string `json:"sha"` } -// Returns HTTP status code for downloaded file, number of bytes in downloaded content, -// and the downloaded content itself. -func downloadTextFile(url string, parameters map[string]string) (int, int64, string) { // #nosec G402 +// DownloadTextFile returns HTTP status code for downloaded file, number of bytes +// in downloaded content, and the downloaded content itself as a string. +func DownloadTextFile(url string, parameters map[string]string) (int, int64, string) { // #nosec G402 tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} client := &http.Client{Transport: tr} req, err := http.NewRequest("GET", url, nil) @@ -69,12 +69,13 @@ func downloadTextFile(url string, parameters map[string]string) (int, int64, str return -1, 0, "" } -// Retrieve information about GitLab repository (project path, repository name and commit SHA) from -// projectURL repository GitLab API endpoint. -func getGitLabProjectAndSha(projectURL string, remoteRef string, token map[string]string, +// GetGitLabProjectAndSha retrieves information about GitLab repository +// (project path, repository name and commit SHA) +// from projectURL GitLab API endpoint. +func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[string]string, downloadFileFunction func(string, map[string]string) (int, int64, string)) (string, string, string) { var remoteUsername, remoteRepo, remoteSha string - log.Debug("Downloading project information from ", projectURL) + log.Trace("Downloading data for GitLab project from ", projectURL) statusCode, _, projectDataResponse := downloadFileFunction(projectURL, token) if statusCode == 200 { var projectData GitLabAPIResponse @@ -85,84 +86,74 @@ func getGitLabProjectAndSha(projectURL string, remoteRef string, token map[strin remoteUsername = strings.Join(projectPath[:projectPathLength-1], "/") remoteRepo = projectPath[projectPathLength-1] } else { - log.Warn("An error occurred while retrieving project information from ", projectURL) + log.Warn("An error occurred while retrieving project data from ", projectURL) } match, errMatch := regexp.MatchString(`v\d+(\.\d+)*`, remoteRef) + var urlPath string if match { - log.Debug("remoteRef = ", remoteRef, " matches tag name regexp.") - tagURL := projectURL + "/repository/tags/" + remoteRef - statusCode, _, tagDataResponse := downloadFileFunction(tagURL, token) - if statusCode == 200 { - var tagData GitLabTagOrBranchResponse - err := json.Unmarshal([]byte(tagDataResponse), &tagData) - checkError(err) - remoteSha = tagData.Commit.ID - } else { - log.Warn("An error occurred while retrieving tag information from ", tagURL) - } + log.Trace("remoteRef = ", remoteRef, " matches tag name regexp.") + urlPath = "tags" } else { - log.Debug("remoteRef = ", remoteRef, " doesn't match tag name regexp.") - branchURL := projectURL + "/repository/branches/" + remoteRef - statusCode, _, branchDataResponse := downloadFileFunction(branchURL, token) - if statusCode == 200 { - var branchData GitLabTagOrBranchResponse - err := json.Unmarshal([]byte(branchDataResponse), &branchData) - checkError(err) - remoteSha = branchData.Commit.ID - } else { - log.Warn("An error occurred while retrieving branch information from ", branchURL) - } + log.Trace("remoteRef = ", remoteRef, " doesn't match tag name regexp.") + urlPath = "branches" + } + tagOrBranchURL := projectURL + "/repository/" + urlPath + "/" + remoteRef + statusCode, _, tagOrBranchDataResponse := downloadFileFunction(tagOrBranchURL, token) + if statusCode == 200 { + var tagOrBranchData GitLabTagOrBranchResponse + err := json.Unmarshal([]byte(tagOrBranchDataResponse), &tagOrBranchData) + checkError(err) + remoteSha = tagOrBranchData.Commit.ID + } else { + log.Warn("An error occurred while retrieving data from ", tagOrBranchURL) } checkError(errMatch) return remoteUsername, remoteRepo, remoteSha } -// Retrieve SHA of the remoteRef from the 'remoteUsername/remoteRepo' GitHub repository. -func getGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, token map[string]string, +// GetGitHubSha retrieves SHA of the remoteRef from the remoteUsername/remoteRepo GitHub repository. +func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, token map[string]string, downloadFileFunction func(string, map[string]string) (int, int64, string)) string { var remoteSha string - log.Debug("Downloading information for GitHub project ", remoteUsername, "/", remoteRepo) + log.Trace("Downloading data for GitHub project ", remoteUsername, "/", remoteRepo) match, errMatch := regexp.MatchString(`v\d+(\.\d+)*`, remoteRef) + var urlPath string if match { - log.Debug("remoteRef = ", remoteRef, " matches tag name regexp.") - tagURL := "https://api.github.com/repos/" + remoteUsername + "/" + remoteRepo + "/git/ref/tags/" + remoteRef - statusCode, _, tagDataResponse := downloadFileFunction(tagURL, token) - if statusCode == 200 { - var tagData GitHubTagOrBranchResponse - err := json.Unmarshal([]byte(tagDataResponse), &tagData) - checkError(err) - remoteSha = tagData.Object.Sha - } else { - log.Warn("An error occurred while retrieving tag information from ", tagURL) - } + log.Trace("remoteRef = ", remoteRef, " matches tag name regexp.") + urlPath = "tags" + } else { - log.Debug("remoteRef = ", remoteRef, " doesn't match tag name regexp.") - branchURL := "https://api.github.com/repos/" + remoteUsername + "/" + remoteRepo + "/git/ref/heads/" + remoteRef - statusCode, _, branchDataResponse := downloadFileFunction(branchURL, token) - if statusCode == 200 { - var branchData GitHubTagOrBranchResponse - err := json.Unmarshal([]byte(branchDataResponse), &branchData) - checkError(err) - remoteSha = branchData.Object.Sha - } else { - log.Warn("An error occurred while retrieving branch information from ", branchURL) - } + log.Trace("remoteRef = ", remoteRef, " doesn't match tag name regexp.") + urlPath = "heads" + } + tagOrBranchURL := "https://api.github.com/repos/" + remoteUsername + "/" + remoteRepo + "/git/ref/" + urlPath + "/" + remoteRef + statusCode, _, tagDataResponse := downloadFileFunction(tagOrBranchURL, token) + if statusCode == 200 { + var tagOrBranchData GitHubTagOrBranchResponse + err := json.Unmarshal([]byte(tagDataResponse), &tagOrBranchData) + checkError(err) + remoteSha = tagOrBranchData.Object.Sha + } else { + log.Warn("An error occurred while retrieving data from ", tagOrBranchURL) } checkError(errMatch) return remoteSha } -// Get information about packages stored in git repositories. -func processDescriptionURL(descriptionURL string, +// ProcessDescriptionURL gets information about the git repository in which the package is stored +// based on the provided descriptionURL to the package DESCRIPTION file. +func ProcessDescriptionURL(descriptionURL string, downloadFileFunction func(string, map[string]string) (int, int64, string), ) (map[string]string, string, string, string, string, string, string, string, string) { token := make(map[string]string) var remoteType, remoteRef, remoteHost, remoteUsername, remoteRepo string var remoteSubdir, remoteSha, packageSource string if strings.HasPrefix(descriptionURL, "https://raw.githubusercontent.com") { - // Expecting URL in form: + // Expecting GitHub URL in form: // https://raw.githubusercontent.com/////DESCRIPTION - token["Authorization"] = "token " + gitHubToken + if gitHubToken != "" { + token["Authorization"] = "token " + gitHubToken + } remoteType = "github" packageSource = "GitHub" shorterURL := strings.TrimPrefix(descriptionURL, "https://raw.githubusercontent.com/") @@ -170,7 +161,7 @@ func processDescriptionURL(descriptionURL string, remoteUsername = strings.Split(shorterURL, "/")[0] remoteRepo = strings.Split(shorterURL, "/")[1] remoteRef = strings.Split(shorterURL, "/")[2] - remoteSha = getGitHubSha(remoteUsername, remoteRepo, remoteRef, token, downloadFileFunction) + remoteSha = GetGitHubSha(remoteUsername, remoteRepo, remoteRef, token, downloadFileFunction) // Check whether package is stored in a subdirectory of the git repository. for i, j := range strings.Split(shorterURL, "/") { if j == "DESCRIPTION" { @@ -178,11 +169,13 @@ func processDescriptionURL(descriptionURL string, } } } else { - // Expecting URL in form: + // Expecting GitLab URL in form: // https://example.gitlab.com/api/v4/projects//repository/files//DESCRIPTION/raw?ref= // contains '/' encoded as '%2F' re := regexp.MustCompile(`ref=.*$`) - token["Private-Token"] = gitLabToken + if gitLabToken != "" { + token["Private-Token"] = gitLabToken + } remoteType = "gitlab" packageSource = "GitLab" shorterURL := strings.TrimPrefix(descriptionURL, "https://") @@ -195,20 +188,20 @@ func processDescriptionURL(descriptionURL string, descriptionPath := strings.Split(strings.ReplaceAll(descriptionPath, "%2F", "/"), "/") remoteSubdir = strings.Join(descriptionPath[:len(descriptionPath)-1], "/") } - remoteUsername, remoteRepo, remoteSha = getGitLabProjectAndSha(projectURL, remoteRef, token, downloadFileFunction) + remoteUsername, remoteRepo, remoteSha = GetGitLabProjectAndSha(projectURL, remoteRef, token, downloadFileFunction) } return token, remoteType, packageSource, remoteHost, remoteUsername, remoteRepo, remoteSubdir, remoteRef, remoteSha } -// Downloads DESCRIPTION files from the list of supplied URLs. -// Returns a list of structures representing the contents of DESCRIPTION file for the packages, -// and the git repositories storing the packages. -func downloadDescriptionFiles(packageDescriptionList []string, +// DownloadDescriptionFiles downloads DESCRIPTION files from packageDescriptionList. +// It returns a list of structures representing: the contents of DESCRIPTION file +// for the packages and various information about git repositories storing the packages. +func DownloadDescriptionFiles(packageDescriptionList []string, downloadFileFunction func(string, map[string]string) (int, int64, string)) []DescriptionFile { var inputDescriptionFiles []DescriptionFile for _, packageDescriptionURL := range packageDescriptionList { token, remoteType, packageSource, remoteHost, remoteUsername, remoteRepo, remoteSubdir, remoteRef, remoteSha := - processDescriptionURL(packageDescriptionURL, downloadFileFunction) + ProcessDescriptionURL(packageDescriptionURL, downloadFileFunction) log.Info( "Downloading ", packageDescriptionURL, "\nremoteType = ", remoteType, ", remoteUsername = ", remoteUsername, ", remoteRepo = ", remoteRepo, @@ -226,7 +219,9 @@ func downloadDescriptionFiles(packageDescriptionList []string, } else { log.Warn( "An error occurred while downloading ", packageDescriptionURL, - " Please make sure you provided an access token (in LOCKSMITH_GITHUBTOKEN ", + "\nIt may have happened because the git repository is not public ", + "and you didn't set the Personal Access Token.", + "\nPlease make sure you provided an access token (in LOCKSMITH_GITHUBTOKEN ", "or LOCKSMITH_GITLABTOKEN environment variable).", ) } @@ -234,10 +229,10 @@ func downloadDescriptionFiles(packageDescriptionList []string, return inputDescriptionFiles } -// Downloads PACKAGES files from repository URLs specified in the repositoryList. +// DownloadPackagesFiles downloads PACKAGES files from repository URLs specified in the repositoryList. // Returns a map from repository URL to the string with the contents of PACKAGES file // for that repository. -func downloadPackagesFiles(repositoryList []string, +func DownloadPackagesFiles(repositoryList []string, downloadFileFunction func(string, map[string]string) (int, int64, string)) map[string]string { inputPackagesFiles := make(map[string]string) for _, repository := range repositoryList { diff --git a/cmd/download_test.go b/cmd/download_test.go index 341cb9e..3368243 100644 --- a/cmd/download_test.go +++ b/cmd/download_test.go @@ -158,8 +158,8 @@ func mockedDownloadTextFile(url string, _ map[string]string) (int, int64, string return 200, 0, "" } -func Test_downloadDescriptionFiles(t *testing.T) { - descriptionFileList := downloadDescriptionFiles([]string{ +func Test_DownloadDescriptionFiles(t *testing.T) { + descriptionFileList := DownloadDescriptionFiles([]string{ "https://gitlab.example.com/api/v4/projects/37706/repository/files/subdirectory%2FDESCRIPTION/raw?ref=v1.3.1", "https://gitlab.example.com/api/v4/projects/38706/repository/files/subdirectory1%2Fsubdirectory2%2FDESCRIPTION/raw?ref=v1.4.2", "https://gitlab.example.com/api/v4/projects/30176/repository/files/DESCRIPTION/raw?ref=v0.2.0", @@ -285,8 +285,8 @@ func Test_downloadDescriptionFiles(t *testing.T) { }) } -func Test_downloadPackagesFiles(t *testing.T) { - packagesFiles := downloadPackagesFiles([]string{ +func Test_DownloadPackagesFiles(t *testing.T) { + packagesFiles := DownloadPackagesFiles([]string{ "https://repo1.example.com/repo1", "https://repo2.example.com/repo2", "https://repo3.example.com/repo3", diff --git a/cmd/parse.go b/cmd/parse.go index 8c6df74..6f18310 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -22,25 +22,27 @@ import ( yaml "gopkg.in/yaml.v3" ) -func parseDescriptionFileList(inputDescriptionFiles []DescriptionFile) []PackageDescription { +// ParseDescriptionFileList iterates through package DESCRIPTION files. +func ParseDescriptionFileList(inputDescriptionFiles []DescriptionFile) []PackageDescription { var allPackages []PackageDescription for _, descriptionFile := range inputDescriptionFiles { - processDescription(descriptionFile, &allPackages) + ProcessDescription(descriptionFile, &allPackages) } return allPackages } -func parsePackagesFiles(repositoryPackageFiles map[string]string) map[string]PackagesFile { +// ParsePackagesFiles iterates through package repository PACKAGES files. +func ParsePackagesFiles(repositoryPackageFiles map[string]string) map[string]PackagesFile { packagesFilesMap := make(map[string]PackagesFile) for repository, packagesFile := range repositoryPackageFiles { - packagesFilesMap[repository] = processPackagesFile(packagesFile) + packagesFilesMap[repository] = ProcessPackagesFile(packagesFile) } return packagesFilesMap } -// Reads a string containing PACKAGES file, and returns structure with -// fields/properties required for further processing. -func processPackagesFile(content string) PackagesFile { +// ProcessPackagesFile reads a string containing PACKAGES file, and returns a structure +// with those fields/properties that are required for further processing. +func ProcessPackagesFile(content string) PackagesFile { var allPackages PackagesFile for _, lineGroup := range strings.Split(content, "\n\n") { if lineGroup == "" { @@ -49,14 +51,14 @@ func processPackagesFile(content string) PackagesFile { // Each lineGroup contains information about one package and is separated by an empty line. firstLine := strings.Split(lineGroup, "\n")[0] packageName := strings.ReplaceAll(firstLine, "Package: ", "") - cleaned := cleanDescriptionOrPackagesEntry(lineGroup) + cleaned := CleanDescriptionOrPackagesEntry(lineGroup) packageMap := make(map[string]string) err := yaml.Unmarshal([]byte(cleaned), &packageMap) if err != nil { log.Error("Error reading ", packageName, " package data from PACKAGES: ", err) } var packageDependencies []Dependency - processDependencyFields(packageMap, &packageDependencies) + ProcessDependencyFields(packageMap, &packageDependencies) allPackages.Packages = append( allPackages.Packages, PackageDescription{ @@ -68,16 +70,16 @@ func processPackagesFile(content string) PackagesFile { return allPackages } -// Reads a string containing DESCRIPTION file and returns a structure with fields/properties -// required for further processing. -func processDescription(description DescriptionFile, allPackages *[]PackageDescription) { - cleaned := cleanDescriptionOrPackagesEntry(description.Contents) +// ProcessDescription reads a string containing DESCRIPTION file and returns a structure +// with those fields/properties that are required for further processing. +func ProcessDescription(description DescriptionFile, allPackages *[]PackageDescription) { + cleaned := CleanDescriptionOrPackagesEntry(description.Contents) packageMap := make(map[string]string) err := yaml.Unmarshal([]byte(cleaned), &packageMap) checkError(err) var packageDependencies []Dependency - processDependencyFields(packageMap, &packageDependencies) + ProcessDependencyFields(packageMap, &packageDependencies) *allPackages = append( *allPackages, PackageDescription{ @@ -88,10 +90,11 @@ func processDescription(description DescriptionFile, allPackages *[]PackageDescr ) } -// Processes a multiline string representing information about one package from PACKAGES file, or the whole -// contents of DESCRIPTION file. Removes newlines occurring within filtered fields (which are predominantly -// fields containing lists of package dependencies). Also removes fields which are not required for further processing. -func cleanDescriptionOrPackagesEntry(description string) string { +// CleanDescriptionOrPackagesEntry processes a multiline string representing information about one package +// from PACKAGES file, or the whole contents of DESCRIPTION file. Removes newlines occurring within +// filtered fields (which are predominantly fields containing lists of package dependencies). +// Also removes fields which are not required for further processing. +func CleanDescriptionOrPackagesEntry(description string) string { lines := strings.Split(description, "\n") filterFields := []string{"Package:", "Version:", "Depends:", "Imports:", "Suggests:", "LinkingTo:"} outputContent := "" @@ -123,9 +126,9 @@ func splitPackageName(r rune) bool { return r == ' ' || r == '(' } -// Processes a map containing a YAML-like object representing dependencies of a package. +// ProcessDependencyFields processes a map containing a YAML-like object representing dependencies of a package. // Returns a list of Dependency structures corresponding to dependency name, and version constraints. -func processDependencyFields(packageMap map[string]string, +func ProcessDependencyFields(packageMap map[string]string, packageDependencies *[]Dependency) { dependencyFields := []string{"Depends", "Imports", "Suggests", "Enhances", "LinkingTo"} re := regexp.MustCompile(`\(.*\)`) diff --git a/cmd/parse_test.go b/cmd/parse_test.go index 742f7be..37a6323 100644 --- a/cmd/parse_test.go +++ b/cmd/parse_test.go @@ -22,10 +22,10 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_processPackagesFile(t *testing.T) { +func Test_ProcessPackagesFile(t *testing.T) { byteValue, err := os.ReadFile("testdata/PACKAGES") checkError(err) - allPackages := processPackagesFile(string(byteValue)) + allPackages := ProcessPackagesFile(string(byteValue)) assert.Equal(t, allPackages, PackagesFile{ []PackageDescription{ @@ -156,7 +156,7 @@ func Test_processPackagesFile(t *testing.T) { ) } -func Test_parseDescriptionFileList(t *testing.T) { +func Test_ParseDescriptionFileList(t *testing.T) { byteValue1, err := os.ReadFile("testdata/DESCRIPTION1") checkError(err) byteValue2, err := os.ReadFile("testdata/DESCRIPTION2") @@ -165,7 +165,7 @@ func Test_parseDescriptionFileList(t *testing.T) { {string(byteValue1), "GitHub", "", "", "", "", "", "", ""}, {string(byteValue2), "GitHub", "", "", "", "", "", "", ""}, } - allPackages := parseDescriptionFileList(descriptionFileList) + allPackages := ParseDescriptionFileList(descriptionFileList) assert.Equal(t, allPackages, []PackageDescription{ { diff --git a/cmd/renv.go b/cmd/renv.go index cacf27b..4bcd2d6 100644 --- a/cmd/renv.go +++ b/cmd/renv.go @@ -20,7 +20,10 @@ import ( "sort" ) -func generateRenvLock(packageList []PackageDescription, repositoryMap map[string]string) RenvLock { +// GenerateRenvLock generates renv.lock file structure which can be then saved as a JSON file. +// It uses a list of package data created by ConstructOutputPackageList, and the map of +// package repositories containing the packages. +func GenerateRenvLock(packageList []PackageDescription, repositoryMap map[string]string) RenvLock { var outputRenvLock RenvLock outputRenvLock.Packages = make(map[string]PackageDescription) for _, p := range packageList { @@ -30,7 +33,7 @@ func generateRenvLock(packageList []PackageDescription, repositoryMap map[string continue } // Replace package repository URL with package repository alias/name. - repositoryKey := getRepositoryKeyByValue(p.Repository, repositoryMap) + repositoryKey := GetRepositoryKeyByValue(p.Repository, repositoryMap) p.Repository = repositoryKey outputRenvLock.Packages[p.Package] = p } @@ -47,7 +50,9 @@ func generateRenvLock(packageList []PackageDescription, repositoryMap map[string return outputRenvLock } -func getRepositoryKeyByValue(repositoryURL string, repositoryMap map[string]string) string { +// GetRepositoryKeyByValue searches for repository URL in repositoryMap and returns +// the name (alias) of that repository which will then be used in output renv.lock file. +func GetRepositoryKeyByValue(repositoryURL string, repositoryMap map[string]string) string { for k, v := range repositoryMap { if v == repositoryURL { return k diff --git a/cmd/renv_test.go b/cmd/renv_test.go index adcbd08..ffddc3e 100644 --- a/cmd/renv_test.go +++ b/cmd/renv_test.go @@ -21,8 +21,8 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_generateRenvLock(t *testing.T) { - renvLock := generateRenvLock([]PackageDescription{ +func Test_GenerateRenvLock(t *testing.T) { + renvLock := GenerateRenvLock([]PackageDescription{ { "package1", "1.0.2", diff --git a/cmd/root.go b/cmd/root.go index fd3d142..70a2185 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -32,6 +32,7 @@ var inputPackageList string var inputRepositoryList string var gitHubToken string var gitLabToken string +var outputRenvLock string var log = logrus.New() @@ -66,8 +67,8 @@ func newRootCommand() { Use: "locksmith", Short: "renv.lock generator", Long: `locksmith is a utility to generate renv.lock file containing all dependencies -of given set of R packages. Given the input list of R packages or git repositories containing -the R packages, as well as a list of R package repositories (e.g. in a package manager, CRAN, +of given set of R packages. Given the input list of git repositories containing the R packages, +as well as a list of R package repositories (e.g. in a package manager, CRAN, BioConductor etc.), locksmith will try to determine the list of all dependencies and their versions required to make the input list of packages work. It will then save the result in an renv.lock-compatible file.`, @@ -80,15 +81,16 @@ in an renv.lock-compatible file.`, fmt.Println("config =", cfgFile) fmt.Println("inputPackageList =", inputPackageList) fmt.Println("inputRepositoryList =", inputRepositoryList) - - packageDescriptionList, repositoryList, repositoryMap := parseInput() - inputDescriptionFiles := downloadDescriptionFiles(packageDescriptionList, downloadTextFile) - inputPackages := parseDescriptionFileList(inputDescriptionFiles) - repositoryPackagesFiles := downloadPackagesFiles(repositoryList, downloadTextFile) - packagesFiles := parsePackagesFiles(repositoryPackagesFiles) - outputPackageList := constructOutputPackageList(inputPackages, packagesFiles, repositoryList) - renvLock := generateRenvLock(outputPackageList, repositoryMap) - writeJSON("renv.lock", renvLock) + fmt.Println("outputRenvLock = ", outputRenvLock) + + packageDescriptionList, repositoryList, repositoryMap := ParseInput() + inputDescriptionFiles := DownloadDescriptionFiles(packageDescriptionList, DownloadTextFile) + inputPackages := ParseDescriptionFileList(inputDescriptionFiles) + repositoryPackagesFiles := DownloadPackagesFiles(repositoryList, DownloadTextFile) + packagesFiles := ParsePackagesFiles(repositoryPackagesFiles) + outputPackageList := ConstructOutputPackageList(inputPackages, packagesFiles, repositoryList) + renvLock := GenerateRenvLock(outputPackageList, repositoryMap) + writeJSON(outputRenvLock, renvLock) }, } rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", @@ -103,6 +105,8 @@ in an renv.lock-compatible file.`, "Token to download non-public files from GitHub.") rootCmd.PersistentFlags().StringVar(&gitLabToken, "gitLabToken", "", "Token to download non-public files from GitLab.") + rootCmd.PersistentFlags().StringVar(&outputRenvLock, "outputRenvLock", "renv.lock", + "File name to save the output renv.lock file.") // Add version command. rootCmd.AddCommand(extension.NewVersionCobraCmd()) @@ -132,7 +136,8 @@ func initConfig() { viper.SetConfigType("yaml") viper.SetConfigName(".locksmith") } - viper.AutomaticEnv() // read in environment variables that match + // Read in environment variables that match. + viper.AutomaticEnv() // If a config file is found, read it in. if err := viper.ReadInConfig(); err == nil { @@ -156,6 +161,7 @@ func initializeConfig() { "inputRepositoryList", "gitHubToken", "gitLabToken", + "outputRenvLock", } { // If the flag has not been set in newRootCommand() and it has been set in initConfig(). // In other words: if it's not been provided in command line, but has been diff --git a/cmd/structs.go b/cmd/structs.go index a1a845d..01865ee 100644 --- a/cmd/structs.go +++ b/cmd/structs.go @@ -16,17 +16,36 @@ limitations under the License. package cmd +// DescriptionFile represents the input package DESCRIPTION file together with related +// information about the git repository where the package is stored. +// This structure represents data about input git packages before +// it is parsed into a PackageDescription struct. type DescriptionFile struct { + // Contents stores the DESCRIPTION file. Contents string `json:"contents"` - // GitHub or GitLab - PackageSource string `json:"source"` - RemoteType string `json:"remoteType"` - RemoteHost string `json:"remoteHost"` + // PackageSource can be either 'GitHub' or 'GitLab'. + PackageSource string `json:"source"` + // RemoteType can be either 'github' or 'gitlab'. + RemoteType string `json:"remoteType"` + // RemoteHost can be 'api.github.com' or the URL of GitLab instance, + // for example: 'https://gitlab.example.com'. + RemoteHost string `json:"remoteHost"` + // RemoteUsername represents the organization or the owner in case of a GitHub + // repository, or the path to the repository in the project tree in case of + // a GitLab repository. RemoteUsername string `json:"remoteUsername"` - RemoteRepo string `json:"remoteRepo"` - RemoteSubdir string `json:"remoteSubdir"` - RemoteRef string `json:"remoteRef"` - RemoteSha string `json:"remoteSha"` + // RemoteRepo contains the name of git repository. + RemoteRepo string `json:"remoteRepo"` + // RemoteSubdir is an optional field storing the path to the package inside + // the git repository in case the package is not located in the root of the + // git repository. + RemoteSubdir string `json:"remoteSubdir"` + // RemoteRef is tag or branch name representing the verion of the provided + // package DESCRIPTION file. If RemoteRef matches `v\d+(\.\d+)*` regex, + // it is treated as a git tag, otherwise it is treated as a git branch. + RemoteRef string `json:"remoteRef"` + // RemoteSha is the commit SHA for the RemoteRef. + RemoteSha string `json:"remoteSha"` } type PackagesFile struct { @@ -47,19 +66,29 @@ type RenvLockContents struct { Repositories []RenvLockRepository `json:"Repositories"` } +// PackageDescrition represents an R package. type PackageDescription struct { - Package string `json:"Package"` - Version string `json:"Version"` - Source string `json:"Source"` - Repository string `json:"Repository,omitempty"` - Dependencies []Dependency `json:"Requirements"` - RemoteType string `json:"RemoteType,omitempty"` - RemoteHost string `json:"RemoteHost,omitempty"` - RemoteUsername string `json:"RemoteUsername,omitempty"` - RemoteRepo string `json:"RemoteRepo,omitempty"` - RemoteSubdir string `json:"RemoteSubdir,omitempty"` - RemoteRef string `json:"RemoteRef,omitempty"` - RemoteSha string `json:"RemoteSha,omitempty"` + // Package stores the package name. + Package string `json:"Package"` + // Version stores the package version. + Version string `json:"Version"` + // Source can be one of: 'GitHub', 'GitLab' (for packages from git repositories) + // or 'Repository' (for packages from package repositories). + Source string `json:"Source"` + // Repository stores the URL or the name (depending on the stage of processing) + // of the package repository, in case Source is 'Repository'. + Repository string `json:"Repository,omitempty"` + // Dependencies contains the list of package dependencies. + Dependencies []Dependency `json:"Requirements,omitempty"` + // When processing packages stored in package repositories, the fields below are empty. + // These fields are documented in the DescriptionFile struct. + RemoteType string `json:"RemoteType,omitempty"` + RemoteHost string `json:"RemoteHost,omitempty"` + RemoteUsername string `json:"RemoteUsername,omitempty"` + RemoteRepo string `json:"RemoteRepo,omitempty"` + RemoteSubdir string `json:"RemoteSubdir,omitempty"` + RemoteRef string `json:"RemoteRef,omitempty"` + RemoteSha string `json:"RemoteSha,omitempty"` } type Dependency struct { diff --git a/cmd/utils.go b/cmd/utils.go index a347374..2d92c9b 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -43,10 +43,10 @@ func stringInSlice(a string, list []string) bool { return false } -// Parses input parameters, and returns: list of package DESCRIPTION URLs, list of -// package repository URLs (in descending priority order), and a map from package +// ParseInput parses CLI input parameters, and returns: list of package DESCRIPTION URLs, +// list of package repository URLs (in descending priority order), and a map from package // repository alias (name) to the package repository URL. -func parseInput() ([]string, []string, map[string]string) { +func ParseInput() ([]string, []string, map[string]string) { if len(inputPackageList) < 1 { log.Fatal("No packages specified. Please use the --inputPackageList flag.") } diff --git a/cmd/utils_test.go b/cmd/utils_test.go index de2cc59..a43a381 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -21,10 +21,10 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_parseInput(t *testing.T) { +func Test_ParseInput(t *testing.T) { inputPackageList = "https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rlistings/v0.2.6/DESCRIPTION" inputRepositoryList = "Repo1=https://repo1.example.com/repo1,Repo2=https://repo2.example.com/repo2,Repo3=https://repo3.example.com/repo3" - packageList, repositoryList, repositoryMap := parseInput() + packageList, repositoryList, repositoryMap := ParseInput() assert.Equal(t, packageList, []string{ "https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION", "https://raw.githubusercontent.com/insightsengineering/rlistings/v0.2.6/DESCRIPTION",