From 59118d213e9b1fe0f1b4d5cd4f7b609fe12ab8bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Fortkamp?= Date: Wed, 7 Feb 2024 06:21:28 -0300 Subject: [PATCH] Extract function to create custom gitignore file (#404) * Extract function to create custom gitignore file * Update version --- CHANGELOG.md | 268 +++++++++++++++++++++++++++++------------- Project.toml | 16 ++- src/project_setup.jl | 76 ++++++++++-- test/project_tests.jl | 17 ++- 4 files changed, 277 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f15a16f5..e3eeb407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,183 +1,287 @@ +# 2.14.0 + +- Add keyword argument `folders_to_gitignore` to the `initialize_project` to customize the `.gitignore` file creation during project setup + # 2.13.0 + - Add `commmit_message` option in `tag!`, which add an additional `"gitmessage"` field in dictionary `d` and include the git message associated with the commit. # 2.12.6 + - Crucial bugfix to `produce_or_load`. When used with a prefix, it attached double prefix to the file (one coming as a duplicate from `savename`). This is now fixed, but it means that some files produced with prefix and `produce_or_load` in v2.12 may be re-produced after this update. # 2.12.0 + - Arbitrary functions extracting strings from data can be used in `produce_or_load` instead of `savename`. `hash` is the most useful function here. An example in Real World Examples highlights this. - Additional keywords in `produce_or_load` propagated to `savename` are deprecated. - The example project created in the Workflow Tutorial no longer has spaces in its name # 2.11.0 + - Now the default project with `initialize_project` will include a documentation and a test folder, and setup CI for both automatically. - `@produce_or_load` was broken but because CI was disabled this was silently ignored. `produce_or_load` now has call signature `produce_or_load(f::Function, config, path::String = "")` and same signature for the macro with `path` mandatory. # 2.10.0 -* Add `Derived` type and function to allow `dict_list` to compute some parameters from the value of other parameters. -* Add `DRWATSON_UPDATE_MSG` ENV VAR for suppressing update messages + +- Add `Derived` type and function to allow `dict_list` to compute some parameters from the value of other parameters. +- Add `DRWATSON_UPDATE_MSG` ENV VAR for suppressing update messages + # 2.9.0 -* Add `filename` option in `produce_or_load`, possibly overriding the default filename generated by `savename` + +- Add `filename` option in `produce_or_load`, possibly overriding the default filename generated by `savename` + # 2.8.0 -* Add filtering of `collect_results` using `rinclude` and `rexclude` keyword arguments. + +- Add filtering of `collect_results` using `rinclude` and `rexclude` keyword arguments. + # 2.7.2 -* By default `storepatch` keywords are `false`. This means that `gitpatch` is NOT stored by default. This is a BUGFIX, because there is an unknown problem of non-halting when storing the patch. + +- By default `storepatch` keywords are `false`. This means that `gitpatch` is NOT stored by default. This is a BUGFIX, because there is an unknown problem of non-halting when storing the patch. + # 2.7.0 -* DrWatson-related `ENV`ironment variables are now available to globally set the default values for e.g. story git patches, tagging, or safe-saving in various functions like `tagsave` or `produce_or_load`. + +- DrWatson-related `ENV`ironment variables are now available to globally set the default values for e.g. story git patches, tagging, or safe-saving in various functions like `tagsave` or `produce_or_load`. + # 2.6.0 -* Use `JLD2`'s jldopen in `collect_results!` to speed up loading of metadata. + +- Use `JLD2`'s jldopen in `collect_results!` to speed up loading of metadata. + # 2.5.0 -* Add an `update` option of `collect_results!` allowing the updating of an existing results collection if data files were modified or deleted. + +- Add an `update` option of `collect_results!` allowing the updating of an existing results collection if data files were modified or deleted. + # 2.4.1 -* `savename`'s default options now have `sigdigits = 3` instead of `digits = 3` as stated in the documentation string. This was supposed to happen already since 2.0 but did not because of a bug. (#284) -* Any subtypes of `AbstractDict` now work with DrWatson (#283). + +- `savename`'s default options now have `sigdigits = 3` instead of `digits = 3` as stated in the documentation string. This was supposed to happen already since 2.0 but did not because of a bug. (#284) +- Any subtypes of `AbstractDict` now work with DrWatson (#283). # 2.4.0 -* Add the macro version of `produce_or_load` to enable tagging with the calling source file and line. + +- Add the macro version of `produce_or_load` to enable tagging with the calling source file and line. + # 2.3.0 -* Enable pass through of kwargs to `wsave` in `produce_or_load`, `tagsave` and `safesave` (to e.g. allow compression in JLD2 files). + +- Enable pass through of kwargs to `wsave` in `produce_or_load`, `tagsave` and `safesave` (to e.g. allow compression in JLD2 files). + # 2.2.0 -* `isdirty(gitpath = projectdir())` function for checking if `gitpath` points to a dirty Git repository. (#263) + +- `isdirty(gitpath = projectdir())` function for checking if `gitpath` points to a dirty Git repository. (#263) + # 2.1.0 -* `initialize_project` now allows for custom templates. + +- `initialize_project` now allows for custom templates. + # 2.0.4 -* `produce_or_load` now will not attempt to `tagsave` for inappropriate file formats, like `.csv`. + +- `produce_or_load` now will not attempt to `tagsave` for inappropriate file formats, like `.csv`. + # 2.0.3 -* Added a kwarg `equals` to `savename` to extend applicability. + +- Added a kwarg `equals` to `savename` to extend applicability. + # 2.0 + ## Breaking -* DrWatson has moved entirely on using JLD2.jl instead of BSON.jl for saving files, and also suggests the same to its users through the documentation. -* `tmpsave` now saves as JLD2.jl and requires loading explicitly the `"params"` field. -* `collect_results` saves as JLD2.jl by default now. -* `produce_or_load` saves as JLD2.jl by default now. -* `savename` will no longer replace a floating point number with its integer version, if they coincide with respect to rounding. Thus, "integer" `AbstractFloat`s will always end with `.0` in `savename`. -* `savename`'s `scientific` keyword has been replaced by `sigdigits`, as this was the only thing it was doing. -* `savename`'s default options now have `sigdigits = 3` instead of `digits = 3`. + +- DrWatson has moved entirely on using JLD2.jl instead of BSON.jl for saving files, and also suggests the same to its users through the documentation. +- `tmpsave` now saves as JLD2.jl and requires loading explicitly the `"params"` field. +- `collect_results` saves as JLD2.jl by default now. +- `produce_or_load` saves as JLD2.jl by default now. +- `savename` will no longer replace a floating point number with its integer version, if they coincide with respect to rounding. Thus, "integer" `AbstractFloat`s will always end with `.0` in `savename`. +- `savename`'s `scientific` keyword has been replaced by `sigdigits`, as this was the only thing it was doing. +- `savename`'s default options now have `sigdigits = 3` instead of `digits = 3`. + ## Non-breaking -* `savename` now allows a keyword `val_to_string` for customization. + +- `savename` now allows a keyword `val_to_string` for customization. # 1.18.3 -* Remove type constraints on `produce_or_load` path argument (#229) + +- Remove type constraints on `produce_or_load` path argument (#229) + # 1.18.2 -* Keyword `storepatch` now exists in `produce_or_load` as well. + +- Keyword `storepatch` now exists in `produce_or_load` as well. + # 1.18.1 -* Fix `dict_list_count` returns wrong number of elements with `@onlyif`. (#223) -* Fix `gitpatch` for git versions not supporting submodules. (#224) + +- Fix `dict_list_count` returns wrong number of elements with `@onlyif`. (#223) +- Fix `gitpatch` for git versions not supporting submodules. (#224) + # 1.18.0 -* Two new functions `tostringdict, tosymboldict` for changing dictionary key types. + +- Two new functions `tostringdict, tosymboldict` for changing dictionary key types. + # 1.17.0 -* `produce_or_load` now supports passing function as the first argument to support the do-block syntax. + +- `produce_or_load` now supports passing function as the first argument to support the do-block syntax. + # 1.16.6 -* `@onlyif` now doesn't expand vector arguments if it's placed inside another vector. (#209) -* `@onlyif` now supports chaining. See real world examples: "Defining parameter sets with restrictions". (#210) + +- `@onlyif` now doesn't expand vector arguments if it's placed inside another vector. (#209) +- `@onlyif` now supports chaining. See real world examples: "Defining parameter sets with restrictions". (#210) + # 1.16.5 -* The patch information for a dirty repository now also contains the diff for submodules. + +- The patch information for a dirty repository now also contains the diff for submodules. + # 1.16.1 -* `dict_list` now retains the value's type from the passed dictionary. + +- `dict_list` now retains the value's type from the passed dictionary. + # 1.16.0 -* Add a `sort` option to `savename`. + +- Add a `sort` option to `savename`. + # 1.15.0 -* Better default readme file for a new project. + +- Better default readme file for a new project. + # 1.14.6 -* new keyword argument `rpath` for `collect_results!` that allows storing relative paths + +- new keyword argument `rpath` for `collect_results!` that allows storing relative paths + # 1.14.0 -* New macro `@onlyif` that allows placing restrictions on values in a dictionary used for expansion with `dict_list` + +- New macro `@onlyif` that allows placing restrictions on values in a dictionary used for expansion with `dict_list` + # 1.13.1 -* `gitpach` now shows a warning if `git` was not found in PATH + +- `gitpach` now shows a warning if `git` was not found in PATH + # 1.13.0 -* `savename` now includes `TimeType` (dates) in the default allowed types. + +- `savename` now includes `TimeType` (dates) in the default allowed types. + # 1.12.0 -* `wsave/wload` now support keyword arguments. + +- `wsave/wload` now support keyword arguments. + # 1.11.0 -* Macros `@pack!, @unpack` are re-exported by DrWatson. + +- Macros `@pack!, @unpack` are re-exported by DrWatson. # 1.10.0 -* New function `struct2ntuple` that converts a struct to a NamedTuple (for saving) + +- New function `struct2ntuple` that converts a struct to a NamedTuple (for saving) # 1.9.0 -* `savename` now has the `ignore` option. + +- `savename` now has the `ignore` option. # 1.8.0 -* `@quickactivate` was enhanced to allow projects that also represent a module. -* `initialize_project` no resolves the folder name for naming the project if the path is given as "." or ".." + +- `@quickactivate` was enhanced to allow projects that also represent a module. +- `initialize_project` no resolves the folder name for naming the project if the path is given as "." or ".." # 1.7.0 -* Improve the introductory file created by DrWatson. + +- Improve the introductory file created by DrWatson. # 1.6.2 -* `@tag!` and `@tagsave` now support using `;` as keywords separator (#111) + +- `@tag!` and `@tagsave` now support using `;` as keywords separator (#111) # 1.6.0 -* `quickactivate` doesn't do anything anymore if you try to activate to already active project. -* New macro `@quickactivate` + +- `quickactivate` doesn't do anything anymore if you try to activate to already active project. +- New macro `@quickactivate` # 1.5.0 -* Started to add support for overloading save/load for custom files. See the updated docs around `wsave` and `wload`. + +- Started to add support for overloading save/load for custom files. See the updated docs around `wsave` and `wload`. # 1.4.1 -* Fix a bug that created incompatible version strings in generated `Project.toml` files on release candidate versions of Julia. + +- Fix a bug that created incompatible version strings in generated `Project.toml` files on release candidate versions of Julia. # 1.4.0 -* `savename` now supports rounding to significant digits with the keyword argument `scientific`, where `scientific` defines the number of significant digits. + +- `savename` now supports rounding to significant digits with the keyword argument `scientific`, where `scientific` defines the number of significant digits. + # 1.3.0 -* `initialize_project` now adds a Julia version under `compat` in the created `Project.toml` when it is called. -* The functions `tag!, tagsave` and their respective macros now obtain their arguments (besides the first two) as keywords instead of positional arguments. The positional versions are deprecated (#93). -* New keyword `force = false` for `tag!` and co. which replaces the existing `gitcommit` field. + +- `initialize_project` now adds a Julia version under `compat` in the created `Project.toml` when it is called. +- The functions `tag!, tagsave` and their respective macros now obtain their arguments (besides the first two) as keywords instead of positional arguments. The positional versions are deprecated (#93). +- New keyword `force = false` for `tag!` and co. which replaces the existing `gitcommit` field. # 1.2.0 -* Improved behavior of `savename` with respect to nested containers. If a nested container is empty, it is not printed instead. For example, `T=100_p=()_x=2` now becomes `T=100_x=2`. (if `p` is not empty then it is expanded as usual) + +- Improved behavior of `savename` with respect to nested containers. If a nested container is empty, it is not printed instead. For example, `T=100_p=()_x=2` now becomes `T=100_x=2`. (if `p` is not empty then it is expanded as usual) # 1.1.0 -* `initialize_project` no longer makes a test directory. + +- `initialize_project` no longer makes a test directory. # 1.0.1 -* Allow `tag!` and derivatives to handle dictionaries with *key type* `Any`. + +- Allow `tag!` and derivatives to handle dictionaries with _key type_ `Any`. + # 1.0.0 + First major release (no notable change from 0.8.0). # 0.8.0 -* **[BREAKING]** : The `gitpath` argument used among many functions + +- **[BREAKING]** : The `gitpath` argument used among many functions can now also point to a subdirectory within a git repository. Previously it had to be the top directory (i.e. the one containing `.git/`). -* **[BREAKING]** : Slightly changed how `produce_or_load` uses `path` and interacts with `savename`, to better incorporate the changes done in version 0.6.0. `prefix` is now also supported. -* `tag!` and co now also store the git diff patch if the repo is dirty, see `gitpatch` (#80). -* **[BREAKING]** : `tag!` now saves the commit information into a field `gitcommit` instead of just `commit`. +- **[BREAKING]** : Slightly changed how `produce_or_load` uses `path` and interacts with `savename`, to better incorporate the changes done in version 0.6.0. `prefix` is now also supported. +- `tag!` and co now also store the git diff patch if the repo is dirty, see `gitpatch` (#80). +- **[BREAKING]** : `tag!` now saves the commit information into a field `gitcommit` instead of just `commit`. # 0.7.1 -* `projectdir()` now warns if no project (other than the standard one) is + +- `projectdir()` now warns if no project (other than the standard one) is active # 0.7.0 -* New macro `@savename` that is a shortcut for `savename(@dict vars...)` -* New function `gitdescribe` (see below) -* **[DEPRECATED]** `current_commit()` has been deprecated and replaced by + +- New macro `@savename` that is a shortcut for `savename(@dict vars...)` +- New function `gitdescribe` (see below) +- **[DEPRECATED]** `current_commit()` has been deprecated and replaced by `gitdescribe()` which now replaces the output of `git describe` if an annotated tag exists, otherwise it will return the latest commit hash. # 0.6.0 -* **[BREAKING]** Reworked the way the functions `projectdir` and derivatives work (#47, #64, #66). Now `projectdir(args...)` uses `joinpath` to connect arguments. None of the functions like `projectdir` and derivatives now end in `/` as well, to ensure more stability and motivate users to use `joinpath` or the new functionality of `projectdir(args...)` instead of using string multiplication `*`. -* New function `parse_savename` that attempts to reverse engineer the result of `savename`. + +- **[BREAKING]** Reworked the way the functions `projectdir` and derivatives work (#47, #64, #66). Now `projectdir(args...)` uses `joinpath` to connect arguments. None of the functions like `projectdir` and derivatives now end in `/` as well, to ensure more stability and motivate users to use `joinpath` or the new functionality of `projectdir(args...)` instead of using string multiplication `*`. +- New function `parse_savename` that attempts to reverse engineer the result of `savename`. # 0.5.1 -* Improvements to `.gitignore` (#55 , #54) + +- Improvements to `.gitignore` (#55 , #54) + # 0.5.0 + This release has **breaking changes**. -* Adjusted return value of `produce_or_load` (#52). It now always return the file and the path it is saved. If `loadfile = false` it returns `nothing, path`. -* The functionality of `default_prefix` has been modified (#51). Now there is a nice interplay between defining a `default_prefix` *and* passing a prefix to `savename`. They are merged like `joinpath(prefix, default_prefix)`. This is valid only when `default_prefix` has a value other than `""` (the default). + +- Adjusted return value of `produce_or_load` (#52). It now always return the file and the path it is saved. If `loadfile = false` it returns `nothing, path`. +- The functionality of `default_prefix` has been modified (#51). Now there is a nice interplay between defining a `default_prefix` _and_ passing a prefix to `savename`. They are merged like `joinpath(prefix, default_prefix)`. This is valid only when `default_prefix` has a value other than `""` (the default). + # 0.4.0 -* Add expand functionality to `savename`, which handles better containers with nested containers (#50) -* `produce_or_load` now allows the possibility of not loading the file -* New function `struct2dict` that converts a struct to a dictionary (for saving) + +- Add expand functionality to `savename`, which handles better containers with nested containers (#50) +- `produce_or_load` now allows the possibility of not loading the file +- New function `struct2dict` that converts a struct to a dictionary (for saving) + # 0.3.0 -* Added `test` as a directory of the default project (#43) -* Added `tmpsave` functionality: save the result of `dict_list` in temporary files and conveniently work with sequential clusters (#45) -* Now all saving related functions of DrWatson first `mkpath` of the path to save at and then save (#45) + +- Added `test` as a directory of the default project (#43) +- Added `tmpsave` functionality: save the result of `dict_list` in temporary files and conveniently work with sequential clusters (#45) +- Now all saving related functions of DrWatson first `mkpath` of the path to save at and then save (#45) + # 0.2.1 -* Improve type-stability of return value of `dict_list` (#41) + +- Improve type-stability of return value of `dict_list` (#41) + # 0.2.0 -* Changed `path` and `projectpath` arguments of various functions (e.g. `tagsave`, `current_commit`) to `gitpath` universally. -* make keyword arguments of `tagsave` positional arguments instead (to work with the macros) -* Added two new macros: `@tag!` and `@tagsave`: these do the same thing as `tag!, tagsave` but in addition are able to record both the script name that called them as well as the line of code that they were called at. + +- Changed `path` and `projectpath` arguments of various functions (e.g. `tagsave`, `current_commit`) to `gitpath` universally. +- make keyword arguments of `tagsave` positional arguments instead (to work with the macros) +- Added two new macros: `@tag!` and `@tagsave`: these do the same thing as `tag!, tagsave` but in addition are able to record both the script name that called them as well as the line of code that they were called at. + # 0.1.0 + This is the first beta release! Changelog is kept with respect to here! diff --git a/Project.toml b/Project.toml index b79ab7b3..c8b64655 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "DrWatson" uuid = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1" repo = "https://github.com/JuliaDynamics/DrWatson.jl.git" -version = "2.13.0" +version = "2.14.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" @@ -38,4 +38,16 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "BSON", "FileIO", "Parameters", "DataFrames", "JLD2", "Statistics", "Dates", "CSVFiles", "CodecZlib", "DataStructures"] +test = [ + "Test", + "BSON", + "FileIO", + "Parameters", + "DataFrames", + "JLD2", + "Statistics", + "Dates", + "CSVFiles", + "CodecZlib", + "DataStructures", +] diff --git a/src/project_setup.jl b/src/project_setup.jl index e920bb18..59e8477d 100644 --- a/src/project_setup.jl +++ b/src/project_setup.jl @@ -269,18 +269,24 @@ The new project remains activated for you to immediately add packages. * `placeholder = false` : Add "hidden" placeholder files in each default folder to ensure that project folder structure is maintained when the directory is cloned (because empty folders are not pushed to a remote). Only used when `git = true`. -""" -function initialize_project(path, name = default_name_from_path(path); - force = false, readme = true, authors = nothing, - git = true, placeholder = false, template = DEFAULT_TEMPLATE, - add_test = true, add_docs = false, - github_name = "PutYourGitHubNameHere" - ) - if git == false; placeholder = false; end - if add_docs == true; add_test = true; end +* `folders_to_gitignore = ["data", "videos","plots","notebooks","_research"]` : Folders to include in the created .gitignore + """ +function initialize_project(path, name=default_name_from_path(path); + force=false, readme=true, authors=nothing, + git=true, placeholder=false, template=DEFAULT_TEMPLATE, + add_test=true, add_docs=false, + github_name="PutYourGitHubNameHere", + folders_to_gitignore=["data", "videos", "plots", "notebooks", "_research"] +) + if git == false + placeholder = false + end + if add_docs == true + add_test = true + end if add_docs == true && github_name == "PutYourGitHubNameHere" - @warn "Docs will be generated but `github_name` is not set. "* - "You'd need to manually change paths to GitHub in `make.jl`." + @warn "Docs will be generated but `github_name` is not set. " * + "You'd need to manually change paths to GitHub in `make.jl`." end # Set up and potentially clean path mkpath(path) @@ -346,8 +352,52 @@ function initialize_project(path, name = default_name_from_path(path); # chmod is needed, as the file permissions are not # set correctly when adding the package with `add`. # First, add all default files - cp(defaultdir("gitignore.txt"), pathdir(".gitignore")) - chmod(pathdir(".gitignore"), 0o644) + + function create_gitignore(gitignore_path, folders_to_ignore, template_path) + + output_lines = [] + line_index = 1 + in_section = false + + input_lines = readlines(template_path) + + start_line = 1 + end_line = n_lines = length(input_lines) + for line in input_lines + + if startswith(line, "# Folders to ignore") + in_section = true + + start_line = line_index + end + + if (in_section && length(line) == 0) + end_line = line_index + break + end + + line_index += 1 + end + + append!(output_lines, input_lines[1:start_line]) + + for p in folders_to_ignore + push!(output_lines, "/" * p) + end + + append!(output_lines, input_lines[end_line:n_lines]) + + open(gitignore_path, "w") do f + for l in output_lines + write(f, l * "\n") + end + end + chmod(gitignore_path, 0o644) + end + + create_gitignore(pathdir(".gitignore"), + folders_to_gitignore, defaultdir("gitignore.txt")) + cp(defaultdir("gitattributes.txt"), pathdir(".gitattributes")) chmod(pathdir(".gitattributes"), 0o644) diff --git a/test/project_tests.jl b/test/project_tests.jl index 49bcab54..44e1dea2 100644 --- a/test/project_tests.jl +++ b/test/project_tests.jl @@ -86,12 +86,23 @@ cd() # Test templates t1 = ["data", "documents" => ["a", "b"]] -initialize_project(path, name; force = true, git = false, template = t1) +initialize_project(path, name; force=true, git=false, template=t1) @test ispath(joinpath(path, "data")) @test ispath(joinpath(path, "documents")) @test ispath(joinpath(path, "documents", "a")) @test !ispath(joinpath(path, "src")) -rm(joinpath(@__DIR__, path); recursive = true, force = true) -@test !isdir(joinpath(@__DIR__, path)) \ No newline at end of file +# Test .gitignore templates + +initialize_project(path, name; force=true, git=true, + folders_to_gitignore=["videos"]) +gitignore_path = joinpath(path, ".gitignore") +@test ispath(gitignore_path) +gitignore_lines = readlines(gitignore_path) +@test "/videos" ∈ gitignore_lines +@test "/notebooks" ∉ gitignore_lines + + +rm(joinpath(@__DIR__, path); recursive=true, force=true) +@test !isdir(joinpath(@__DIR__, path))