Add support for .kafeignore

libkafe · Aug 6, 2020 · a799e88 · a799e88
1 parent 4059c87
commit a799e88
Show file tree

Hide file tree

Showing 7 changed files with 144 additions and 62 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,4 +1,7 @@
-* Wed Jun 25 2020 Matiss Treinis <mrtreinis@gmail.com> - 1.1.1
+* Thu Aug 6 2020 Matiss Treinis <mrtreinis@gmail.com> - 1.1.2
+- Add support for .kafeignore files.
+
+* Thu Jun 25 2020 Matiss Treinis <mrtreinis@gmail.com> - 1.1.1
 - Update build configuration, define SOVERSION. No functional changes.
 
 * Wed Mar 11 2020 Matiss Treinis <mrtreinis@gmail.com> - 1.1.0

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.11.4)
 
 # IMPORTANT: updating version might require update in package dependencies at the end of this file.
-set(KAFE_VERSION "1.1.1")
+set(KAFE_VERSION "1.1.2")
 set(KAFE_SOVERSION "1.1")
 set(KAFE_VERSION_INT 11)
 set(KAFE_VERSION_DEP_NEXT_MAJOR "2.0.0")
@@ -77,4 +77,4 @@ set(CPACK_DEBIAN_LIBKAFE_PACKAGE_DEPENDS "libstdc++6, libc6, libgcc1, libc6, lib
 set(CPACK_DEBIAN_LIBKAFE-DEV_PACKAGE_DEPENDS "libkafe (=${KAFE_VERSION})")
 
 set(CPACK_ARCHIVE_COMPONENT_INSTALL ON)
-include(CPack)
+include(CPack)
diff --git a/docs/SCRIPTING_API_L1.md b/docs/SCRIPTING_API_L1.md
@@ -7,7 +7,7 @@ This documentation assumes virtual module `kafe` is included as variable `k`, as
 local k = require('kafe')
 ```
 
-For future compatibility, all method calls are prefixed with this variable because more modules might be 
+For future compatibility, all method calls are prefixed with this variable because more modules might be
 introduced at some point.
 
 ### Before using the scripting API
@@ -19,20 +19,20 @@ The scripting API does not interact with remote privilege escalation tools (`su`
 escalate privileges as needed by using the native tools provided by the remote system. Remote prompts are not
 supported at this time. One way to deal with privilege escalation is to use `sudo` command with
 preconfigured sudo lines in `sudoers` configuration. You should only add the commands you intend to execute
-to `sudoers` and never allow wildcard sudo commands to be executed without password prompt 
+to `sudoers` and never allow wildcard sudo commands to be executed without password prompt
 (e.g. `ALL=(ALL:ALL) NOPASSWD: ALL`). Instead, try to design your remote automation as lean as root-less as possible.
 
 All command invocation is strictly sequential - there is no parallelization and parallel invocation of the
-scripting API is not supported - even though you can achieve this in Lua. This is a conscious design decision. 
-Parallelization of remote management tasks might seem a good idea at first, but it rarely is - parallelization makes 
+scripting API is not supported - even though you can achieve this in Lua. This is a conscious design decision.
+Parallelization of remote management tasks might seem a good idea at first, but it rarely is - parallelization makes
 remote error handling much more difficult, might leave remote systems in an inconsistent state and requires a
 much larger investment in writing the automation tasks. In remote application deployment, parallelization is often used
 to ensure all remote hosts receive deployments nearly at the same time. Parallelization is rarely a good solution
 to this - you should split your tasks in groups of smaller tasks instead. For example, you can handle near same time deployment
 by splitting up the tasks like this - upload all artifacts to all servers, unpack all artifacts on all servers,
 symlink all the deployment on all servers, reload services on all servers.
 
-SSH connections to remote servers are reused - one remote server with unique combination of host, user and port will 
+SSH connections to remote servers are reused - one remote server with unique combination of host, user and port will
 only have one connection created regardless of environment and role.
 
 ## Defining tasks
@@ -226,9 +226,9 @@ end)
 
 ### (string stdout, string stderr, int exit_code) k.exec(string command [, bool print_output = true])
 
-Execute a remote shell command and return it's outputs along with exit code. 
+Execute a remote shell command and return it's outputs along with exit code.
 
-Second optional argument indicates if the remote output should also be 
+Second optional argument indicates if the remote output should also be
 logged in output of the tool. This option is enabled by default.
 
 ##### An example of usage
@@ -290,9 +290,9 @@ k.task('example_task', function()
 end)
 ```
 
-### void k.archive_dir(string directory, string archive_file)
+### void k.archive_dir(string archive_file, string directory)
 
-Create a `.tar.gz` archive from given *local* directory and get the full path to the archive oncre created.
+Create a `.tar.gz` archive from given *local* directory and get the full path to the archive once created.
 The resulting archive will be created in the path given as second argument on the *local* machine.
 
 Results in hard failure if:
@@ -306,7 +306,7 @@ Results in hard failure if:
 local k = require('kafe')
 
 k.task('example_task', function()
-    k.archive_dir_tmp('/home/example/some_folder', '/home/example/some_archive.tar.gz')
+    k.archive_dir('/home/example/some_archive.tar.gz', '/home/example/some_folder')
 end)
 ```
 
@@ -366,7 +366,7 @@ end)
 
 ### bool k.upload_str(string content, string remote_file)
 
-Upload text as file to remote server in given path. `remote_file` must be valid file. 
+Upload text as file to remote server in given path. `remote_file` must be valid file.
 
 This command returns true if upload succeeded, and false on failure.
 
@@ -506,8 +506,8 @@ print(user) -- prints <username>
 ## (string stdout, int code) k.local_exec(string command [, bool print_output = true])
 
 Execute local shell command and return it's stdout and exit code.
- 
-Second optional argument indicates if the remote output should also be 
+
+Second optional argument indicates if the remote output should also be
 logged in output of the tool. This option is enabled by default.
 ```lua
 local k = require('kafe')
@@ -555,4 +555,33 @@ k.task('example_task', function()
 end)
 ```
 
+## `.kafeignore` support for file archiving
+#### New in version 1.1.2
+
+It is possible to create an exclusion list of files to be excluded from archival operations. This is done by
+creating a `.kafeignore` file at the root of the directory containing pattern list of all files and directories to
+be ignored by the archival operations.
+
+The pattern matching is implemented using GNU fnmatch() with flags `FNM_PATHNAME | FNM_EXTMATCH | FNM_LEADING_DIR`.
+See https://www.gnu.org/software/libc/manual/html_node/Wildcard-Matching.html
+and https://www.man7.org/linux/man-pages/man3/fnmatch.3.html for more information.
+
+**IMPORTANT:** `.kafeignore` patterns are NOT exactly compatible with other ignore formats, such as `.gitignore`,
+    nested `.kafeignore` files are not supported - only the topmost ignore file will be parsed,
+    and `.kafeignore` file itself is NOT automatically ignored, since you might want to preserve it in some cases.
 
+#### Basic pattern examples
+
+```ignorelang
+# Ignore the directory, all subdirectories and files.
+some_dir
+
+# Ignore all subdirectories and files, retain the directory itself.
+some_dir/*
+
+# Ignore anything in subdirectories, but retain directory and immediate descendants.
+some_dir/**/*
+
+# Ignore everything in the directory, except for file or directory named `some_file`
+some_dir/!(some_file)
+```
diff --git a/libkafe/include/kafe/io/archive.hpp b/libkafe/include/kafe/io/archive.hpp
@@ -32,9 +32,10 @@ using namespace std;
 namespace kafe::io {
     class Archive {
     public:
-        static string tmp_archive_from_directory(const string &directory);
+        static string tmp_archive_from_directory(const string &directory, kafe::ILogEventListener *p_listener);
 
-        static void archive_from_directory(const string &archive_path, const string &directory);
+        static void
+        archive_from_directory(const string &archive_path, const string &directory, ILogEventListener *p_listener);
     };
 }
 

diff --git a/libkafe/src/io/archive.cpp b/libkafe/src/io/archive.cpp
@@ -19,6 +19,8 @@
 
 #include <fstream>
 #include <iostream>
+#include <kafe/logging.hpp>
+#include "fnmatch.h"
 
 #include "kafe/io/archive.hpp"
 #include "kafe/io/file_system.hpp"
@@ -31,21 +33,43 @@ using namespace kafe::runtime;
 namespace kafe::io {
     static const int ARCHIVE_FILE_BUFFER_S = 4096;
 
-    string Archive::tmp_archive_from_directory(const string &directory) {
+    vector<string> read_ignore_file(std_fs::path *i_file) {
+        vector<string> patterns = {};
+        std::ifstream file(*i_file);
+        std::string str;
+        while (std::getline(file, str))
+        {
+            if (str.empty()) {
+                continue;
+            }
+
+            if (str[0] == '#') {
+              continue;
+            }
+
+            patterns.push_back(str);
+        }
+        return patterns;
+    }
+
+    string Archive::tmp_archive_from_directory(const string &directory, ILogEventListener *logger) {
         auto *name = tmpnam(nullptr); // TODO replace
         auto upload_name = string(name) + ".tar.gz";
-        archive_from_directory(upload_name, directory);
+        archive_from_directory(upload_name, directory, logger);
         return upload_name;
     }
 
-    // TODO: implement .kafeignore
-    void Archive::archive_from_directory(const string &archive_path, const string &directory) {
+    void Archive::archive_from_directory(
+        const string &archive_path,
+        const string &directory,
+        ILogEventListener *logger
+    ) {
         if (!FileSystem::is_directory(directory)) {
             throw RuntimeException("Can not create archive - directory <%s> not found", directory.c_str());
         }
 
         if (FileSystem::is_file_or_symlink(archive_path)) {
-            throw RuntimeException("Can not create archive - path <%s> exists", directory.c_str());
+            throw RuntimeException("Can not create archive - path <%s> exists", archive_path.c_str());
         }
 
         auto archive_dir_name = std_fs::path(archive_path).parent_path();
@@ -59,6 +83,13 @@ namespace kafe::io {
             FileSystem::mkdirs(archive_dir_name);
         }
 
+        vector<string> ignore_patterns = {};
+        auto ignore_file = std_fs::path(directory).append(".kafeignore");
+        if (std_fs::is_regular_file(ignore_file)) {
+          logger->emit_debug("Loading .kafeignore file from %s", ignore_file.c_str());
+          ignore_patterns = read_ignore_file(&ignore_file);
+        }
+
         std_fs::recursive_directory_iterator iter(directory), end;
 
         auto *archive = archive_write_new();
@@ -88,6 +119,22 @@ namespace kafe::io {
                 size = std_fs::file_size(path_abs);
             }
 
+            bool is_ignored = false;
+
+            for (const auto& pattern : ignore_patterns) {
+                auto res = fnmatch(pattern.c_str(), path_rel.c_str(), FNM_PATHNAME | FNM_EXTMATCH | FNM_LEADING_DIR);
+                if (FNM_NOMATCH != res) {
+                    logger->emit_debug("Ignoring %s, matched by <%s>", path_rel.c_str(), pattern.c_str());
+                    is_ignored = true;
+                    break;
+                }
+            }
+
+            if (is_ignored) {
+                ++iter;
+                continue;
+            }
+
             struct stat buf{};
             stat(path_abs.c_str(), &buf);
 

diff --git a/libkafe/src/scripting/script.cpp b/libkafe/src/scripting/script.cpp
@@ -447,6 +447,7 @@ namespace kafe::scripting {
 
     int lua_api_archive_dir_tmp(lua_State *L) {
         auto *scope = get_scope(L);
+        auto *logger = const_cast<ILogEventListener *>(scope->get_context()->get_log_listener());
 
         if (scope->has_current_api()) {
             scope->get_context()->get_log_listener()->emit_warning(
@@ -467,7 +468,7 @@ namespace kafe::scripting {
                 directory_norm.c_str()
         );
 
-        auto path = Archive::tmp_archive_from_directory(directory_norm);
+        auto path = Archive::tmp_archive_from_directory(directory_norm, logger);
 
         scope->get_context()->get_log_listener()->emit_success(
                 &timer,
@@ -484,6 +485,7 @@ namespace kafe::scripting {
 
     int lua_api_archive_dir(lua_State *L) {
         const auto *scope = get_scope(L);
+        auto *logger = const_cast<ILogEventListener *>(scope->get_context()->get_log_listener());
 
         if (scope->has_current_api()) {
             scope->get_context()->get_log_listener()->emit_warning(
@@ -507,7 +509,7 @@ namespace kafe::scripting {
                 archive_norm.c_str()
         );
 
-        Archive::archive_from_directory(archive_norm, directory_norm);
+        Archive::archive_from_directory(archive_norm, directory_norm, logger);
 
         scope->get_context()->get_log_listener()->emit_success(
                 &timer,
@@ -973,4 +975,4 @@ namespace kafe::scripting {
     void Script::initialize() {
         lua_bootstrap(this->lua_state);
     }
-}
+}
diff --git a/publish.sh b/publish.sh
@@ -1,53 +1,53 @@
 #!/usr/bin/env bash
 
 # EL
-cloudsmith push rpm kafe/libkafe/el/7 build-artifact/centos-7/kafe-cli-1.1.1-1.x86_64.el7.rpm
-cloudsmith push rpm kafe/libkafe/el/7 build-artifact/centos-7/libkafe-1.1.1-1.x86_64.el7.rpm
-cloudsmith push rpm kafe/libkafe/el/7 build-artifact/centos-7/libkafe-devel-1.1.1-1.x86_64.el7.rpm
+cloudsmith push rpm kafe/libkafe/el/7 build-artifact/centos-7/kafe-cli-1.1.2-1.x86_64.el7.rpm
+cloudsmith push rpm kafe/libkafe/el/7 build-artifact/centos-7/libkafe-1.1.2-1.x86_64.el7.rpm
+cloudsmith push rpm kafe/libkafe/el/7 build-artifact/centos-7/libkafe-devel-1.1.2-1.x86_64.el7.rpm
 
-cloudsmith push rpm kafe/libkafe/el/8 build-artifact/centos-8/kafe-cli-1.1.1-1.x86_64.el8.rpm
-cloudsmith push rpm kafe/libkafe/el/8 build-artifact/centos-8/libkafe-1.1.1-1.x86_64.el8.rpm
-cloudsmith push rpm kafe/libkafe/el/8 build-artifact/centos-8/libkafe-devel-1.1.1-1.x86_64.el8.rpm
+cloudsmith push rpm kafe/libkafe/el/8 build-artifact/centos-8/kafe-cli-1.1.2-1.x86_64.el8.rpm
+cloudsmith push rpm kafe/libkafe/el/8 build-artifact/centos-8/libkafe-1.1.2-1.x86_64.el8.rpm
+cloudsmith push rpm kafe/libkafe/el/8 build-artifact/centos-8/libkafe-devel-1.1.2-1.x86_64.el8.rpm
 
 # Debian
-cloudsmith push deb kafe/libkafe/debian/stretch build-artifact/debian-9/kafe-cli_1.1.1_amd64.deb9.deb
-cloudsmith push deb kafe/libkafe/debian/stretch build-artifact/debian-9/libkafe_1.1.1_amd64.deb9.deb
-cloudsmith push deb kafe/libkafe/debian/stretch build-artifact/debian-9/libkafe-dev_1.1.1_amd64.deb9.deb
+cloudsmith push deb kafe/libkafe/debian/stretch build-artifact/debian-9/kafe-cli_1.1.2_amd64.deb9.deb
+cloudsmith push deb kafe/libkafe/debian/stretch build-artifact/debian-9/libkafe_1.1.2_amd64.deb9.deb
+cloudsmith push deb kafe/libkafe/debian/stretch build-artifact/debian-9/libkafe-dev_1.1.2_amd64.deb9.deb
 
-cloudsmith push deb kafe/libkafe/debian/buster build-artifact/debian-10/kafe-cli_1.1.1_amd64.deb10.deb
-cloudsmith push deb kafe/libkafe/debian/buster build-artifact/debian-10/libkafe_1.1.1_amd64.deb10.deb
-cloudsmith push deb kafe/libkafe/debian/buster build-artifact/debian-10/libkafe-dev_1.1.1_amd64.deb10.deb
+cloudsmith push deb kafe/libkafe/debian/buster build-artifact/debian-10/kafe-cli_1.1.2_amd64.deb10.deb
+cloudsmith push deb kafe/libkafe/debian/buster build-artifact/debian-10/libkafe_1.1.2_amd64.deb10.deb
+cloudsmith push deb kafe/libkafe/debian/buster build-artifact/debian-10/libkafe-dev_1.1.2_amd64.deb10.deb
 
-cloudsmith push deb kafe/libkafe/debian/bullseye build-artifact/debian-11/kafe-cli_1.1.1_amd64.deb11.deb
-cloudsmith push deb kafe/libkafe/debian/bullseye build-artifact/debian-11/libkafe_1.1.1_amd64.deb11.deb
-cloudsmith push deb kafe/libkafe/debian/bullseye build-artifact/debian-11/libkafe-dev_1.1.1_amd64.deb11.deb
+cloudsmith push deb kafe/libkafe/debian/bullseye build-artifact/debian-11/kafe-cli_1.1.2_amd64.deb11.deb
+cloudsmith push deb kafe/libkafe/debian/bullseye build-artifact/debian-11/libkafe_1.1.2_amd64.deb11.deb
+cloudsmith push deb kafe/libkafe/debian/bullseye build-artifact/debian-11/libkafe-dev_1.1.2_amd64.deb11.deb
 
 # Ubuntu
-cloudsmith push deb kafe/libkafe/ubuntu/bionic build-artifact/ubuntu-1804/kafe-cli_1.1.1_amd64.ubu1804.deb
-cloudsmith push deb kafe/libkafe/ubuntu/bionic build-artifact/ubuntu-1804/libkafe_1.1.1_amd64.ubu1804.deb
-cloudsmith push deb kafe/libkafe/ubuntu/bionic build-artifact/ubuntu-1804/libkafe-dev_1.1.1_amd64.ubu1804.deb
+cloudsmith push deb kafe/libkafe/ubuntu/bionic build-artifact/ubuntu-1804/kafe-cli_1.1.2_amd64.ubu1804.deb
+cloudsmith push deb kafe/libkafe/ubuntu/bionic build-artifact/ubuntu-1804/libkafe_1.1.2_amd64.ubu1804.deb
+cloudsmith push deb kafe/libkafe/ubuntu/bionic build-artifact/ubuntu-1804/libkafe-dev_1.1.2_amd64.ubu1804.deb
 
-cloudsmith push deb kafe/libkafe/ubuntu/eoan build-artifact/ubuntu-1910/kafe-cli_1.1.1_amd64.ubu1910.deb
-cloudsmith push deb kafe/libkafe/ubuntu/eoan build-artifact/ubuntu-1910/libkafe_1.1.1_amd64.ubu1910.deb
-cloudsmith push deb kafe/libkafe/ubuntu/eoan build-artifact/ubuntu-1910/libkafe-dev_1.1.1_amd64.ubu1910.deb
+cloudsmith push deb kafe/libkafe/ubuntu/eoan build-artifact/ubuntu-1910/kafe-cli_1.1.2_amd64.ubu1910.deb
+cloudsmith push deb kafe/libkafe/ubuntu/eoan build-artifact/ubuntu-1910/libkafe_1.1.2_amd64.ubu1910.deb
+cloudsmith push deb kafe/libkafe/ubuntu/eoan build-artifact/ubuntu-1910/libkafe-dev_1.1.2_amd64.ubu1910.deb
 
-cloudsmith push deb kafe/libkafe/ubuntu/focal build-artifact/ubuntu-2004/kafe-cli_1.1.1_amd64.ubu2004.deb
-cloudsmith push deb kafe/libkafe/ubuntu/focal build-artifact/ubuntu-2004/libkafe_1.1.1_amd64.ubu2004.deb
-cloudsmith push deb kafe/libkafe/ubuntu/focal build-artifact/ubuntu-2004/libkafe-dev_1.1.1_amd64.ubu2004.deb
+cloudsmith push deb kafe/libkafe/ubuntu/focal build-artifact/ubuntu-2004/kafe-cli_1.1.2_amd64.ubu2004.deb
+cloudsmith push deb kafe/libkafe/ubuntu/focal build-artifact/ubuntu-2004/libkafe_1.1.2_amd64.ubu2004.deb
+cloudsmith push deb kafe/libkafe/ubuntu/focal build-artifact/ubuntu-2004/libkafe-dev_1.1.2_amd64.ubu2004.deb
 
-cloudsmith push deb kafe/libkafe/ubuntu/groovy build-artifact/ubuntu-2010/kafe-cli_1.1.1_amd64.ubu2010.deb
-cloudsmith push deb kafe/libkafe/ubuntu/groovy build-artifact/ubuntu-2010/libkafe_1.1.1_amd64.ubu2010.deb
-cloudsmith push deb kafe/libkafe/ubuntu/groovy build-artifact/ubuntu-2010/libkafe-dev_1.1.1_amd64.ubu2010.deb
+cloudsmith push deb kafe/libkafe/ubuntu/groovy build-artifact/ubuntu-2010/kafe-cli_1.1.2_amd64.ubu2010.deb
+cloudsmith push deb kafe/libkafe/ubuntu/groovy build-artifact/ubuntu-2010/libkafe_1.1.2_amd64.ubu2010.deb
+cloudsmith push deb kafe/libkafe/ubuntu/groovy build-artifact/ubuntu-2010/libkafe-dev_1.1.2_amd64.ubu2010.deb
 
 # Fedora
-cloudsmith push rpm kafe/libkafe/fedora/31 build-artifact/fedora-31/kafe-cli-1.1.1-1.x86_64.f31.rpm
-cloudsmith push rpm kafe/libkafe/fedora/31 build-artifact/fedora-31/libkafe-1.1.1-1.x86_64.f31.rpm
-cloudsmith push rpm kafe/libkafe/fedora/31 build-artifact/fedora-31/libkafe-devel-1.1.1-1.x86_64.f31.rpm
+cloudsmith push rpm kafe/libkafe/fedora/31 build-artifact/fedora-31/kafe-cli-1.1.2-1.x86_64.f31.rpm
+cloudsmith push rpm kafe/libkafe/fedora/31 build-artifact/fedora-31/libkafe-1.1.2-1.x86_64.f31.rpm
+cloudsmith push rpm kafe/libkafe/fedora/31 build-artifact/fedora-31/libkafe-devel-1.1.2-1.x86_64.f31.rpm
 
-cloudsmith push rpm kafe/libkafe/fedora/32 build-artifact/fedora-32/kafe-cli-1.1.1-1.x86_64.f32.rpm
-cloudsmith push rpm kafe/libkafe/fedora/32 build-artifact/fedora-32/libkafe-1.1.1-1.x86_64.f32.rpm
-cloudsmith push rpm kafe/libkafe/fedora/32 build-artifact/fedora-32/libkafe-devel-1.1.1-1.x86_64.f32.rpm
+cloudsmith push rpm kafe/libkafe/fedora/32 build-artifact/fedora-32/kafe-cli-1.1.2-1.x86_64.f32.rpm
+cloudsmith push rpm kafe/libkafe/fedora/32 build-artifact/fedora-32/libkafe-1.1.2-1.x86_64.f32.rpm
+cloudsmith push rpm kafe/libkafe/fedora/32 build-artifact/fedora-32/libkafe-devel-1.1.2-1.x86_64.f32.rpm
 
-cloudsmith push rpm kafe/libkafe/fedora/33 build-artifact/fedora-33/kafe-cli-1.1.1-1.x86_64.f33.rpm
-cloudsmith push rpm kafe/libkafe/fedora/33 build-artifact/fedora-33/libkafe-1.1.1-1.x86_64.f33.rpm
-cloudsmith push rpm kafe/libkafe/fedora/33 build-artifact/fedora-33/libkafe-devel-1.1.1-1.x86_64.f33.rpm
+cloudsmith push rpm kafe/libkafe/fedora/33 build-artifact/fedora-33/kafe-cli-1.1.2-1.x86_64.f33.rpm
+cloudsmith push rpm kafe/libkafe/fedora/33 build-artifact/fedora-33/libkafe-1.1.2-1.x86_64.f33.rpm
+cloudsmith push rpm kafe/libkafe/fedora/33 build-artifact/fedora-33/libkafe-devel-1.1.2-1.x86_64.f33.rpm