Merge pull request #3 from michalszmidt/dev

Dev

.github/workflows/publish_dragonflybsd_amd64.yml

@@ -0,0 +1,38 @@
+name: Publish for DragonflyBSD amd64
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  test:
+    runs-on: macos-12
+    name: Build for DragonflyBSD
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build in DragonflyBSD VM
+      id: test
+      uses: vmactions/dragonflybsd-vm@v0
+      with:
+        usesh: false
+        mem: 8192
+        prepare: |
+          pkg update
+          pkg upgrade -y
+          pkg install -y curl rust pkgconf openssl
+          rehash
+
+        run: |
+          rehash
+          ls -lah
+          rustc --version
+          cargo build --release
+          
+    - name: Upload binaries to release
+      uses: svenstaro/upload-release-action@v2
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: target/release/hctl
+        asset_name: hctl-dragonflybsd-amd64
+        tag: ${{ github.ref }}

.github/workflows/publish_freebsd.yml → .github/workflows/publish_freebsd_amd64.yml

@@ -1,4 +1,4 @@
-name: Publish for FreeBSD
+name: Publish for FreeBSD amd64
 
 on:
   push:
@@ -8,19 +8,20 @@ on:
 jobs:
   build_freebsd:
     runs-on: macos-12
-    name: Build for FreeBSD-RELEASE
+    name: Build for FreeBSD-RELEASE repo latest
     steps:
     - uses: actions/checkout@v3
     - name: Compile in FreeBSD VM
       id: compile
       uses: vmactions/freebsd-vm@v0
       with:
         usesh: false
-        mem: 2048
+        mem: 8192
         prepare: |
           mkdir -p /usr/local/etc/pkg/repos
           cp /etc/pkg/FreeBSD.conf /usr/local/etc/pkg/repos
           sed -i 's/FreeBSD:/FreeBSDlatest:/g'
+          sed -i 's/quarterly/latest/g'
           echo "FreeBSD: { enabled: no }" >> /usr/local/etc/pkg/repos/FreeBSD.conf
           pkg update
           pkg upgrade -y

.github/workflows/publish_linux_musl.yml → .github/workflows/publish_linux_musl_amd64.yml

@@ -1,4 +1,4 @@
-name: Publish for Linux linked against musl
+name: Publish for Linux musl amd64
 
 on:
   push:
@@ -16,7 +16,7 @@ jobs:
           branch: latest-stable
       - name: Install dependencies
         run: |
-          apk add curl rust cargo pkgconf openssl-dev
+          apk add curl rust cargo
         shell: alpine.sh --root {0}
       - name: Compile rootless
         run: |

.github/workflows/publish_netbsd_amd64.yml

@@ -0,0 +1,33 @@
+name: Publish for NetBSD amd64
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  test:
+    runs-on: macos-12
+    name: Build for NetBSD
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build in NetBSD VM
+      id: test
+      uses: vmactions/netbsd-vm@v0
+      with:
+        usesh: false
+        mem: 8192        
+        prepare: |
+          /usr/sbin/pkg_add curl rust pkgconf openssl
+        run: |
+          ls -lah
+          rustc --version
+          cargo build --release
+          
+    - name: Upload binaries to release
+      uses: svenstaro/upload-release-action@v2
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: target/release/hctl
+        asset_name: hctl-netbsd-amd64
+        tag: ${{ github.ref }}

.github/workflows/publish_openbsd_amd64.yml

@@ -0,0 +1,34 @@
+name: Publish for OpenBSD amd64
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  test:
+    runs-on: macos-12
+    name: Build for OpenBSD
+    steps:
+    - uses: actions/checkout@v3
+    - name: Compile in OpenBSD VM
+      id: test
+      uses: vmactions/openbsd-vm@v0
+      with:
+        usesh: false
+        mem: 8192
+        prepare: |
+          pkg_add curl rust pkgconf openssl
+
+        run: |
+          ls -lah
+          rustc --version
+          cargo build --release
+          
+    - name: Upload binaries to release
+      uses: svenstaro/upload-release-action@v2
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: target/release/hctl
+        asset_name: hctl-openbsd-amd64
+        tag: ${{ github.ref }}

.github/workflows/publish_supported.yml

@@ -17,7 +17,7 @@ jobs:
             asset_name: hctl-linux-glib-amd64
           - os: windows-latest
             artifact_name: hctl.exe
-            asset_name: hctl-windows-amd64
+            asset_name: hctl-windows-amd64.exe
           - os: macos-latest
             artifact_name: hctl
             asset_name: hctl-macos-amd64

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hctl"
-version = "0.1.3"
+version = "0.1.4"
 edition = "2021"
 authors = [ "Michał Szmidt" ]
 repository = "https://github.com/michalszmidt/hctl"

README.md

@@ -1,27 +1,68 @@
-# Hostlists tools cli (hctl)
+```
+                                              __            __     __
+                                             / /_   _____  / /_   / /
+                                            / __ \ / ___/ / __/  / / 
+                                           / / / // /__  / /_   / /  
+                                          /_/ /_/ \___/  \__/  /_/   
+```
+<div>
+  <img alt="" src="https://img.shields.io/github/repo-size/michalszmidt/hctl" />
+  <img alt="GitHub Last Commit" src="https://img.shields.io/github/last-commit/michalszmidt/hctl" />
+  <img alt="GitHub Issues" src="https://img.shields.io/github/issues/michalszmidt/hctl" />
+  <img alt="GitHub Pull Requests" src="https://img.shields.io/github/issues-pr/michalszmidt/hctl" />
+</div>
+
+## CI/CD:
+### Builds
+- <img alt="GitHub Workflow Status (with event)" src="https://img.shields.io/github/actions/workflow/status/michalszmidt/hctl/publish_freebsd_amd64.yml"> FreeBSD [amd64]
+- <img alt="GitHub Workflow Status (with event)" src="https://img.shields.io/github/actions/workflow/status/michalszmidt/hctl/publish_dragonflybsd_amd64.yml"> DragonflyBSD [amd64]
+- <img alt="GitHub Workflow Status (with event)" src="https://img.shields.io/github/actions/workflow/status/michalszmidt/hctl/publish_netbsd_amd64.yml"> NetBSD [amd64]
+- <img alt="GitHub Workflow Status (with event)" src="https://img.shields.io/github/actions/workflow/status/michalszmidt/hctl/publish_openbsd_amd64.yml"> OpenBSD [amd64]
+- <img alt="GitHub Workflow Status (with event)" src="https://img.shields.io/github/actions/workflow/status/michalszmidt/hctl/publish_linux_musl_amd64.yml"> Linux musl [amd64]
+- <img alt="GitHub Workflow Status (with event)" src="https://img.shields.io/github/actions/workflow/status/michalszmidt/hctl/publish_supported.yml"> Linux glibc [amd64]
+- <img alt="GitHub Workflow Status (with event)" src="https://img.shields.io/github/actions/workflow/status/michalszmidt/hctl/publish_supported.yml"> MacOS X [amd64]
+- <img alt="GitHub Workflow Status (with event)" src="https://img.shields.io/github/actions/workflow/status/michalszmidt/hctl/publish_supported.yml"> Windows [amd64]
+
+<!--
+|   | amd64 | aarch64 |
+|---|---|---|---------|
+| Linux glibc |   |   |
+| Linux musl|   |   | 
+| FreeBSD |   |   |
+| MacOS X |   |   |
+| DragonflyBSD |   | N/A |
+| OpenBSD |   |   |
+| NetBSD |   |   |
+| Windows |   | N/|
+-->
+
 ## About
 
-This is simple cli tool written in rust to manage (transform, clear from comments etc.) your hostlists
+Hostlists tools cli `hctl` is cli tool written in rust to manage (merge, transform, clear from comments etc.) your hostlists from diffrent formats to selected by you!
 
 ## Features
 
 - [x] Merge hostlist
 - [x] Remove duplicates
-- [x] Remove commented lines
+- [x] Remove commented lines (default behaviour)
 - [x] Fetch lists directly from network (https)
+- [x] YAML config
 - [x] Parallel processing
 - [x] Progressbar
 - [x] Rejected lines may be saved with information about source list
 - [x] Support for Windows encoded text files (line ending with \n\r)
-- [x] Supported input formats (linewise, 127.0.0.1, 0.0.0.0)
-- [x] Supported output formats (linewise, 127.0.0.1, 0.0.0.0, dnsmasq)
-- [ ] Whitelists
+- [x] Supported input formats (linewise, 127.0.0.1, 0.0.0.0, ~dnsmasq~, ~snort~, ~squid~, ~unbound~, ~bind~, ~netgar~, ~adblock~)
+- [x] Supported output formats (linewise, 127.0.0.1, 0.0.0.0, dnsmasq, ~snort~, ~squid~, ~unbound~, ~bind~, ~netgar~, ~adblock~)
+- [x] Output to stdout to be used with unix pipe stream control
+- [ ] Record existance validation from DNS servers (unencrypted and over tls)
+- [ ] Whitelists (enumerated, from external lists file/https)
+- [ ] Deep analysis (eliminating subdomains)
 - [ ] Fault-tolerance, better error handling
 - [ ] Deep rejected analysis
 
 Jump [here](https://github.com/michalszmidt/hctl/issues/1) to feature-request issue.
 
-## Magic behind 
+## Magic behind the scene
 
 There is heavy usage of rust iterators to enable lazy-reading
 
@@ -38,4 +79,4 @@ See rules.rs if you want to embed similar solution in your app.
 [BSD-3-clause-no-military](https://github.com/michalszmidt/hctl/blob/main/LICENSE)
 
 TD;LR
-Modified [BSD-3-clause](https://choosealicens
+Modified [BSD-3-clause](https://choosealicense.com/licenses/bsd-3-clause-clear/) that does not allow usage for military purpose. 

src/commands.rs

@@ -18,7 +18,7 @@ pub fn get_args_domain() -> Vec<Arg> {
         .action(ArgAction::Set);
 
     let arg_out: Arg = Arg::new("out")
-        .help("Path to the out file [path without quotes]")
+        .help("Path to the out file [stdout/path without quotes]")
         .short('o')
         .long("out")
         .action(ArgAction::Set);

src/main.rs

@@ -82,11 +82,11 @@ fn main() {
                 match optimize.as_str() {
                     "speed" => {
                         (entries_len, rejected_len) =
-                            process_parallel_list_to_file(path, out, rejected_b, format)
+                            process_parallel_list_to_file(path, out.clone(), rejected_b, format)
                     }
                     "memory" => {
                         (entries_len, rejected_len) =
-                            process_single_list_seq_file(path, out, rejected_b, format)
+                            process_single_list_seq_file(path, out.clone(), rejected_b, format)
                     }
                     _ => return,
                 };
@@ -97,18 +97,20 @@ fn main() {
                     return;
                 }
                 (entries_len, rejected_len) =
-                    process_multiple_lists_to_file(path, out, rejected_b, format);
+                    process_multiple_lists_to_file(path, out.clone(), rejected_b, format);
             }
             "config" => {
                 (entries_len, rejected_len) =
-                    config_process_lists(config, out, intro_b, rejected_b, format)
+                    config_process_lists(config, out.clone(), intro_b, rejected_b, format)
             }
             _ => return,
         }
-        println!(
-            "Unique records: {}\nRemoved records: {}\n",
-            entries_len, rejected_len
-        );
+        if out.as_str() != "stdout" {
+            println!(
+                "Unique records: {}\nRemoved records: {}\n",
+                entries_len, rejected_len
+            );
+        }
     } else {
         unreachable!()
     }

src/processing.rs

@@ -29,11 +29,16 @@ pub fn process_parallel_list_to_file(
     let file_opened = file_to_lines(list_path).unwrap();
     let reader = BufReader::new(file_opened);
 
-    let file_out = file_write(out_path).unwrap();
-    let mut writer_out = LineWriter::new(file_out);
+    // let file_out = file_write(out_path).unwrap();
+    // let mut writer_out = BufWriter::new(file_out);
+
+    let mut writer_out = match out_path.as_str() {
+        "stdout" => Box::new(io::stdout()) as Box<dyn Write>,
+        _ => Box::new(file_write(out_path).unwrap()) as Box<dyn Write>,
+    };
 
     let file_rejected = file_write("./rejected.txt".to_string()).unwrap();
-    let mut writer_rejected = LineWriter::new(file_rejected);
+    let mut writer_rejected = BufWriter::new(file_rejected);
 
     let arc_mux_set_rejected = Arc::new(Mutex::new(BTreeSet::new()));
     let mut count_entries: usize = 0;
@@ -109,11 +114,14 @@ pub fn process_single_list_seq_file(
 
     let file_opened = file_to_lines(list_path).unwrap();
     let reader = BufReader::new(file_opened);
-    let file_out = file_write(out_path).unwrap();
-    let mut writer_out = LineWriter::new(file_out);
+
+    let mut writer_out = match out_path.as_str() {
+        "stdout" => Box::new(io::stdout()) as Box<dyn Write>,
+        _ => Box::new(file_write(out_path).unwrap()) as Box<dyn Write>,
+    };
 
     let file_rejected = file_write("./rejected.txt".to_string()).unwrap();
-    let mut writer_rejected = LineWriter::new(file_rejected);
+    let mut writer_rejected = BufWriter::new(file_rejected);
 
     let mut set_rejected: BTreeSet<String> = BTreeSet::new();
     let mut count_entries: usize = 0;
@@ -216,11 +224,13 @@ pub fn process_multiple_lists_to_file(
     save_rejected: bool,
     format: String,
 ) -> (usize, usize) {
-    let file_out = file_write(out_path).unwrap();
-    let mut writer_out = LineWriter::new(file_out);
+    let mut writer_out = match out_path.as_str() {
+        "stdout" => Box::new(io::stdout()) as Box<dyn Write>,
+        _ => Box::new(file_write(out_path).unwrap()) as Box<dyn Write>,
+    };
 
     let file_rejected = file_write("./rejected.txt".to_string()).unwrap();
-    let mut writer_rejected = LineWriter::new(file_rejected);
+    let mut writer_rejected = BufWriter::new(file_rejected);
 
     let arc_mux_set_rejected = Arc::new(Mutex::new(BTreeSet::new()));
     let mut count_entries: usize = 0;
@@ -301,11 +311,13 @@ pub fn config_process_lists(
         .as_vec()
         .unwrap();
 
-    let file_out = file_write(out_path).unwrap();
-    let mut writer_out = LineWriter::new(file_out);
+    let mut writer_out = match out_path.as_str() {
+        "stdout" => Box::new(io::stdout()) as Box<dyn Write>,
+        _ => Box::new(file_write(out_path).unwrap()) as Box<dyn Write>,
+    };
 
     let file_rejected = file_write("./rejected.txt".to_string()).unwrap();
-    let mut writer_rejected = LineWriter::new(file_rejected);
+    let mut writer_rejected = BufWriter::new(file_rejected);
 
     let arc_mux_set_rejected = Arc::new(Mutex::new(BTreeSet::new()));
     let mut count_entries: usize = 0;