diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 69d175d6..4581b531 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: 0918nobita/setup-cljstyle@v0.5.2 + - uses: 0918nobita/setup-cljstyle@v0.5.4 with: cljstyle-version: 0.15.0 - run: cljstyle check diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index a1db6fe3..1b628ca4 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -2,6 +2,19 @@ All notable changes to this project will be documented in this file. This change == Unreleased (dev) +== 1.3.0 (2021-11-18) + +// {{{ +=== Added +* https://github.com/liquidz/antq/issues/115[#115]: Added support for detecting libraries in `:local/root` dependencies. + +=== Changed +* Bumped tools.deps.alpha to 0.12.1071. + +=== Fixed +* https://github.com/liquidz/antq/issues/109[#109]: Fixed to correctly check versions of libraries in private repositories. +// }}} + == 1.2.0 (2021-11-06) // {{{ === Added diff --git a/README.adoc b/README.adoc index 3f2c9a34..8055881b 100644 --- a/README.adoc +++ b/README.adoc @@ -84,7 +84,7 @@ From Clojure CLI ver `1.10.3.933`, https://clojure.org/reference/deps_and_cli#to [source,sh] ---- # install -clojure -Ttools install com.github.liquidz/antq '{:git/tag "1.2.0"}' :as antq +clojure -Ttools install com.github.liquidz/antq '{:git/tag "1.3.0"}' :as antq # uninstall clojure -Ttools remove :tool antq # execute diff --git a/deps.edn b/deps.edn index 42cc7017..d2ba6aee 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ org.clojure/data.xml {:mvn/version "0.2.0-alpha6"} org.clojure/data.zip {:mvn/version "1.0.0"} org.clojure/tools.cli {:mvn/version "1.0.206"} - org.clojure/tools.deps.alpha {:mvn/version "0.12.1067"} + org.clojure/tools.deps.alpha {:mvn/version "0.12.1071"} org.clojure/data.json {:mvn/version "2.4.0"} clj-commons/clj-yaml {:mvn/version "0.7.107"} version-clj/version-clj {:mvn/version "2.0.2"} diff --git a/pom.xml b/pom.xml index efecc8cd..256b3fe7 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.github.liquidz antq - 1.2.0 + 1.3.0 antq Point out your outdated dependencies https://github.com/liquidz/antq @@ -47,7 +47,7 @@ org.clojure tools.deps.alpha - 0.12.1067 + 0.12.1071 org.clojure diff --git a/src/antq/dep/clojure.clj b/src/antq/dep/clojure.clj index 409964d6..04451ef2 100644 --- a/src/antq/dep/clojure.clj +++ b/src/antq/dep/clojure.clj @@ -5,15 +5,20 @@ [antq.util.dep :as u.dep] [clojure.edn :as edn] [clojure.java.io :as io] + [clojure.string :as str] + [clojure.tools.deps.alpha :as alpha] [clojure.tools.deps.alpha.extensions.git :as git] [clojure.walk :as walk])) (def ^:private project-file "deps.edn") -(defn- ignore? - [opt] - (and (map? opt) - (contains? opt :local/root))) +(declare load-deps) + +(defn user-deps-repository + [] + (let [file (io/file (alpha/user-deps-path))] + (when (.exists file) + (-> file slurp edn/read-string :mvn/repos)))) (defmulti extract-type-and-version (fn [opt] @@ -67,10 +72,33 @@ opt) opt)) +(defn- get-relative-path-by-current-working-directory + [current-working-directory relative-path] + (let [file (io/file current-working-directory) + dir (if (.isDirectory file) + file + (.getParentFile file))] + (if dir + (-> (str (u.dep/relative-path dir) + (System/getProperty "file.separator") + relative-path) + (str/replace #"\./" "")) + relative-path))) + +(defn- get-local-root-relative-path + [current-file-path opt] + (let [local-root (:local/root opt)] + (if (str/starts-with? local-root "/") + local-root + (get-relative-path-by-current-working-directory + current-file-path local-root)))) + (defn extract-deps - [file-path deps-edn-content-str] + [file-path deps-edn-content-str & [loaded-dir-set]] (let [deps (atom []) - edn (edn/read-string deps-edn-content-str)] + edn (edn/read-string deps-edn-content-str) + loaded-dir-set (or loaded-dir-set (atom #{})) + cross-project-repositories (user-deps-repository)] (walk/postwalk (fn [form] (when (and (sequential? form) (#{:deps :extra-deps :replace-deps :override-deps} (first form)) @@ -81,25 +109,45 @@ (swap! deps concat))) form) edn) - (for [[dep-name opt] @deps - :let [opt (adjust-version-via-deduction dep-name opt) - type-and-version (extract-type-and-version opt)] - :when (and (not (ignore? opt)) - (string? (:version type-and-version)) - (seq (:version type-and-version)))] - (-> {:project :clojure - :file file-path - :name (if (qualified-symbol? dep-name) - (str dep-name) - (str dep-name "/" dep-name)) - :repositories (:mvn/repos edn)} - (merge type-and-version) - (r/map->Dependency))))) + (->> @deps + (mapcat (fn [[dep-name opt]] + (let [opt (adjust-version-via-deduction dep-name opt) + type-and-version (extract-type-and-version opt)] + (cond + (not (map? opt)) + [nil] + + (contains? opt :local/root) + (let [path (get-local-root-relative-path file-path opt)] + (load-deps path loaded-dir-set)) + + (and (string? (:version type-and-version)) + (seq (:version type-and-version))) + (-> {:project :clojure + :file file-path + :name (if (qualified-symbol? dep-name) + (str dep-name) + (str dep-name "/" dep-name)) + :repositories (merge cross-project-repositories + (:mvn/repos edn))} + (merge type-and-version) + (r/map->Dependency) + (vector)) + + :else + [nil])))) + (remove nil?)))) (defn load-deps ([] (load-deps ".")) - ([dir] - (let [file (io/file dir project-file)] - (when (.exists file) - (extract-deps (u.dep/relative-path file) - (slurp file)))))) + ([dir] (load-deps dir (atom #{}))) + ([dir loaded-dir-set] + (let [dir (u.dep/normalize-path dir)] + ;; Avoid infinite loop + (when-not (contains? @loaded-dir-set dir) + (swap! loaded-dir-set conj dir) + (let [file (io/file dir project-file)] + (when (.exists file) + (extract-deps (u.dep/relative-path file) + (slurp file) + loaded-dir-set))))))) diff --git a/src/antq/util/dep.clj b/src/antq/util/dep.clj index 40c23ea4..023bc1a7 100644 --- a/src/antq/util/dep.clj +++ b/src/antq/util/dep.clj @@ -30,3 +30,22 @@ (defmethod normalize-by-name :default [dep] dep) + +(defn normalize-path + [^String path] + (let [sep (System/getProperty "file.separator")] + (loop [[v :as elements] (seq (.split path sep)) + accm []] + (if-not v + (str/join sep accm) + (recur (rest elements) + (condp = v + "." (cond + (seq accm) accm + (seq (rest elements)) accm + :else (conj accm v)) + + ".." (if (seq accm) + (vec (butlast accm)) + (conj accm v)) + (conj accm v))))))) diff --git a/src/antq/util/env.clj b/src/antq/util/env.clj new file mode 100644 index 00000000..f993cf8a --- /dev/null +++ b/src/antq/util/env.clj @@ -0,0 +1,5 @@ +(ns antq.util.env) + +(defn getenv + [x] + (System/getenv x)) diff --git a/src/antq/util/leiningen.clj b/src/antq/util/leiningen.clj new file mode 100644 index 00000000..dd324f11 --- /dev/null +++ b/src/antq/util/leiningen.clj @@ -0,0 +1,23 @@ +(ns antq.util.leiningen + (:require + [antq.util.env :as u.env] + [clojure.string :as str])) + +(defn- env-name + "cf. https://github.com/technomancy/leiningen/blob/master/doc/DEPLOY.md#credentials-in-the-environment" + [kw] + (cond + (and (qualified-keyword? kw) + (= "env" (namespace kw))) + (str/upper-case (name kw)) + + (= :env kw) + "LEIN_PASSWORD" + + :else + nil)) + +(defn env + [kw] + (some-> (env-name kw) + (u.env/getenv))) diff --git a/src/antq/util/maven.clj b/src/antq/util/maven.clj index fa5054ba..eaf8526d 100644 --- a/src/antq/util/maven.clj +++ b/src/antq/util/maven.clj @@ -2,6 +2,7 @@ (:require [antq.constant :as const] [antq.log :as log] + [antq.util.leiningen :as u.lein] [clojure.java.io :as io] [clojure.string :as str] [clojure.tools.deps.alpha.util.maven :as deps.util.maven] @@ -11,6 +12,9 @@ Model Scm) org.apache.maven.model.io.xpp3.MavenXpp3Reader + (org.apache.maven.settings + Server + Settings) (org.eclipse.aether DefaultRepositorySystemSession RepositorySystem) @@ -49,6 +53,35 @@ (normalize-repos)) :snapshots? (snapshot? (:version dep))}) +(defn ensure-username-or-password + [x] + (if (string? x) + x + (or (u.lein/env x) + (str x)))) + +(defn- ^Server new-repository-server + [{:keys [id username password]}] + (doto (Server.) + (.setId id) + (.setUsername (ensure-username-or-password username)) + (.setPassword (ensure-username-or-password password)))) + +(defn ^Settings get-maven-settings + [opts] + (let [settings ^Settings (deps.util.maven/get-settings) + server-ids (set (map #(.getId %) (.getServers settings)))] + ;; NOTE + ;; In Leiningen, authentication information is defined in project.clj instead of ~/.m2/settings.xml, + ;; so if there is authentication information in `:repositories`, apply to `settings` + (doseq [[id {:keys [username password]}] (:repositories opts)] + (when (and username + password + (not (contains? server-ids id))) + (.addServer settings + (new-repository-server {:id id :username username :password password})))) + settings)) + (def ^TransferListener custom-transfer-listener "Copy from clojure.tools.deps.alpha.util.maven/console-listener But no outputs for `transferStarted`" @@ -68,12 +101,13 @@ (let [lib (cond-> name (string? name) symbol) local-repo deps.util.maven/default-local-repo system ^RepositorySystem (deps.util.session/retrieve :mvn/system #(deps.util.maven/make-system)) - session ^DefaultRepositorySystemSession (deps.util.maven/make-session system local-repo) + settings ^Settings (get-maven-settings opts) + session ^DefaultRepositorySystemSession (deps.util.maven/make-session system settings local-repo) ;; Overwrite TransferListener not to show "Downloading" messages _ (.setTransferListener session custom-transfer-listener) ;; c.f. https://stackoverflow.com/questions/35488167/how-can-you-find-the-latest-version-of-a-maven-artifact-from-java-using-aether artifact (deps.util.maven/coord->artifact lib {:mvn/version version}) - remote-repos (deps.util.maven/remote-repos (:repositories opts))] + remote-repos (deps.util.maven/remote-repos (:repositories opts) settings)] {:system system :session session :artifact artifact @@ -93,7 +127,9 @@ (catch java.net.ConnectException e (if (= "Operation timed out" (.getMessage e)) (log/warning (str "Fetching pom from " url " failed because it timed out, retrying")) - (throw e)))) + (throw e))) + (catch java.io.IOException e + (log/warning (str "Fetching pom from " url " failed because of the following error: " (.getMessage e))))) (recur (inc i)))))) (defn ^String get-url diff --git a/test/antq/dep/babashka_test.clj b/test/antq/dep/babashka_test.clj index a962d4e7..351ae57b 100644 --- a/test/antq/dep/babashka_test.clj +++ b/test/antq/dep/babashka_test.clj @@ -9,5 +9,6 @@ :file "test/resources/dep/bb.edn" :name "bb/core" :version "1.0.0" - :project :clojure})] + :project :clojure + :repositories nil})] (sut/load-deps "test/resources/dep")))) diff --git a/test/antq/dep/clojure_test.clj b/test/antq/dep/clojure_test.clj index 9200074a..2dc6c1e7 100644 --- a/test/antq/dep/clojure_test.clj +++ b/test/antq/dep/clojure_test.clj @@ -3,10 +3,12 @@ [antq.dep.clojure :as sut] [antq.record :as r] [clojure.java.io :as io] - [clojure.test :as t])) + [clojure.test :as t] + [clojure.tools.deps.alpha :as alpha])) (def ^:private file-path - "path/to/deps.edn") + ;; "path/to/deps.edn" + (.getAbsolutePath (io/file (io/resource "dep/deps.edn")))) (defn- java-dependency [m] @@ -47,17 +49,38 @@ :extra {:url "https://github.com/example/sha.git"}}) (git-sha-dependency {:name "git-sha/git-sha" :version "dummy-git-sha" :extra {:url "https://github.com/example/git-sha.git"}}) - (git-sha-dependency {:name "com.github.liquidz/dummy" - :version "dummy-inferring-url" - :extra {:url "https://github.com/liquidz/dummy.git"}}) (git-tag-dependency {:name "tag-short-sha/tag-short-sha" :version "v1.2.3" :extra {:url "https://github.com/example/tag-short.git" :sha "123abcd"}}) (git-tag-dependency {:name "git-tag-long-sha/git-tag-long-sha" :version "v2.3.4" :extra {:url "https://github.com/example/git-tag-long.git" - :sha "1234567890abcdefghijklmnopqrstuvwxyz1234"}})} + :sha "1234567890abcdefghijklmnopqrstuvwxyz1234"}}) + (git-sha-dependency {:name "com.github.liquidz/dummy" + :version "dummy-inferring-url" + :extra {:url "https://github.com/liquidz/dummy.git"}}) + (java-dependency {:name "local/core" :version "9.9.9" + :file (.getAbsolutePath (io/file (io/resource "dep/local/deps.edn"))) + :repositories nil}) + (java-dependency {:name "local/nested-core" :version "8.8.8" + :file (.getAbsolutePath (io/file (io/resource "dep/local/nested/deps.edn"))) + :repositories nil})} (set deps))))) +(t/deftest extract-deps-cross-project-configuration-test + (let [cross-project-path (.getAbsolutePath + (io/file + (.getParentFile (io/file (io/resource "dep/deps.edn"))) + "cross-project" + "deps.edn")) + content (pr-str '{:deps {foo/bar {:mvn/version "0.0.1"}}})] + (with-redefs [alpha/user-deps-path (constantly cross-project-path)] + (t/is (= [(java-dependency + {:name "foo/bar" + :version "0.0.1" + :file "dummy" + :repositories {"cross-project" {:url "https://cross-project.example.com"}}})] + (sut/extract-deps "dummy" content)))))) + (t/deftest extract-deps-unexpected-test (t/is (empty? (sut/extract-deps file-path "[:deps \"foo\"]"))) (t/is (empty? (sut/extract-deps file-path "{:deps \"foo\"}"))) diff --git a/test/antq/util/dep_test.clj b/test/antq/util/dep_test.clj index 9e9d1ef6..29d4cc5b 100644 --- a/test/antq/util/dep_test.clj +++ b/test/antq/util/dep_test.clj @@ -49,3 +49,18 @@ (sut/name-candidates "foo/foo"))) (t/is (= #{} (sut/name-candidates "")))) + +(t/deftest normalize-path-test + (t/are [expected input] (= expected (sut/normalize-path input)) + "foo/bar" "foo/bar" + "foo/bar" "foo/./bar" + "bar" "foo/../bar" + "bar/baz" "foo/../bar/baz" + "bar/baz" "foo/../bar/./baz" + "../foo" "../foo" + "../bar" "foo/../../bar" + ".." ".." + "." "." + "" "" + "foo" "foo" + "foo" "./foo")) diff --git a/test/antq/util/leiningen_test.clj b/test/antq/util/leiningen_test.clj new file mode 100644 index 00000000..4dabacb6 --- /dev/null +++ b/test/antq/util/leiningen_test.clj @@ -0,0 +1,13 @@ +(ns antq.util.leiningen-test + (:require + [antq.util.env :as u.env] + [antq.util.leiningen :as sut] + [clojure.test :as t])) + +(t/deftest env-test + (with-redefs [u.env/getenv identity] + (t/is (= "LEIN_PASSWORD" (sut/env :env))) + (t/is (= "FOO" (sut/env :env/foo))) + (t/is (= "FOO_BAR" (sut/env :env/foo_bar))) + (t/is (= nil (sut/env :invalid/foo_bar))) + (t/is (= nil (sut/env "string"))))) diff --git a/test/antq/util/maven_test.clj b/test/antq/util/maven_test.clj index 6874b6c5..83c89f48 100644 --- a/test/antq/util/maven_test.clj +++ b/test/antq/util/maven_test.clj @@ -1,8 +1,43 @@ (ns antq.util.maven-test (:require [antq.record :as r] + [antq.util.env :as u.env] [antq.util.maven :as sut] - [clojure.test :as t])) + [clojure.test :as t] + [clojure.tools.deps.alpha.util.maven :as deps.util.maven]) + (:import + (org.apache.maven.settings + Server + Settings))) + +(def ^:private dummy-settings + (doto (Settings.) + (.addServer (doto (Server.) + (.setId "serv1"))) + (.addServer (doto (Server.) + (.setId "serv2") + (.setUsername "two-user") + (.setPassword "two-pass"))))) + +(def ^:private dummy-repos + {;; duplicated with dummy-settings + "serv1" {:url "https://one.example.com"} + ;; duplicated with dummy-settings + "serv2" {:url "https://two.example.com"} + ;; new to appear + "serv3" {:url "https://three.example.com" + :username "three-user" + :password "three-pass"} + ;; new to appear + "serv4" {:url "https://three.example.com" + :username :env + :password :env/four} + ;; should not be added because of missing username and password + "dummy" {:url "https://dummy.example.com"}}) + +(def ^:private dummy-env + {"LEIN_PASSWORD" "lein-pass" + "FOUR" "env-four"}) (t/deftest normalize-repo-url-test (t/are [expected in] (= expected (sut/normalize-repo-url in)) @@ -42,6 +77,26 @@ (sut/dep->opts (r/map->Dependency {:repositories {"foo" {:url "s3p://foo"}} :version "1.0.0-SNAPSHOT"}))))) +(t/deftest get-maven-settings-test + (with-redefs [deps.util.maven/get-settings (constantly dummy-settings) + u.env/getenv #(get dummy-env %)] + (let [settings (sut/get-maven-settings {:repositories dummy-repos}) + servers (map #(hash-map + :id (.getId %) + :username (.getUsername %) + :password (.getPassword %)) + (.getServers settings))] + (t/is (= 4 (count servers))) + + (t/is (= #{{:id "serv1" :username nil :password nil} + ;; from settings.xml + {:id "serv2" :username "two-user" :password "two-pass"} + ;; from project.clj + {:id "serv3" :username "three-user" :password "three-pass"} + ;; from project.clj with environmental variable + {:id "serv4" :username "lein-pass" :password "env-four"}} + (set servers)))))) + (t/deftest get-url-test (let [model (sut/read-pom "pom.xml")] (t/is (= "https://github.com/liquidz/antq" diff --git a/test/resources/dep/cross-project/deps.edn b/test/resources/dep/cross-project/deps.edn new file mode 100644 index 00000000..890d482e --- /dev/null +++ b/test/resources/dep/cross-project/deps.edn @@ -0,0 +1 @@ +{:mvn/repos {"cross-project" {:url "https://cross-project.example.com"}}} diff --git a/test/resources/dep/deps.edn b/test/resources/dep/deps.edn index bcca4d31..4deb4a75 100644 --- a/test/resources/dep/deps.edn +++ b/test/resources/dep/deps.edn @@ -14,14 +14,17 @@ ;; :git/tag (long sha) git-tag-long-sha {:git/url "https://github.com/example/git-tag-long.git" :git/tag "v2.3.4" :git/sha "1234567890abcdefghijklmnopqrstuvwxyz1234"} + ;; inferring :git/url from lib + com.github.liquidz/dummy {:sha "dummy-inferring-url"} + + ;; local/root + local-repo/local-repo {:local/root "./local"} - ;; local/root should be ignored - local-repo/local-repo {:local/root "/path/to/local/repo"} + ;; should be ignored + local-repo/non-existing {:local/root "/path/to/non-existing-local/repo"} ;; invalid versions should be ignored ver-not-string {:mvn/version :version} ver-empty {:mvn/version ""} - ;; inferring :git/url from lib - com.github.liquidz/dummy {:sha "dummy-inferring-url"} ;; no version no-version {}} diff --git a/test/resources/dep/local/deps.edn b/test/resources/dep/local/deps.edn new file mode 100644 index 00000000..a6ec8f41 --- /dev/null +++ b/test/resources/dep/local/deps.edn @@ -0,0 +1,2 @@ +{:deps {local/core {:mvn/version "9.9.9"} + local-repo2/local-repo2 {:local/root "./nested"}}} diff --git a/test/resources/dep/local/nested/deps.edn b/test/resources/dep/local/nested/deps.edn new file mode 100644 index 00000000..8c2cdd51 --- /dev/null +++ b/test/resources/dep/local/nested/deps.edn @@ -0,0 +1,3 @@ +{:deps {local/nested-core {:mvn/version "8.8.8"} + ;; should be skip + local-repo/local-repo {:local/root ".."}}}