Merge pull request #1932 from dadoonet/ssl-verification

Implement SSL Verification

.github/workflows/pr.yml

@@ -55,7 +55,7 @@ jobs:
           key: fscrawler-docker-cache-${{ runner.os }}-${{ hashFiles('pom.xml') }}
         continue-on-error: true
       - name: Run the integration tests
-        run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Dtests.parallelism=1
+        run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Dtests.parallelism=1 -Dtests.output=always
 
   # We run integration tests with elastic stack 7
   it-es7:

docs/source/admin/fs/elasticsearch.rst

@@ -24,7 +24,7 @@ Here is a list of Elasticsearch settings (under ``elasticsearch.`` prefix)`:
 +-----------------------------------+---------------------------+---------------------------------+
 | ``elasticsearch.pipeline``        | ``null``                  | :ref:`ingest_node`              |
 +-----------------------------------+---------------------------+---------------------------------+
-| ``elasticsearch.nodes``           | ``http://127.0.0.1:9200`` | `Node settings`_                |
+| ``elasticsearch.nodes``           | ``https://127.0.0.1:9200``| `Node settings`_                |
 +-----------------------------------+---------------------------+---------------------------------+
 | ``elasticsearch.path_prefix``     | ``null``                  | `Path prefix`_                  |
 +-----------------------------------+---------------------------+---------------------------------+
@@ -34,7 +34,9 @@ Here is a list of Elasticsearch settings (under ``elasticsearch.`` prefix)`:
 +-----------------------------------+---------------------------+---------------------------------+
 | ``elasticsearch.password``        | ``null``                  | :ref:`credentials`              |
 +-----------------------------------+---------------------------+---------------------------------+
-| ``elasticsearch.ssl_verification``| ``true``                  | :ref:`credentials`              |
+| ``elasticsearch.ssl_verification``| ``true``                  | :ref:`ssl`                      |
++-----------------------------------+---------------------------+---------------------------------+
+| ``elasticsearch.ca_certificate``  | ``null``                  | :ref:`ssl`                      |
 +-----------------------------------+---------------------------+---------------------------------+
 
 Index settings
@@ -274,7 +276,7 @@ You can define multiple nodes:
          nodes:
          - url: "https://CLUSTERID.eu-west-1.aws.found.io:9243"
 
-    For more information, read :ref:`ssl`.
+    For more information about HTTPS and SSL, read :ref:`ssl`.
 
 Path prefix
 ^^^^^^^^^^^
@@ -404,20 +406,52 @@ Then, you can assign this role to the user who will be defined within the ``user
 SSL Configuration
 ^^^^^^^^^^^^^^^^^
 
-In order to ingest documents to Elasticsearch over HTTPS based connection, you need to perform additional configuration
-steps:
+In order to ingest documents to Elasticsearch over HTTPS based connection, you obviously need to set the URL
+to ``https://your-server-address``. If your server is using a certificate that has been signed
+by a Certificate Authority, then you're good to go. For example, that's the case if you are running Elasticsearch
+from cloud.elastic.co.
+
+But if you are using a self signed certificate, which is the case in development mode, you need to either
+ignore the ssl check (not recommended) or provide the certificate to the Elasticsearch client.
+
+To bypass the SSL Certificate verification, you can use the ``ssl_verification`` option:
+
+.. code:: yaml
+
+   name: "test"
+   elasticsearch:
+     api_key: "VnVhQ2ZHY0JDZGJrUW0tZTVhT3g6dWkybHAyYXhUTm1zeWFrdzl0dk5udw=="
+     ssl_verification: false
+
+If you are running Elasticsearch from a Docker container, you can copy the self-signed certificate
+generated in ``/usr/share/elasticsearch/config/certs/http_ca.crt`` to your local machine:
+
+.. code:: sh
+
+    docker cp CONTAINER_NAME:/usr/share/elasticsearch/config/certs/http_ca.crt /path/to/certificate
+
+And then, you can specify this file in the ``elasticsearch.ca_certificate`` option:
+
+.. code:: yaml
+
+   name: "test"
+   elasticsearch:
+     api_key: "VnVhQ2ZHY0JDZGJrUW0tZTVhT3g6dWkybHAyYXhUTm1zeWFrdzl0dk5udw=="
+     ca_certificate: /path/to/certificate/http_ca.crt
+
+.. note::
 
-.. important::
+    You can also import your certificate into ``<JAVA_HOME>\lib\security\cacerts``.
 
-    Prerequisite: you need to have root CA chain certificate or Elasticsearch server certificate
-    in DER format. DER format files have a ``.cer`` extension. Certificate verification can be disabled by option ``ssl_verification: false``
+    For example, if you have a root CA chain certificate or Elasticsearch server certificate
+    in DER format (it's a binary format using a ``.cer`` extension), you need to:
 
 1. Logon to server (or client machine) where FSCrawler is running
 2. Run:
 
 .. code:: sh
 
-    keytool -import -alias <alias name> -keystore " <JAVA_HOME>\lib\security\cacerts" -file <Path of Elasticsearch Server certificate or Root certificate>
+    keytool -import -alias <alias name> -keystore "<JAVA_HOME>\lib\security\cacerts" -file <Path of Elasticsearch Server certificate or Root certificate>
 
 It will prompt you for the password. Enter the certificate password like ``changeit``.
 
@@ -427,8 +461,9 @@ It will prompt you for the password. Enter the certificate password like ``chang
 
     name: "test"
     elasticsearch:
-      nodes:
-      - url: "https://localhost:9243"
+     api_key: "VnVhQ2ZHY0JDZGJrUW0tZTVhT3g6dWkybHAyYXhUTm1zeWFrdzl0dk5udw=="
+     nodes:
+     - url: "https://localhost:9243"
 
 .. tip::
 

elasticsearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClient.java

@@ -46,21 +46,15 @@
 import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
 import org.glassfish.jersey.logging.LoggingFeature;
 
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
+import javax.net.ssl.*;
+import java.io.*;
 import java.net.ConnectException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
-import java.security.KeyManagementException;
-import java.security.NoSuchAlgorithmException;
-import java.security.SecureRandom;
+import java.security.*;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
 import java.util.AbstractMap;
 import java.util.ArrayList;
@@ -122,52 +116,26 @@ public void start() throws ElasticsearchClientException {
             return;
         }
 
-            /*
-        if (settings.getPathPrefix() != null) {
-            builder.setPathPrefix(settings.getPathPrefix());
-        }
-
-        if (settings.getUsername() != null) {
-            if (settings.getSslVerification()) {
-                builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
-            } else {
-                builder.setHttpClientConfigCallback(httpClientBuilder -> {
-                    SSLContext sc;
-                    try {
-                        sc = SSLContext.getInstance("SSL");
-                        sc.init(null, trustAllCerts, new SecureRandom());
-                    } catch (KeyManagementException | NoSuchAlgorithmException e) {
-                        logger.warn("Failed to get SSL Context", e);
-                        throw new RuntimeException(e);
-                    }
-                    httpClientBuilder.setSSLStrategy(new SSLIOSessionStrategy(sc, new NullHostNameVerifier()));
-                    httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
-                    return httpClientBuilder;
-                });
-            }
-        }
-
-        return builder;
-    }
-    */
-
         // Create the client
         ClientConfig config = new ClientConfig();
         // We need to suppress this, so we can do DELETE with body
         config.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
 
         SSLContext sslContext = null;
         if (settings.getElasticsearch().isSslVerification()) {
-            // TODO implement this part and add elasticsearch ssl settings
-            // If we have a truststore and a keystore, let's use it
-            /*
-            SslConfigurator sslConfig = SslConfigurator.newInstance()
-                    .trustStoreFile("./truststore_client")
-                    .trustStorePassword("secret-password-for-truststore")
-                    .keyStoreFile("./keystore_client")
-                    .keyPassword("secret-password-for-keystore");
-            sslContext = sslConfig.createSSLContext();
-            */
+            String caCertificatePath = settings.getElasticsearch().getCaCertificate();
+            if (caCertificatePath != null) {
+                try {
+                    File certFile = new File(caCertificatePath);
+                    sslContext = sslContextFromHttpCaCrt(certFile);
+                    logger.debug("Using provided CA Certificate from [{}]", caCertificatePath);
+                    HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
+                    HttpsURLConnection.setDefaultHostnameVerifier(new NullHostNameVerifier());
+                } catch (IOException e) {
+                    logger.warn("Failed to load the CA certificate", e);
+                    throw new RuntimeException(e);
+                }
+            }
         } else {
             // Trusting all certificates. For test purposes only.
             try {
@@ -199,7 +167,7 @@ public void start() throws ElasticsearchClientException {
         if (sslContext != null) {
             clientBuilder.sslContext(sslContext);
         }
-        client =  clientBuilder.build();
+        client = clientBuilder.build();
         if (logger.isTraceEnabled()) {
             client
 //                    .property(LoggingFeature.LOGGING_FEATURE_LOGGER_NAME_CLIENT, ElasticsearchClient.class.getName())
@@ -237,6 +205,26 @@ public void start() throws ElasticsearchClientException {
                 .build();
     }
 
+    private static SSLContext sslContextFromHttpCaCrt(File file) throws IOException {
+        try(InputStream in = new FileInputStream(file)) {
+            CertificateFactory cf = CertificateFactory.getInstance("X.509");
+            Certificate certificate = cf.generateCertificate(in);
+
+            final KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
+            keyStore.load(null, null);
+            keyStore.setCertificateEntry("elasticsearch-ca", certificate);
+
+            TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+            tmf.init(keyStore);
+
+            SSLContext sslContext = SSLContext.getInstance("TLS");
+            sslContext.init(null, tmf.getTrustManagers(), null);
+            return sslContext;
+        } catch (CertificateException | NoSuchAlgorithmException | KeyManagementException | KeyStoreException | IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public List<String> getAvailableNodes() {
         return hosts;
     }
@@ -897,8 +885,12 @@ private String httpDelete(String path, Object data) throws ElasticsearchClientEx
     }
 
     @SafeVarargs
-    private String httpCall(String method, String path, Object data, Map.Entry<String, Object>... params) throws ElasticsearchClientException {
+    private String httpCall(String method, String localPath, Object data, Map.Entry<String, Object>... params) throws ElasticsearchClientException {
         String node = getNode();
+        String path = localPath;
+        if (settings.getElasticsearch().getPathPrefix() != null) {
+            path = settings.getElasticsearch().getPathPrefix() + "/" + localPath;
+        }
         logger.trace("Calling {} {}/{} with params {}", method, node, path == null ? "" : path, params);
         try {
             Invocation.Builder callBuilder = prepareHttpCall(node, path, params);

integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java

@@ -104,6 +104,7 @@ public abstract class AbstractITCase extends AbstractFSCrawlerTestCase {
     protected static String testApiKey = getSystemProperty("tests.cluster.apiKey", null);
     protected final static boolean testKeepData = getSystemProperty("tests.leaveTemporary", true);
     protected final static boolean testCheckCertificate = getSystemProperty("tests.cluster.check_ssl", true);
+    private static String testCaCertificate = null;
 
     protected static Elasticsearch elasticsearchConfiguration;
     protected static FsCrawlerManagementServiceElasticsearchImpl managementService = null;
@@ -270,6 +271,12 @@ public static void startServices() throws IOException, ElasticsearchClientExcept
         if (fsSettings == null) {
             staticLogger.info("Elasticsearch is not running on [{}]. We start TestContainer.", testClusterUrl);
             testClusterUrl = testContainerHelper.startElasticsearch(testKeepData);
+            // Write the Ca Certificate on disk if exists (with versions < 8, no self-signed certificate)
+            if (testContainerHelper.getCertAsBytes() != null) {
+                Path clusterCaCrtPath = rootTmpDir.resolve("cluster-ca.crt");
+                Files.write(clusterCaCrtPath, testContainerHelper.getCertAsBytes());
+                testCaCertificate = clusterCaCrtPath.toAbsolutePath().toString();
+            }
             fsSettings = startClient();
         }
 
@@ -302,11 +309,12 @@ public static void startServices() throws IOException, ElasticsearchClientExcept
     }
 
     private static FsSettings startClient() throws IOException, ElasticsearchClientException {
-        staticLogger.info("Starting a client against [{}]", testClusterUrl);
+        staticLogger.info("Starting a client against [{}] with [{}] as a CA certificate", testClusterUrl, testCaCertificate);
         // We build the elasticsearch Client based on the parameters
         elasticsearchConfiguration = Elasticsearch.builder()
                 .setNodes(Collections.singletonList(new ServerUrl(testClusterUrl)))
-                .setSslVerification(false)
+                .setSslVerification(true)
+                .setCaCertificate(testCaCertificate)
                 .setCredentials(testApiKey, testClusterUser, testClusterPass)
                 .build();
         FsSettings fsSettings = FsSettings.builder("esClient").setElasticsearch(elasticsearchConfiguration).build();
@@ -341,6 +349,7 @@ public static void stopServices() throws IOException {
             managementService = null;
             staticLogger.info("Management service stopped");
         }
+        testCaCertificate = null;
     }
 
     @Before

settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Elasticsearch.java

@@ -61,6 +61,7 @@ public class Elasticsearch {
     private String pathPrefix;
     private boolean sslVerification = true;
     private boolean pushTemplates = true;
+    private String caCertificate;
 
     public Elasticsearch() {
 
@@ -69,7 +70,9 @@ public Elasticsearch() {
     private Elasticsearch(List<ServerUrl> nodes, String index, String indexFolder, int bulkSize,
                           TimeValue flushInterval, ByteSizeValue byteSize, String apiKey,
                           String username, String password, String pipeline,
-                          String pathPrefix, boolean sslVerification, boolean pushTemplates) {
+                          String pathPrefix, boolean sslVerification,
+                          String caCertificate,
+                          boolean pushTemplates) {
         this.nodes = nodes;
         this.index = index;
         this.indexFolder = indexFolder;
@@ -82,6 +85,7 @@ private Elasticsearch(List<ServerUrl> nodes, String index, String indexFolder, i
         this.pipeline = pipeline;
         this.pathPrefix = pathPrefix;
         this.sslVerification = sslVerification;
+        this.caCertificate = caCertificate;
         this.pushTemplates = pushTemplates;
     }
 
@@ -199,6 +203,14 @@ public void setPushTemplates(boolean pushTemplates) {
         this.pushTemplates = pushTemplates;
     }
 
+    public String getCaCertificate() {
+        return caCertificate;
+    }
+
+    public void setCaCertificate(String caCertificate) {
+        this.caCertificate = caCertificate;
+    }
+
     @SuppressWarnings("UnusedReturnValue")
     public static class Builder {
         private List<ServerUrl> nodes = Collections.singletonList(NODE_DEFAULT);
@@ -212,6 +224,7 @@ public static class Builder {
         private String pipeline = null;
         private String pathPrefix = null;
         private boolean sslVerification = true;
+        private String caCertificate;
         private boolean pushTemplates = true;
         private String apiKey = null;
 
@@ -312,6 +325,11 @@ public Builder setSslVerification(boolean sslVerification) {
             return this;
         }
 
+        public Builder setCaCertificate(String caCertificate) {
+            this.caCertificate = caCertificate;
+            return this;
+        }
+
         public Builder setPushTemplates(boolean pushTemplates) {
             this.pushTemplates = pushTemplates;
             return this;
@@ -320,7 +338,9 @@ public Builder setPushTemplates(boolean pushTemplates) {
         public Elasticsearch build() {
             return new Elasticsearch(nodes, index, indexFolder, bulkSize, flushInterval, byteSize, apiKey,
                     username, password,
-                    pipeline, pathPrefix, sslVerification, pushTemplates);
+                    pipeline, pathPrefix,
+                    sslVerification, caCertificate,
+                    pushTemplates);
         }
     }
 
@@ -341,6 +361,7 @@ public boolean equals(Object o) {
         if (!Objects.equals(pipeline, that.pipeline)) return false;
         if (!Objects.equals(pathPrefix, that.pathPrefix)) return false;
         if (!Objects.equals(sslVerification, that.sslVerification)) return false;
+        if (!Objects.equals(caCertificate, that.caCertificate)) return false;
         if (!Objects.equals(pushTemplates, that.pushTemplates)) return false;
         return Objects.equals(flushInterval, that.flushInterval);
 
@@ -357,6 +378,7 @@ public int hashCode() {
         result = 31 * result + (pathPrefix != null ? pathPrefix.hashCode() : 0);
         result = 31 * result + bulkSize;
         result = 31 * result + (flushInterval != null ? flushInterval.hashCode() : 0);
+        result = 31 * result + (caCertificate != null ? caCertificate.hashCode() : 0);
         result = 31 * result + (sslVerification? 1: 0);
         result = 31 * result + (pushTemplates? 1: 0);
         return result;
@@ -374,6 +396,7 @@ public String toString() {
                 ", pipeline='" + pipeline + '\'' +
                 ", pathPrefix='" + pathPrefix + '\'' +
                 ", sslVerification='" + sslVerification + '\'' +
+                ", caCertificatePath='" + caCertificate + '\'' +
                 ", pushTemplates='" + pushTemplates + '\'' +
                 '}';
     }