merge #2672, #2682 and #2683 from master to jetty 1.12.x branch

Signed-off-by: Henry Avetisyan <hga@yahooinc.com>

libs/java/cert_refresher/src/test/java/com/oath/auth/KeyRefresherTest.java

@@ -122,14 +122,15 @@ public void scanForFileChangesTestNoChanges() throws Exception {
         final String certFile = Objects.requireNonNull(classLoader.getResource("gdpr.aws.core.cert.pem")).getFile();
         final String keyFile = Objects.requireNonNull(classLoader.getResource("unit_test_gdpr.aws.core.key.pem")).getFile();
 
-        KeyRefresher keyRefresher = new KeyRefresher(certFile, keyFile, mockedTrustStore, mockedKeyManagerProxy, mockedTrustManagerProxy) {
+        KeyRefresher keyRefresher = new KeyRefresher(certFile, keyFile, mockedTrustStore,
+		mockedKeyManagerProxy, mockedTrustManagerProxy) {
             @Override
             protected boolean haveFilesBeenChanged(String filePath, byte[] checksum) {
                 return false;
             }
         };
-        keyRefresher.startup(1);
-        Thread.sleep(200);
+        keyRefresher.startup(100);
+        Thread.sleep(3000);
         keyRefresher.shutdown();
     }
 
@@ -145,16 +146,16 @@ public void scanForFileChangesTestWithChanges() throws Exception {
         final String certFile = Objects.requireNonNull(classLoader.getResource("gdpr.aws.core.cert.pem")).getFile();
         final String keyFile = Objects.requireNonNull(classLoader.getResource("unit_test_gdpr.aws.core.key.pem")).getFile();
 
-        KeyRefresher keyRefresher = new KeyRefresher(certFile, keyFile,
-            mockedTrustStore, mockedKeyManagerProxy, mockedTrustManagerProxy, listener) {
+        KeyRefresher keyRefresher = new KeyRefresher(certFile, keyFile, mockedTrustStore,
+		mockedKeyManagerProxy, mockedTrustManagerProxy, listener) {
             @Override
             protected boolean haveFilesBeenChanged(String filePath, byte[] checksum) {
                 return true;
             }
         };
 
-        keyRefresher.startup(1);
-        Thread.sleep(1000);
+        keyRefresher.startup(100);
+        Thread.sleep(3000);
         assertTrue(listener.keyChanged);
         keyRefresher.shutdown();
     }

libs/java/gcp_zts_creds/src/main/java/com/yahoo/athenz/creds/gcp/GCPSIACredentials.java

@@ -18,12 +18,14 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.athenz.auth.util.Crypto;
 import com.yahoo.athenz.zts.InstanceIdentity;
+import com.yahoo.athenz.zts.InstanceRefreshInformation;
 import com.yahoo.athenz.zts.InstanceRegisterInformation;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -32,13 +34,16 @@
 import org.bouncycastle.asn1.DERIA5String;
 import org.bouncycastle.asn1.x509.GeneralName;
 
+import javax.net.ssl.SSLContext;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
 
+import static org.apache.http.conn.ssl.SSLConnectionSocketFactory.getDefaultHostnameVerifier;
+
 public class GCPSIACredentials {
 
     // Configurations.
@@ -90,7 +95,7 @@ public static X509KeyPair getGCPFunctionServiceCertificate(String athenzDomain,
         final String instanceId = "gcp-function-" + gcpProjectId;
         return getWorkloadServiceCertificate(athenzDomain, athenzService, athenzProvider, ztsUrl,
                 sanDNSDomain, rdnCountry, rdnState, rdnLocality, rdnOrganization, rdnOrganizationUnit,
-                spiffeTrustDomain, instanceId);
+                spiffeTrustDomain, instanceId, null);
     }
 
     /**
@@ -121,13 +126,46 @@ public static X509KeyPair getGCPWorkloadServiceCertificate(String athenzDomain,
         final String instanceId = getGcpWorkloadInstanceId();
         return getWorkloadServiceCertificate(athenzDomain, athenzService, athenzProvider, ztsUrl,
                 sanDNSDomain, rdnCountry, rdnState, rdnLocality, rdnOrganization, rdnOrganizationUnit,
-                spiffeTrustDomain, instanceId);
+                spiffeTrustDomain, instanceId, null);
+    }
+
+    /**
+     * Refresh the registered instance with ZTS server by generating a new private key,
+     * requesting a new x.509 certificate based on the generated CSR and return
+     * both to the client in order to establish mtls connections with other
+     * Athenz enabled services. The client is required to provide the SSLContext
+     * that was created based on the previous certificate and private key.
+     * @param athenzDomain name of the domain
+     * @param athenzService name of the service
+     * @param athenzProvider name of the provider service for GCP Workloads
+     * @param ztsUrl ZTS Server URL e.g. https://zts.athenz.io:4443/zts/v1
+     * @param sanDNSDomain String identifying the DNS domain for generating SAN fields.
+     *                     For example, for the domain "sports", service "api" and certDomain "athenz.io",
+     *                     the sanDNS entry in the certificate will be set to "api.sports.athenz.io"
+     * @param rdnCountry Optional field in the certificate's Subject rdn (relative distinguished name).
+     * @param rdnState Optional field in the certificate's Subject rdn (relative distinguished name).
+     * @param rdnLocality Optional field in the certificate's Subject rdn (relative distinguished name).
+     * @param rdnOrganization Optional field in the certificate's Subject rdn (relative distinguished name).
+     * @param rdnOrganizationUnit Optional field in the certificate's Subject rdn (relative distinguished name).
+     * @param spiffeTrustDomain Optional spiffe trust domain
+     * @param sslContext SSLContext that was created based on the previous certificate and private key.
+     * @return private key and certificate from ZTS server.
+     */
+    public static X509KeyPair refreshGCPWorkloadServiceCertificate(String athenzDomain, String athenzService,
+            String athenzProvider, String ztsUrl, String sanDNSDomain, String rdnCountry,
+            String rdnState, String rdnLocality, String rdnOrganization, String rdnOrganizationUnit,
+            String spiffeTrustDomain, SSLContext sslContext) throws Exception {
+
+        final String instanceId = getGcpWorkloadInstanceId();
+        return getWorkloadServiceCertificate(athenzDomain, athenzService, athenzProvider, ztsUrl,
+                sanDNSDomain, rdnCountry, rdnState, rdnLocality, rdnOrganization, rdnOrganizationUnit,
+                spiffeTrustDomain, instanceId, sslContext);
     }
 
     private static X509KeyPair getWorkloadServiceCertificate(String athenzDomain, String athenzService,
             String athenzProvider, String ztsUrl, String sanDNSDomain, String rdnCountry,
             String rdnState, String rdnLocality, String rdnOrganization, String rdnOrganizationUnit,
-            String spiffeTrustDomain, String instanceId) throws Exception {
+            String spiffeTrustDomain, String instanceId, SSLContext sslContext) throws Exception {
 
         athenzDomain = athenzDomain.toLowerCase();
         athenzService = athenzService.toLowerCase();
@@ -154,9 +192,13 @@ private static X509KeyPair getWorkloadServiceCertificate(String athenzDomain, St
         final String x500Principal = certDn.isEmpty() ? "cn=" + athenzPrincipal : certDn + ",cn=" + athenzPrincipal;
         String csr = Crypto.generateX509CSR(response.privateKey, x500Principal, sanArray);
 
-        // Request the Athenz certificate from ZTS server.
-        InstanceIdentity identity = postInstanceRegisterInformation(athenzDomain, athenzService,
-                athenzProvider, ztsUrl, attestationData, csr);
+        // Request the Athenz certificate from ZTS server. If the SSL Context is provided,
+        // then it's a refresh operation as opposed to a new register operation
+        InstanceIdentity identity = sslContext == null ?
+                postInstanceRegisterInformation(athenzDomain, athenzService, athenzProvider, ztsUrl,
+                        attestationData, csr) :
+                postInstanceRefreshInformation(athenzDomain, athenzService, athenzProvider, instanceId,
+                        ztsUrl, attestationData, csr, sslContext);
 
         response.certificatePem = identity.x509Certificate;
         response.certificate = Crypto.loadX509Certificate(identity.x509Certificate);
@@ -249,37 +291,75 @@ private static GeneralName[] buildAlternativeDnsNames(final String athenzDomain,
         };
     }
 
+    /** Request the Athenz certificate from ZTS server */
+    private static InstanceIdentity postInstanceRefreshInformation(final String athenzDomain,
+            final String athenzService, final String athenzProvider, final String athenzInstanceId, final String ztsUrl,
+            final String attestationData, final String csr, SSLContext sslContext) throws Exception {
+
+        // construct the payload and http uri request for the refresh operation
+
+        InstanceRefreshInformation postPayloadObject = new InstanceRefreshInformation()
+                .setAttestationData(attestationData)
+                .setCsr(csr);
+
+        final String uri = ztsUrl + "/instance/" + athenzProvider + "/" + athenzDomain + "/" +
+                athenzService + "/" + athenzInstanceId;
+
+        final String postPayload = OBJECT_MAPPER.writeValueAsString(postPayloadObject);
+            HttpEntity httpEntity = new StringEntity(postPayload, ContentType.APPLICATION_JSON);
+            HttpUriRequest httpUriRequest = RequestBuilder.post()
+                    .setUri(uri)
+                    .setEntity(httpEntity)
+                    .addHeader("Content-Type", "application/json")
+                    .build();
+
+        return getServiceIdentity(httpUriRequest, sslContext);
+    }
+
     /** Request the Athenz certificate from ZTS server */
     private static InstanceIdentity postInstanceRegisterInformation(final String athenzDomain,
             final String athenzService, final String athenzProvider, final String ztsUrl,
             final String attestationData, final String csr) throws Exception {
 
+        // construct the payload and http uri request for the register operation
+
+        InstanceRegisterInformation postPayloadObject = new InstanceRegisterInformation()
+                .setDomain(athenzDomain)
+                .setService(athenzService)
+                .setProvider(athenzProvider)
+                .setAttestationData(attestationData)
+                .setCsr(csr);
+
+        final String postPayload = OBJECT_MAPPER.writeValueAsString(postPayloadObject);
+        HttpEntity httpEntity = new StringEntity(postPayload, ContentType.APPLICATION_JSON);
+        HttpUriRequest httpUriRequest = RequestBuilder.post()
+                .setUri(ztsUrl + "/instance")
+                .setEntity(httpEntity)
+                .addHeader("Content-Type", "application/json")
+                .build();
+
+        return getServiceIdentity(httpUriRequest, null);
+    }
+
+    /** Request the Athenz certificate from ZTS server */
+    private static InstanceIdentity getServiceIdentity(HttpUriRequest httpUriRequest, SSLContext sslContext) throws Exception {
+
         // Construct an HTTP client.
         RequestConfig config = RequestConfig.custom()
                 .setConnectTimeout(ZTS_CONNECT_TIMEOUT_MS)
                 .setSocketTimeout(ZTS_READ_TIMEOUT_MS)
                 .setRedirectsEnabled(false)
                 .build();
+
+        SSLConnectionSocketFactory sslConnectionSocketFactory = sslContext == null ? null :
+                new SSLConnectionSocketFactory(sslContext, new String[]{"TLSv1.2", "TLSv1.3"}, null,
+                        getDefaultHostnameVerifier());
+
         try (CloseableHttpClient httpClient = HttpClients.custom()
                 .setDefaultRequestConfig(config)
+                .setSSLSocketFactory(sslConnectionSocketFactory)
                 .build()) {
 
-            // Construct an HTTP POST request.
-            InstanceRegisterInformation postPayloadObject = new InstanceRegisterInformation()
-                    .setDomain(athenzDomain)
-                    .setService(athenzService)
-                    .setProvider(athenzProvider)
-                    .setAttestationData(attestationData)
-                    .setCsr(csr);
-
-            final String postPayload = OBJECT_MAPPER.writeValueAsString(postPayloadObject);
-            HttpEntity httpEntity = new StringEntity(postPayload, ContentType.APPLICATION_JSON);
-            HttpUriRequest httpUriRequest = RequestBuilder.post()
-                    .setUri(ztsUrl + "/instance")
-                    .setEntity(httpEntity)
-                    .addHeader("Content-Type", "application/json")
-                    .build();
-
             // Execute the request and process the response.
             HttpEntity httpResponseEntity = null;
             try (CloseableHttpResponse httpResponse = httpClient.execute(httpUriRequest)) {

libs/java/gcp_zts_creds/src/test/java/com/yahoo/athenz/creds/gcp/GCPSIACredentialsTest.java

@@ -21,9 +21,13 @@
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 import com.sun.net.httpserver.HttpServer;
+import com.yahoo.athenz.zts.InstanceRefreshInformation;
 import com.yahoo.athenz.zts.InstanceRegisterInformation;
+import org.mockito.Mockito;
 import org.testng.annotations.Test;
 
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -206,6 +210,41 @@ public void testGetGCPWorkloadServiceCertificateSuccess() throws Exception {
         }
     }
 
+    @Test
+    public void testRefreshGCPWorkloadServiceCertificateSuccess() throws Exception {
+        GCPSIACredentials.ATTESTATION_DATA_URL_PREFIX = "http://localhost:7356/mock-gcf-attestation-data?zts=";
+        GCPSIACredentials.INSTANCE_ID_META_DATA_URL = "http://localhost:7356/mock-instance-id";
+        MockGcfAttestationDataGoodHandler mockGcfAttestationDataHandler = new MockGcfAttestationDataGoodHandler();
+        MockGcfInstanceIdDataGoodHandler mockGcfInstanceIdDataHandler = new MockGcfInstanceIdDataGoodHandler();
+        MockZtsInstanceGoodHandler mockZtsInstanceHandler = new MockZtsInstanceGoodHandler();
+        SSLContext sslContext = Mockito.mock(SSLContext.class);
+        SSLSocketFactory sslSocketFactory = Mockito.mock(SSLSocketFactory.class);
+        Mockito.when(sslContext.getSocketFactory()).thenReturn(sslSocketFactory);
+        try (AutoCloseable ignored = startHttpServerForAttestationAndZtsInstance(
+                mockGcfAttestationDataHandler, mockGcfInstanceIdDataHandler,
+                mockZtsInstanceHandler)) {
+            GCPSIACredentials.X509KeyPair x509KeyPair = GCPSIACredentials.refreshGCPWorkloadServiceCertificate(
+                    "athenzDomain",
+                    "athenzService",
+                    "athenzProvider",
+                    "http://localhost:7356/mock-zts-instance",
+                    "certDomain",
+                    "optionalCountry",
+                    "optionalState",
+                    "optionalLocality",
+                    "optionalOrganization",
+                    "optionalOrganizationUnit",
+                    "",
+                    sslContext);
+            assertEquals(mockGcfAttestationDataHandler.requestedUri, "/mock-gcf-attestation-data?zts=http://localhost:7356/mock-zts-instance");
+            assertEquals(mockZtsInstanceHandler.requestedUri, "/mock-zts-instance/instance/athenzprovider/athenzdomain/athenzservice/instance-id");
+            InstanceRefreshInformation requestBody = new ObjectMapper().readValue(mockZtsInstanceHandler.requestedBody, InstanceRefreshInformation.class);
+            assertEquals(requestBody.attestationData, "{\"identityToken\":\"<MOCK-ATTESTATION-DATA>\"}");
+            assertEquals(x509KeyPair.certificatePem, MOCK_ATHENZ_CERT);
+            assertEquals(x509KeyPair.caCertificatesPem, MOCK_CA_CERTS);
+        }
+    }
+
     @Test
     public void testGCPSIACredentials() {
         GCPSIACredentials credentials = new GCPSIACredentials();

servers/zms/conf/zms.properties

@@ -574,3 +574,10 @@ athenz.zms.no_auth_uri_list=/zms/v1/schema
 # The setting specifies whether the server disallows groups to be added
 # as members in the admin role. The default value is true.
 #athenz.zms.disallow_groups_in_admin_role=true
+
+# This URL points the user to the documentation that explains how the
+# user authority filter works in the ZMS Server and what steps they
+# need to take to satisfy the requirements. By default, it just points
+# to the Athenz documentation but the system administrator should change
+# this to their own documentation with specific requirements.
+#athenz.zms.user_authority_filter_documentation_url=https://athenz.github.io/athenz/

servers/zms/src/main/java/com/yahoo/athenz/zms/DBService.java

@@ -6979,7 +6979,7 @@ int getMemberUserAuthorityState(final String roleMemberName, final Set<String> a
         int newState;
         if (bUser) {
             if (ZMSUtils.isUserAuthorityFilterValid(zmsConfig.getUserAuthority(), authorityFilterSet,
-                    roleMemberName)) {
+                    roleMemberName, null)) {
                 newState = currentState & ~Principal.State.AUTHORITY_FILTER_DISABLED.getValue();
             } else {
                 newState = currentState | Principal.State.AUTHORITY_FILTER_DISABLED.getValue();

servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSConsts.java

@@ -165,15 +165,14 @@ public final class ZMSConsts {
     public static final String ZMS_PROP_QUOTA_GROUP_MEMBER = "athenz.zms.quota_group_member";
     public static final String ZMS_PROP_QUOTA_ROLE_TAG     = "athenz.zms.quota_role_tag";
     public static final String ZMS_PROP_QUOTA_DOMAIN_TAG   = "athenz.zms.quota_domain_tag";
-    public static final String ZMS_PROP_QUOTA_GROUP_TAG   = "athenz.zms.quota_group_tag";
+    public static final String ZMS_PROP_QUOTA_GROUP_TAG    = "athenz.zms.quota_group_tag";
     public static final String ZMS_PROP_QUOTA_POLICY_TAG   = "athenz.zms.quota_policy_tag";
-    public static final String ZMS_PROP_QUOTA_SERVICE_TAG   = "athenz.zms.quota_service_tag";
+    public static final String ZMS_PROP_QUOTA_SERVICE_TAG  = "athenz.zms.quota_service_tag";
 
     public static final String ZMS_PROP_MYSQL_SERVER_TIMEZONE = "athenz.zms.mysql_server_timezone";
     public static final String ZMS_PROP_MYSQL_SERVER_TRUST_ROLES_UPDATE_TIMEOUT = "athenz.zms.mysql_server_trust_roles_update_timeout";
 
     public static final String ZMS_PRINCIPAL_AUTHORITY_CLASS  = "com.yahoo.athenz.auth.impl.PrincipalAuthority";
-
     public static final String ZMS_UNKNOWN_DOMAIN     = "unknown_domain";
     public static final String ZMS_INVALID_DOMAIN     = "invalid_domain";
     public static final String ZMS_SERVICE            = "zms";
@@ -446,6 +445,8 @@ public final class ZMSConsts {
     public static final String ZMS_PROP_REVIEW_DAYS_PERCENTAGE  = "athenz.zms.review_days_percentage";
     public static final Integer ZMS_PROP_REVIEW_DAYS_PERCENTAGE_DEFAULT = 68;
 
+    public static final String ZMS_PROP_USER_AUTHORITY_FILTER_DOC_URL = "athenz.zms.user_authority_filter_documentation_url";
+
     // ZMS feature flag bits
     public static final int ZMS_FEATURE_ALLOW_SERVICE_UNDERSCORE = 0x01;