Skip to content

Commit

Permalink
fixed conflicts
Browse files Browse the repository at this point in the history 
Signed-off-by: fahadshamiinsta <fshami@netapp.com>
  • Loading branch information
@fahadshamiinsta
fahadshamiinsta committed Jan 11, 2024
1 parent e4583a4 commit 5f991c8
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 29 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/))
- Deprecate CamelCase `PathHierarchy` tokenizer name in favor to lowercase `path_hierarchy` ([#10894](https://github.com/opensearch-project/OpenSearch/pull/10894))
- Switched to more reliable OpenSearch Lucene snapshot location([#11728](https://github.com/opensearch-project/OpenSearch/pull/11728))
- Added support for Google Application Default Credentials in repository-gcs ([#8394](https://github.com/opensearch-project/OpenSearch/pull/8394))

### Deprecated

Expand Down
29 changes: 29 additions & 0 deletions ...cs/src/main/java/org/opensearch/repositories/gcs/GoogleApplicationDefaultCredentials.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.repositories.gcs;

import com.google.auth.oauth2.GoogleCredentials;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.IOException;

public class GoogleApplicationDefaultCredentials {
private static final Logger logger = LogManager.getLogger(GoogleApplicationDefaultCredentials.class);

public GoogleCredentials get() {
GoogleCredentials credentials = null;
try {
credentials = SocketAccess.doPrivilegedIOException(GoogleCredentials::getApplicationDefault);
} catch (IOException e) {
logger.error("Failed to retrieve \"Application Default Credentials\"",e);
}
return credentials;
}
}
19 changes: 15 additions & 4 deletions ...pository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.ServiceOptions;
import com.google.cloud.http.HttpTransportOptions;
Expand Down Expand Up @@ -70,6 +71,15 @@ public class GoogleCloudStorageService {
*/
private volatile Map<String, Storage> clientCache = emptyMap();

final private GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials;

public GoogleCloudStorageService() {
this.googleApplicationDefaultCredentials = new GoogleApplicationDefaultCredentials();
}

public GoogleCloudStorageService(GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials) {
this.googleApplicationDefaultCredentials = googleApplicationDefaultCredentials;
}
/**
* Refreshes the client settings and clears the client cache. Subsequent calls to
* {@code GoogleCloudStorageService#client} will return new clients constructed
Expand Down Expand Up @@ -213,10 +223,11 @@ StorageOptions createStorageOptions(
storageOptionsBuilder.setProjectId(clientSettings.getProjectId());
}
if (clientSettings.getCredential() == null) {
logger.warn(
"\"Application Default Credentials\" are not supported out of the box."
+ " Additional file system permissions have to be granted to the plugin."
);
logger.info("\"Application Default Credentials\" will be in use");
final GoogleCredentials credentials = googleApplicationDefaultCredentials.get();
if (credentials != null) {
storageOptionsBuilder.setCredentials(credentials);
}
} else {
ServiceAccountCredentials serviceAccountCredentials = clientSettings.getCredential();
// override token server URI
Expand Down
112 changes: 87 additions & 25 deletions ...ory-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,12 @@
package org.opensearch.repositories.gcs;

import com.google.auth.Credentials;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.http.HttpTransportOptions;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import org.hamcrest.MatcherAssert;
import org.mockito.Mockito;
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
Expand All @@ -44,6 +48,10 @@
import org.opensearch.test.OpenSearchTestCase;
import org.hamcrest.Matchers;

import java.io.IOException;
import java.net.Proxy;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.util.Base64;
Expand All @@ -56,16 +64,17 @@

public class GoogleCloudStorageServiceTests extends OpenSearchTestCase {

final TimeValue connectTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
final TimeValue readTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
final String applicationName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
final String endpoint = randomFrom("http://", "https://")
+ randomFrom("www.opensearch.org", "www.googleapis.com", "localhost/api", "google.com/oauth")
+ ":"
+ randomIntBetween(1, 65535);
final String projectIdName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);

public void testClientInitializer() throws Exception {
final String clientName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
final TimeValue connectTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
final TimeValue readTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
final String applicationName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
final String endpoint = randomFrom("http://", "https://")
+ randomFrom("www.opensearch.org", "www.googleapis.com", "localhost/api", "google.com/oauth")
+ ":"
+ randomIntBetween(1, 65535);
final String projectIdName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
final Settings settings = Settings.builder()
.put(
GoogleCloudStorageClientSettings.CONNECT_TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(),
Expand All @@ -82,31 +91,35 @@ public void testClientInitializer() throws Exception {
.put(GoogleCloudStorageClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint)
.put(GoogleCloudStorageClientSettings.PROJECT_ID_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectIdName)
.build();
final GoogleCloudStorageService service = new GoogleCloudStorageService();
GoogleCredentials mockGoogleCredentials = Mockito.mock(GoogleCredentials.class);
GoogleApplicationDefaultCredentials mockDefaultCredentials = Mockito.mock(GoogleApplicationDefaultCredentials.class);
Mockito.when(mockDefaultCredentials.get()).thenReturn(mockGoogleCredentials);

final GoogleCloudStorageService service = new GoogleCloudStorageService(mockDefaultCredentials);
service.refreshAndClearCache(GoogleCloudStorageClientSettings.load(settings));
GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
final IllegalArgumentException e = expectThrows(
IllegalArgumentException.class,
() -> service.client("another_client", "repo", statsCollector)
);
assertThat(e.getMessage(), Matchers.startsWith("Unknown client name"));
MatcherAssert.assertThat(e.getMessage(), Matchers.startsWith("Unknown client name"));
assertSettingDeprecationsAndWarnings(
new Setting<?>[] { GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING.getConcreteSettingForNamespace(clientName) }
);
final Storage storage = service.client(clientName, "repo", statsCollector);
assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName));
assertThat(storage.getOptions().getHost(), Matchers.is(endpoint));
assertThat(storage.getOptions().getProjectId(), Matchers.is(projectIdName));
assertThat(storage.getOptions().getTransportOptions(), Matchers.instanceOf(HttpTransportOptions.class));
assertThat(
MatcherAssert.assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName));
MatcherAssert.assertThat(storage.getOptions().getHost(), Matchers.is(endpoint));
MatcherAssert.assertThat(storage.getOptions().getProjectId(), Matchers.is(projectIdName));
MatcherAssert.assertThat(storage.getOptions().getTransportOptions(), Matchers.instanceOf(HttpTransportOptions.class));
MatcherAssert.assertThat(
((HttpTransportOptions) storage.getOptions().getTransportOptions()).getConnectTimeout(),
Matchers.is((int) connectTimeValue.millis())
);
assertThat(
MatcherAssert.assertThat(
((HttpTransportOptions) storage.getOptions().getTransportOptions()).getReadTimeout(),
Matchers.is((int) readTimeValue.millis())
);
assertThat(storage.getOptions().getCredentials(), Matchers.nullValue(Credentials.class));
MatcherAssert.assertThat(storage.getOptions().getCredentials(), Matchers.instanceOf(Credentials.class));
}

public void testReinitClientSettings() throws Exception {
Expand All @@ -122,33 +135,33 @@ public void testReinitClientSettings() throws Exception {
final GoogleCloudStorageService storageService = plugin.storageService;
GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
final Storage client11 = storageService.client("gcs1", "repo1", statsCollector);
assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
MatcherAssert.assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
final Storage client12 = storageService.client("gcs2", "repo2", statsCollector);
assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
MatcherAssert.assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
// client 3 is missing
final IllegalArgumentException e1 = expectThrows(
IllegalArgumentException.class,
() -> storageService.client("gcs3", "repo3", statsCollector)
);
assertThat(e1.getMessage(), containsString("Unknown client name [gcs3]."));
MatcherAssert.assertThat(e1.getMessage(), containsString("Unknown client name [gcs3]."));
// update client settings
plugin.reload(settings2);
// old client 1 not changed
assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
MatcherAssert.assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
// new client 1 is changed
final Storage client21 = storageService.client("gcs1", "repo1", statsCollector);
assertThat(client21.getOptions().getProjectId(), equalTo("project_gcs21"));
MatcherAssert.assertThat(client21.getOptions().getProjectId(), equalTo("project_gcs21"));
// old client 2 not changed
assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
MatcherAssert.assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
// new client2 is gone
final IllegalArgumentException e2 = expectThrows(
IllegalArgumentException.class,
() -> storageService.client("gcs2", "repo2", statsCollector)
);
assertThat(e2.getMessage(), containsString("Unknown client name [gcs2]."));
MatcherAssert.assertThat(e2.getMessage(), containsString("Unknown client name [gcs2]."));
// client 3 emerged
final Storage client23 = storageService.client("gcs3", "repo3", statsCollector);
assertThat(client23.getOptions().getProjectId(), equalTo("project_gcs23"));
MatcherAssert.assertThat(client23.getOptions().getProjectId(), equalTo("project_gcs23"));
}
}

Expand Down Expand Up @@ -193,4 +206,53 @@ public void testToTimeout() {
assertEquals(-1, GoogleCloudStorageService.toTimeout(TimeValue.ZERO).intValue());
assertEquals(0, GoogleCloudStorageService.toTimeout(TimeValue.MINUS_ONE).intValue());
}

/*
The following method test the Google Application Default Credential instead of
using service account file.
Considered use of Junit Mocking due to static method GoogleCredentials.getApplicationDefault
and avoiding environment variables to set which later use GCE.
*/
public void testApplicationDefaultCredential() throws Exception {
GoogleCloudStorageClientSettings settings = getGCSClientSettingsWithoutCredentials();
GoogleCredentials mockGoogleCredentials = Mockito.mock(GoogleCredentials.class);
HttpTransportOptions mockHttpTransportOptions = Mockito.mock(HttpTransportOptions.class);
GoogleApplicationDefaultCredentials mockDefaultCredentials = Mockito.mock(GoogleApplicationDefaultCredentials.class);
Mockito.when(mockDefaultCredentials.get()).thenReturn(mockGoogleCredentials);

GoogleCloudStorageService service = new GoogleCloudStorageService(mockDefaultCredentials);
StorageOptions storageOptions = service.createStorageOptions(settings,mockHttpTransportOptions);
assertNotNull(storageOptions);
assertEquals(storageOptions.getCredentials().toString(),mockGoogleCredentials.toString());
}

/**
* The application default credential throws exception when there are no
* Environment Variables provided or Google Compute Engine is not running
* @throws Exception
*/
public void applicationDefaultCredentialThrowsErrorWhenNotAvailable() throws Exception {
GoogleCloudStorageClientSettings settings = getGCSClientSettingsWithoutCredentials();
HttpTransportOptions mockHttpTransportOptions = Mockito.mock(HttpTransportOptions.class);
GoogleCloudStorageService service = new GoogleCloudStorageService();
StorageOptions storageOptions = service.createStorageOptions(settings,mockHttpTransportOptions);

Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault);
assertNotNull(storageOptions);
assertNull(storageOptions.getCredentials());
MatcherAssert.assertThat(exception.getMessage(),containsString("The Application Default Credentials are not available"));
}

private GoogleCloudStorageClientSettings getGCSClientSettingsWithoutCredentials() throws URISyntaxException {
return new GoogleCloudStorageClientSettings(
null,
endpoint,
projectIdName,
connectTimeValue,
readTimeValue,
applicationName,
new URI(""),
new ProxySettings(Proxy.Type.DIRECT, null, 0, null, null));
}

}

0 comments on commit 5f991c8

Please sign in to comment.