-
Notifications
You must be signed in to change notification settings - Fork 72
Commit
Signed-off-by: Petar Dzepina <petar.dzepina@gmail.com> Co-authored-by: Subhobrata Dey <sbcd90@gmail.com> (cherry picked from commit 2e869e5)
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
package org.opensearch.securityanalytics.logtype; | ||
|
||
import java.io.IOException; | ||
import java.io.InputStream; | ||
import java.net.URISyntaxException; | ||
import java.nio.charset.StandardCharsets; | ||
import java.nio.file.Files; | ||
import java.nio.file.Path; | ||
import java.util.ArrayList; | ||
import java.util.List; | ||
import java.util.Map; | ||
import java.util.Objects; | ||
import java.util.function.Function; | ||
import java.util.stream.Collectors; | ||
import java.util.stream.Stream; | ||
import org.apache.logging.log4j.LogManager; | ||
import org.apache.logging.log4j.Logger; | ||
import org.opensearch.common.settings.SettingsException; | ||
import org.opensearch.common.xcontent.XContentHelper; | ||
import org.opensearch.common.xcontent.json.JsonXContent; | ||
import org.opensearch.securityanalytics.model.LogType; | ||
import org.opensearch.securityanalytics.util.FileUtils; | ||
|
||
public class BuiltinLogTypeLoader { | ||
|
||
private static final Logger logger = LogManager.getLogger(BuiltinLogTypeLoader.class); | ||
|
||
private static final String BASE_PATH = "OSMapping/"; | ||
|
||
private static final String LOG_TYPE_FILE_SUFFIX = "_logtype.json"; | ||
|
||
private static List<LogType> logTypes; | ||
private static Map<String, LogType> logTypeMap; | ||
|
||
|
||
static { | ||
ensureLogTypesLoaded(); | ||
} | ||
Check warning on line 42 in src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java Codecov / codecov/patchsrc/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java#L41-L42
|
||
|
||
public static List<LogType> getAllLogTypes() { | ||
ensureLogTypesLoaded(); | ||
return logTypes; | ||
Check warning on line 46 in src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java Codecov / codecov/patchsrc/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java#L45-L46
|
||
} | ||
|
||
public static LogType getLogTypeByName(String logTypeName) { | ||
ensureLogTypesLoaded(); | ||
return logTypeMap.get(logTypeName); | ||
Check warning on line 51 in src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java Codecov / codecov/patchsrc/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java#L50-L51
|
||
} | ||
|
||
public static boolean logTypeExists(String logTypeName) { | ||
ensureLogTypesLoaded(); | ||
return logTypeMap.containsKey(logTypeName); | ||
Check warning on line 56 in src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java Codecov / codecov/patchsrc/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java#L55-L56
|
||
} | ||
|
||
private static void ensureLogTypesLoaded() { | ||
try { | ||
if (logTypes != null) { | ||
return; | ||
} | ||
logTypes = loadBuiltinLogTypes(); | ||
logTypeMap = logTypes.stream() | ||
.collect(Collectors.toMap(LogType::getName, Function.identity())); | ||
} catch (Exception e) { | ||
logger.error("Failed loading builtin log types from disk!", e); | ||
} | ||
} | ||
Check warning on line 70 in src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java Codecov / codecov/patchsrc/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java#L64-L70
|
||
|
||
private static List<LogType> loadBuiltinLogTypes() throws URISyntaxException, IOException { | ||
List<LogType> logTypes = new ArrayList<>(); | ||
|
||
final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); | ||
|
||
Path dirPath = null; | ||
if (url.contains("!")) { | ||
final String[] paths = url.split("!"); | ||
dirPath = FileUtils.getFs().getPath(paths[1]); | ||
} else { | ||
dirPath = Path.of(url); | ||
Check warning on line 82 in src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java Codecov / codecov/patchsrc/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java#L79-L82
|
||
} | ||
|
||
Stream<Path> folder = Files.list(dirPath); | ||
List<Path> logTypePaths = folder.filter(e -> e.toString().endsWith(LOG_TYPE_FILE_SUFFIX)).collect(Collectors.toList()); | ||
Check warning on line 86 in src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java Codecov / codecov/patchsrc/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java#L85-L86
|
||
|
||
for (Path logTypePath : logTypePaths) { | ||
try ( | ||
InputStream is = BuiltinLogTypeLoader.class.getResourceAsStream(logTypePath.toString()) | ||
) { | ||
String logTypeFilePayload = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); | ||
|
||
if (logTypeFilePayload != null) { | ||
Map<String, Object> logTypeFileAsMap = | ||
XContentHelper.convertToMap(JsonXContent.jsonXContent, logTypeFilePayload, false); | ||
Check warning on line 96 in src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java Codecov / codecov/patchsrc/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java#L95-L96
|
||
|
||
logTypes.add(new LogType(logTypeFileAsMap)); | ||
|
||
logger.info("Loaded [{}] log type", logTypePath.getFileName()); | ||
} | ||
} catch (Exception e) { | ||
throw new SettingsException("Failed to load builtin log types", e); | ||
} | ||
} | ||
Check warning on line 105 in src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java Codecov / codecov/patchsrc/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java#L102-L105
|
||
|
||
return logTypes; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
package org.opensearch.securityanalytics.logtype; | ||
|
||
import java.util.List; | ||
import java.util.Map; | ||
import java.util.Optional; | ||
import java.util.stream.Collectors; | ||
import org.apache.logging.log4j.LogManager; | ||
import org.apache.logging.log4j.Logger; | ||
import org.opensearch.securityanalytics.model.LogType; | ||
import org.opensearch.securityanalytics.util.SecurityAnalyticsException; | ||
|
||
|
||
/** | ||
* | ||
* */ | ||
public class LogTypeService { | ||
|
||
private static final Logger logger = LogManager.getLogger(LogTypeService.class); | ||
|
||
private BuiltinLogTypeLoader builtinLogTypeLoader; | ||
|
||
public LogTypeService() { | ||
this.builtinLogTypeLoader = new BuiltinLogTypeLoader(); | ||
} | ||
|
||
|
||
public List<LogType> getAllLogTypes() { | ||
return BuiltinLogTypeLoader.getAllLogTypes(); | ||
} | ||
|
||
public LogType getLogTypeByName(String logType) { | ||
return BuiltinLogTypeLoader.getLogTypeByName(logType); | ||
} | ||
|
||
/** | ||
* Returns sigmaRule rawField to ECS field mapping | ||
* | ||
* @param logType Log type | ||
* @return Map of rawField to ecs field | ||
*/ | ||
public Map<String, String> getRuleFieldMappings(String logType) { | ||
LogType lt = getLogTypeByName(logType); | ||
|
||
if (lt == null) { | ||
throw SecurityAnalyticsException.wrap(new IllegalArgumentException("Can't get rule field mappings for invalid logType: [" + logType + "]")); | ||
} | ||
if (lt.getMappings() == null) { | ||
return Map.of(); | ||
} else { | ||
return lt.getMappings() | ||
.stream() | ||
.collect(Collectors.toMap(LogType.Mapping::getRawField, LogType.Mapping::getEcs)); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,137 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
package org.opensearch.securityanalytics.model; | ||
|
||
import java.io.IOException; | ||
import java.util.ArrayList; | ||
import java.util.List; | ||
import java.util.Map; | ||
import java.util.stream.Collectors; | ||
import org.opensearch.common.io.stream.StreamInput; | ||
import org.opensearch.common.io.stream.StreamOutput; | ||
import org.opensearch.common.io.stream.Writeable; | ||
import org.opensearch.core.xcontent.ToXContentObject; | ||
import org.opensearch.core.xcontent.XContentBuilder; | ||
|
||
public class LogType implements Writeable { | ||
|
||
private static final String ID = "id"; | ||
private static final String NAME = "name"; | ||
private static final String DESCRIPTION = "description"; | ||
private static final String IS_BUILTIN = "is_builtin"; | ||
private static final String MAPPINGS = "mappings"; | ||
private static final String RAW_FIELD = "raw_field"; | ||
private static final String ECS = "ecs"; | ||
private static final String OCSF = "ocsf"; | ||
|
||
private String id; | ||
private String name; | ||
private String description; | ||
private Boolean isBuiltIn; | ||
private List<Mapping> mappings; | ||
|
||
public LogType(StreamInput sin) throws IOException { | ||
this.id = sin.readString(); | ||
this.isBuiltIn = sin.readOptionalBoolean(); | ||
this.name = sin.readString(); | ||
this.description = sin.readString(); | ||
this.mappings = sin.readList(Mapping::readFrom); | ||
} | ||
|
||
public LogType(String id, String name, String description, boolean isBuiltIn, List<Mapping> mappings) { | ||
this.id = id; | ||
this.name = name; | ||
this.description = description; | ||
this.isBuiltIn = isBuiltIn; | ||
this.mappings = mappings == null ? List.of() : mappings; | ||
} | ||
|
||
public LogType(Map<String, Object> logTypeAsMap) { | ||
this.id = (String) logTypeAsMap.get(ID); | ||
this.name = (String) logTypeAsMap.get(NAME); | ||
this.description = (String) logTypeAsMap.get(DESCRIPTION); | ||
if (logTypeAsMap.containsKey(IS_BUILTIN)) { | ||
this.isBuiltIn = (Boolean) logTypeAsMap.get(IS_BUILTIN); | ||
} | ||
List<Map<String, String>> mappings = (List<Map<String, String>>)logTypeAsMap.get(MAPPINGS); | ||
if (mappings.size() > 0) { | ||
this.mappings = new ArrayList<>(mappings.size()); | ||
this.mappings = mappings.stream().map(e -> | ||
new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF)) | ||
).collect(Collectors.toList()); | ||
} | ||
} | ||
|
||
public String getName() { | ||
return name; | ||
} | ||
|
||
public String getDescription() { | ||
return description; | ||
} | ||
|
||
public boolean getIsBuiltIn() { return isBuiltIn; } | ||
|
||
public List<Mapping> getMappings() { | ||
return mappings; | ||
} | ||
|
||
@Override | ||
public void writeTo(StreamOutput out) throws IOException { | ||
out.writeString(id); | ||
out.writeOptionalBoolean(isBuiltIn); | ||
out.writeString(name); | ||
out.writeString(description); | ||
out.writeCollection(mappings); | ||
} | ||
|
||
@Override | ||
public String toString() { | ||
return name; | ||
} | ||
|
||
public static class Mapping implements Writeable { | ||
|
||
private String rawField; | ||
private String ecs; | ||
private String ocsf; | ||
|
||
public Mapping(StreamInput sin) throws IOException { | ||
this.rawField = sin.readString(); | ||
this.ecs = sin.readOptionalString(); | ||
this.ocsf = sin.readOptionalString(); | ||
} | ||
|
||
public Mapping(String rawField, String ecs, String ocsf) { | ||
this.rawField = rawField; | ||
this.ecs = ecs; | ||
this.ocsf = ocsf; | ||
} | ||
|
||
public String getRawField() { | ||
return rawField; | ||
} | ||
|
||
public String getEcs() { | ||
return ecs; | ||
} | ||
|
||
public String getOcsf() { | ||
return ocsf; | ||
} | ||
|
||
@Override | ||
public void writeTo(StreamOutput out) throws IOException { | ||
out.writeString(rawField); | ||
out.writeOptionalString(ecs); | ||
out.writeOptionalString(ocsf); | ||
} | ||
|
||
public static Mapping readFrom(StreamInput sin) throws IOException { | ||
return new Mapping(sin); | ||
} | ||
} | ||
|
||
} |