Skip to content

Commit

Permalink
[kAAv6pCq] Extra cherry-picks
Browse files Browse the repository at this point in the history
  • Loading branch information
@gem-neo4j
gem-neo4j committed Sep 29, 2023
1 parent baa5577 commit 985db08
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 69 deletions.
72 changes: 31 additions & 41 deletions core/src/test/java/apoc/export/cypher/ExportCypherTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,9 @@
import apoc.util.TestUtil;
import apoc.util.Util;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
Expand All @@ -49,6 +47,8 @@
import java.util.stream.Stream;

import static apoc.export.cypher.ExportCypherTest.ExportCypherResults.*;
import static apoc.export.cypher.ExportCypherTest.ExportCypherResults.EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED3;
import static apoc.export.cypher.ExportCypherTest.ExportCypherResults.EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED4;
import static apoc.export.util.ExportFormat.*;
import static apoc.util.BinaryTestUtil.getDecompressedData;
import static apoc.util.TestUtil.assertError;
Expand Down Expand Up @@ -773,8 +773,7 @@ public void testExportWithCompressionQueryCypherShellUnwindBatchParamsWithOddDat
}

@Test
@Ignore("non-deterministic index order")
public void testExportAllCypherPlainOptimized() throws Exception {
public void testExportAllCypherPlainOptimized() {
String fileName = "queryPlainOptimized.cypher";
TestUtil.testCall(db, "CALL apoc.export.cypher.query('MATCH (f:Foo)-[r:KNOWS]->(b:Bar) return f,r,b', $file,{format:'cypher-shell', useOptimizations: {type: 'unwind_batch'}})",
map("file", fileName),
Expand All @@ -788,7 +787,12 @@ public void testExportAllCypherPlainOptimized() throws Exception {
assertTrue("Should get time greater than 0",((long) r.get("time")) >= 0);
});
String actual = readFile(fileName);
assertTrue("expected generated output",EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED.equals(actual) || EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED2.equals(actual));

assertTrue("expected generated output ",
EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED.equals(actual) ||
EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED2.equals(actual) ||
EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED3.equals(actual) ||
EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED4.equals(actual));
}

@Test
Expand Down Expand Up @@ -836,32 +840,6 @@ public void exportMultiTokenIndex() {
});
}

@Ignore("It doesn't fail anymore because it skips not supported indexes")
@Test(expected = QueryExecutionException.class)
public void shouldFailExportMultiTokenIndexForRelationship() {
// given
db.executeTransactionally("CREATE (n:TempNode {value:'value'})");
db.executeTransactionally("CREATE (n:TempNode2 {value:'value'})");
db.executeTransactionally("CALL db.index.fulltext.createNodeIndex('MyCoolNodeFulltextIndex',['TempNode', 'TempNode2'],['value'])");

// TODO: We can't manage full-text rel indexes because of this bug: https://github.com/neo4j/neo4j/issues/12304
db.executeTransactionally("CREATE (s:TempNode)-[:REL{rel_value: 'the rel value'}]->(e:TempNode2)");
db.executeTransactionally("CALL db.index.fulltext.createRelationshipIndex('MyCoolRelFulltextIndex',['REL'],['rel_value'])");
String query = "MATCH (t:TempNode) return t";
Map<String, Object> config = map("awaitForIndexes", 3000);

try {
// when
TestUtil.testCall(db, "CALL apoc.export.cypher.query($query, $file, $config)",
map("query", query, "file", null, "config", config),
(r) -> {});
} catch (Exception e) {
String expected = "Full-text indexes on relationships are not supported, please delete them in order to complete the process";
assertEquals(expected, ExceptionUtils.getRootCause(e).getMessage());
throw e;
}
}

@Test
public void shouldNotCreateUniqueImportIdForUniqueConstraint() {
db.executeTransactionally("CREATE (n:Bar:Baz{name: 'A'})");
Expand Down Expand Up @@ -1380,6 +1358,20 @@ public static class ExportCypherResults {
"CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" +
"COMMIT%n");

static final String EXPECTED_QUERY_NODES_OPTIMIZED3 = String.format("BEGIN%n" +
"UNWIND [{_id:0, properties:{born:date('2018-10-31'), name:\"foo\"}}, {_id:4, properties:{born:date('2017-09-29'), name:\"foo2\"}}] AS row%n" +
"CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Foo;%n" +
"UNWIND [{name:\"bar2\", properties:{age:44}}, {name:\"bar\", properties:{age:42}}] AS row%n" +
"CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" +
"COMMIT%n");

static final String EXPECTED_QUERY_NODES_OPTIMIZED4 = String.format("BEGIN%n" +
"UNWIND [{_id:4, properties:{born:date('2017-09-29'), name:\"foo2\"}}, {_id:0, properties:{born:date('2018-10-31'), name:\"foo\"}}] AS row%n" +
"CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Foo;%n" +
"UNWIND [{name:\"bar2\", properties:{age:44}}, {name:\"bar\", properties:{age:42}}] AS row%n" +
"CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" +
"COMMIT%n");

public static final String EXPECTED_RELATIONSHIPS_OPTIMIZED = String.format("BEGIN%n" +
"UNWIND [{start: {_id:0}, end: {name:\"bar\"}, properties:{since:2016}}, {start: {_id:4}, end: {name:\"bar2\"}, properties:{since:2015}}] AS row%n" +
"MATCH (start:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row.start._id})%n" +
Expand Down Expand Up @@ -1542,6 +1534,8 @@ public static class ExportCypherResults {

static final String EXPECTED_QUERY_NODES = EXPECTED_SCHEMA_OPTIMIZED + EXPECTED_QUERY_NODES_OPTIMIZED + EXPECTED_RELATIONSHIPS_OPTIMIZED + DROP_UNIQUE_OPTIMIZED;
static final String EXPECTED_QUERY_NODES2 = EXPECTED_SCHEMA_OPTIMIZED + EXPECTED_QUERY_NODES_OPTIMIZED2 + EXPECTED_RELATIONSHIPS_OPTIMIZED + DROP_UNIQUE_OPTIMIZED;
static final String EXPECTED_QUERY_NODES3 = EXPECTED_SCHEMA + EXPECTED_QUERY_NODES_OPTIMIZED3 + EXPECTED_RELATIONSHIPS_OPTIMIZED + EXPECTED_CLEAN_UP;
static final String EXPECTED_QUERY_NODES4 = EXPECTED_SCHEMA + EXPECTED_QUERY_NODES_OPTIMIZED4 + EXPECTED_RELATIONSHIPS_OPTIMIZED + EXPECTED_CLEAN_UP;

static final String EXPECTED_CYPHER_OPTIMIZED_BATCH_SIZE_UNWIND = EXPECTED_SCHEMA_OPTIMIZED + EXPECTED_NODES_OPTIMIZED_BATCH_SIZE_UNWIND + EXPECTED_RELATIONSHIPS_OPTIMIZED + DROP_UNIQUE_OPTIMIZED_BATCH;

Expand Down Expand Up @@ -1569,17 +1563,13 @@ public static class ExportCypherResults {
.replace(NEO4J_SHELL.schemaAwait(), EXPECTED_INDEXES_AWAIT)
.replace(NEO4J_SHELL.schemaAwait(), CYPHER_SHELL.schemaAwait());

static final String EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED = EXPECTED_QUERY_NODES
.replace(NEO4J_SHELL.begin(), CYPHER_SHELL.begin())
.replace(NEO4J_SHELL.commit(), CYPHER_SHELL.commit())
.replace(NEO4J_SHELL.schemaAwait(), EXPECTED_INDEXES_AWAIT_QUERY)
.replace(NEO4J_SHELL.schemaAwait(), CYPHER_SHELL.schemaAwait());
static final String EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED = convertToCypherShellFormat(EXPECTED_QUERY_NODES);

static final String EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED2 = EXPECTED_QUERY_NODES2
.replace(NEO4J_SHELL.begin(), CYPHER_SHELL.begin())
.replace(NEO4J_SHELL.commit(), CYPHER_SHELL.commit())
.replace(NEO4J_SHELL.schemaAwait(), EXPECTED_INDEXES_AWAIT_QUERY)
.replace(NEO4J_SHELL.schemaAwait(), CYPHER_SHELL.schemaAwait());
static final String EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED2 = convertToCypherShellFormat(EXPECTED_QUERY_NODES2);

static final String EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED3 = convertToCypherShellFormat(EXPECTED_QUERY_NODES3);

static final String EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED4 = convertToCypherShellFormat(EXPECTED_QUERY_NODES4);

public static final String EXPECTED_CYPHER_SHELL_OPTIMIZED = EXPECTED_NEO4J_SHELL_OPTIMIZED
.replace(NEO4J_SHELL.begin(), CYPHER_SHELL.begin())
Expand Down
28 changes: 15 additions & 13 deletions full-it/src/test/java/apoc/full/it/SystemDbEnterpriseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,52 +21,52 @@
import apoc.util.Neo4jContainerExtension;
import apoc.util.TestContainerUtil;
import apoc.util.TestUtil;
import org.apache.commons.io.FileUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.neo4j.driver.Session;
import org.neo4j.driver.types.Node;

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.UUID;

import static apoc.systemdb.SystemDb.REMOTE_SENSITIVE_PROP;
import static apoc.systemdb.SystemDb.USER_SENSITIVE_PROP;
import static apoc.util.TestContainerUtil.createEnterpriseDB;
import static apoc.util.TestContainerUtil.importFolder;
import static org.junit.Assert.assertTrue;

public class SystemDbEnterpriseTest {
private static final String USERNAME = "nonadmin";
private static final String PASSWORD = "keystore-password";

private static final String KEYSTORE_NAME_PKCS_12 = "keystore-name.pkcs12";
// we put the keystore file in the import folder for simplicity (because it's bound during the creation of the container)
private static final File KEYSTORE_FILE = new File("import", KEYSTORE_NAME_PKCS_12);

private static Neo4jContainerExtension neo4jContainer;
private static Session session;



@BeforeClass
public static void beforeClass() throws Exception {
final String randomKeyAlias = UUID.randomUUID().toString();

// we put the keystore file in the import folder for simplicity (because it's bound during the creation of the container)
final String keystoreName = "keystore-name.pkcs12";
final File keystoreFile = new File(importFolder, keystoreName);

// certificate file creation
final String[] args = new String[] { "keytool",
"-genseckey", "-keyalg", "aes", "-keysize", "256", "-storetype", "pkcs12",
"-keystore", keystoreFile.getCanonicalPath(),
"-keystore", KEYSTORE_FILE.getCanonicalPath(),
"-alias", randomKeyAlias,
"-storepass", PASSWORD};

Process proc = new ProcessBuilder(args).start();
proc.waitFor();

// We build the project, the artifact will be placed into ./build/libs
final String pathPwdValue = "/var/lib/neo4j/import/" + keystoreName;
// we add config useful to create a remote db alias
final String pathPwdValue = "/var/lib/neo4j/import/" + KEYSTORE_FILE.getName();

// we add config useful to create a remote db alias
neo4jContainer = createEnterpriseDB(List.of(TestContainerUtil.ApocPackage.FULL), !TestUtil.isRunningInCI())
.withNeo4jConfig("systemdb.secrets.keystore.path", pathPwdValue)
.withNeo4jConfig("systemdb.secrets.keystore.password", PASSWORD)
Expand All @@ -76,9 +76,11 @@ public static void beforeClass() throws Exception {
session = neo4jContainer.getSession();

}

@AfterClass
public static void afterClass() {
public static void afterClass() throws IOException {
FileUtils.forceDelete(KEYSTORE_FILE);

session.close();
neo4jContainer.close();
}
Expand Down
18 changes: 4 additions & 14 deletions full/src/main/java/apoc/systemdb/SystemDb.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,22 +23,20 @@
import apoc.Extended;
import apoc.export.cypher.ExportFileManager;
import apoc.export.cypher.FileManagerFactory;
import apoc.export.util.ExportConfig;
import apoc.export.util.ProgressReporter;
import apoc.result.ProgressInfo;
import apoc.result.RowResult;
import apoc.result.VirtualNode;
import apoc.result.VirtualRelationship;
import apoc.systemdb.metadata.ExportMetadata;
import apoc.util.Util;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Label;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.Transaction;
import org.neo4j.internal.helpers.collection.Iterables;
import org.neo4j.internal.helpers.collection.Pair;
import org.neo4j.internal.kernel.api.procs.ProcedureCallContext;
import org.neo4j.internal.kernel.api.security.SecurityContext;
import org.neo4j.kernel.impl.coreapi.TransactionImpl;
import org.neo4j.procedure.Admin;
import org.neo4j.procedure.Context;
Expand All @@ -57,7 +55,6 @@
import java.util.stream.Stream;
import java.util.stream.StreamSupport;


@Extended
public class SystemDb {
public static final String REMOTE_SENSITIVE_PROP = "password";
Expand All @@ -66,12 +63,6 @@ public class SystemDb {
@Context
public ApocConfig apocConfig;

@Context
public SecurityContext securityContext;

@Context
public ProcedureCallContext callContext;

@Context
public GraphDatabaseService db;

Expand All @@ -95,7 +86,7 @@ public Stream<ProgressInfo> metadata(@Name(value = "config",defaultValue = "{}")

ProgressInfo progressInfo = new ProgressInfo(fileName, null, "cypher");
ProgressReporter progressReporter = new ProgressReporter(null, null, progressInfo);
ExportFileManager cypherFileManager = FileManagerFactory.createFileManager(fileName + ".cypher", true);
ExportFileManager cypherFileManager = FileManagerFactory.createFileManager(fileName + ".cypher", true, ExportConfig.EMPTY);
withSystemDbTransaction(tx -> {
tx.getAllNodes()
.stream()
Expand All @@ -121,9 +112,9 @@ public Stream<ProgressInfo> metadata(@Name(value = "config",defaultValue = "{}")
return progressReporter.stream();
}

@Admin
@Procedure
public Stream<NodesAndRelationshipsResult> graph() {
Util.checkAdmin(securityContext, callContext,"apoc.systemdb.graph");
return withSystemDbTransaction(tx -> {
Map<Long, Node> virtualNodes = new HashMap<>();
for (Node node: tx.getAllNodes()) {
Expand All @@ -145,10 +136,9 @@ public Stream<NodesAndRelationshipsResult> graph() {
});
}

@Admin
@Procedure
public Stream<RowResult> execute(@Name("DDL commands, either a string or a list of strings") Object ddlStringOrList, @Name(value="params", defaultValue = "{}") Map<String ,Object> params) {
Util.checkAdmin(securityContext, callContext, "apoc.systemdb.execute");

List<String> commands;
if (ddlStringOrList instanceof String) {
commands = Collections.singletonList((String)ddlStringOrList);
Expand Down
1 change: 0 additions & 1 deletion test-utils/src/main/java/apoc/util/TestContainerUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,6 @@ public static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> apo
.withNeo4jConfig("dbms.logs.http.enabled", "true")
.withNeo4jConfig("dbms.logs.debug.level", "DEBUG")
.withNeo4jConfig("dbms.routing.driver.logging.level", "DEBUG")
.withNeo4jConfig("internal.dbms.type_constraints", "true")
.withFileSystemBind(canonicalPath, "/var/lib/neo4j/import") // map the "target/import" dir as the Neo4j's import dir
.withCreateContainerCmdModifier(cmd -> cmd.withMemory(2024 * 1024 * 1024L)) // 2gb
.withExposedPorts(7687, 7473, 7474)
Expand Down

0 comments on commit 985db08

Please sign in to comment.