From 64ee10baacb608df28ea4c234a223637dd202762 Mon Sep 17 00:00:00 2001 From: thinkAfCod Date: Mon, 31 Jul 2023 16:07:05 +0800 Subject: [PATCH] create hildr-batcher project --- .gitignore | 1 + hildr-batcher/build.gradle | 266 ++++++++++++++++++ .../main/java/io/optimism/HildrBatcher.java | 41 +++ .../batcher/compressor/Compressor.java | 64 +++++ .../batcher/compressor/CompressorFactory.java | 28 ++ .../batcher/compressor/Compressors.java | 39 +++ .../optimism/batcher/compressor/Config.java | 35 +++ .../batcher/compressor/RatioCompressor.java | 66 +++++ .../batcher/compressor/ShadowCompressor.java | 65 +++++ .../ex/CompressorFullException.java | 36 +++ settings.gradle | 1 + 11 files changed, 642 insertions(+) create mode 100644 hildr-batcher/build.gradle create mode 100644 hildr-batcher/src/main/java/io/optimism/HildrBatcher.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressor.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/CompressorFactory.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressors.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/Config.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/RatioCompressor.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/ShadowCompressor.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/ex/CompressorFullException.java diff --git a/.gitignore b/.gitignore index 30d1d4d5..6ba0fa28 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ # Ignore Gradle build output directory build .DS_Store +.vscode */logs *.gz *.log \ No newline at end of file diff --git a/hildr-batcher/build.gradle b/hildr-batcher/build.gradle new file mode 100644 index 00000000..00716239 --- /dev/null +++ b/hildr-batcher/build.gradle @@ -0,0 +1,266 @@ +plugins { + id 'java' + id 'application' + id 'checkstyle' + id "jacoco" + id "com.diffplug.spotless" version "6.19.0" + id "net.ltgt.errorprone" version "3.1.0" + id 'org.graalvm.buildtools.native' version '0.9.22' +} + +group = 'io.optimism' +version = '0.1.0' + +repositories { + // Use Maven Central for resolving dependencies. + mavenCentral() + maven { + url "https://artifacts.consensys.net/public/teku/maven/" + } + maven { + url "https://dl.cloudsmith.io/public/libp2p/jvm-libp2p/maven/" + } + maven { + url "https://hyperledger.jfrog.io/artifactory/besu-maven/" + } + maven { + url "https://artifacts.consensys.net/public/maven/maven/" + } +} + +application { + // Define the main class for the application. + mainClass = 'io.optimism.HildrBatcher' +} + +dependencies { + // This dependency is used by the application. + implementation 'com.google.guava:guava:31.1-jre' + implementation 'com.github.rholder:guava-retrying:2.0.0' + + // define any required OkHttp artifacts without version + implementation("com.squareup.okhttp3:okhttp:5.0.0-alpha.2") + implementation("com.squareup.okhttp3:logging-interceptor:5.0.0-alpha.2") + + implementation('org.web3j:core:4.9.8') { + exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on' + exclude group: 'com.squareup.okhttp3', module: 'okhttp' + exclude group: 'com.squareup.okhttp3', module: 'logging-interceptor' + } + implementation('net.osslabz.evm:evm-abi-decoder:0.0.6') + implementation 'com.github.gestalt-config:gestalt-core:0.20.4' + implementation 'com.github.gestalt-config:gestalt-toml:0.20.4' + + implementation 'com.fasterxml.jackson:jackson-bom:2.15.2' + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml' + implementation 'org.jctools:jctools-core:4.0.1' + + implementation 'io.jsonwebtoken:jjwt-api:0.11.5' + implementation 'io.jsonwebtoken:jjwt-impl:0.11.5' + implementation 'io.jsonwebtoken:jjwt-jackson:0.11.5' + + //jsonrpc + implementation('io.vertx:vertx-auth-jwt:4.4.2') + implementation('io.vertx:vertx-core:4.4.2') + implementation('io.vertx:vertx-web:4.4.2') + + implementation 'io.micrometer:micrometer-registry-prometheus:1.11.0' + implementation platform('io.micrometer:micrometer-tracing-bom:1.1.1') + implementation 'io.micrometer:micrometer-tracing' + implementation 'io.micrometer:micrometer-tracing-bridge-otel' + + // Logback + implementation 'ch.qos.logback:logback-core:1.4.7' + implementation 'ch.qos.logback:logback-classic:1.4.7' + implementation 'org.slf4j:slf4j-api:2.0.7' + + implementation platform("io.opentelemetry:opentelemetry-bom-alpha:1.26.0-alpha") + // OpenTelemetry core + implementation(platform("io.opentelemetry:opentelemetry-bom:1.26.0")) + implementation 'io.opentelemetry:opentelemetry-api' + implementation 'io.opentelemetry:opentelemetry-sdk' + implementation 'io.opentelemetry:opentelemetry-sdk-logs' + + // OpenTelemetry log4j appenders + implementation platform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha:1.26.0-alpha") + runtimeOnly 'io.opentelemetry.instrumentation:opentelemetry-logback-mdc-1.0' + + + implementation 'info.picocli:picocli:4.7.3' + annotationProcessor 'info.picocli:picocli-codegen:4.7.3' + +// implementation fileTree(dir: '../lib', include: '*.jar') + // Use JUnit Jupiter for testing. + testImplementation 'org.junit.jupiter:junit-jupiter:5.9.1' + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.9.0' + // https://mvnrepository.com/artifact/org.junit.platform/junit-platform-suite-api + testImplementation 'org.junit.platform:junit-platform-suite-api:1.9.1' + // https://mvnrepository.com/artifact/org.junit.platform/junit-platform-suite-engine + testRuntimeOnly 'org.junit.platform:junit-platform-suite-engine:1.9.1' + + testRuntimeOnly 'org.junit.platform:junit-platform-reporting:1.9.1' + + testImplementation 'org.mockito:mockito-junit-jupiter:2.19.0' + testImplementation("com.squareup.okhttp3:mockwebserver:5.0.0-alpha.2") + + errorprone("com.google.errorprone:error_prone_core:2.18.0") +} + +// Apply a specific Java toolchain to ease working on different environments. +java { + toolchain { + languageVersion = JavaLanguageVersion.of(20) + } +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } + finalizedBy jacocoTestReport +} + +jacoco { + toolVersion = "0.8.9" +} + +jacocoTestReport { + dependsOn test + + reports { + csv.required = true + } +} + +jacocoTestCoverageVerification { + + violationRules { + rule { + limit { + minimum = 0 + } + } + } +} + +checkstyle { + toolVersion = '10.10.0' + // default checkstyle config -- specific to your team agreement + configFile = project(":").file("config/checkstyle/google_checks.xml") + // Google style (idiosyncratic to Google): + // configFile = project(":").file("config/checkstyle/google_checks.xml") + // SUN style (closest to modern Java styles) -- the basis for this project: + // configFile = project(":").file("config/checkstyle/sun_checks.xml") +// ignoreFailures = false +// maxWarnings = 0 +} + +spotless { + // optional: limit format enforcement to just the files changed by this feature branch +// ratchetFrom 'origin/main' + + format 'misc', { + // define the files to apply `misc` to + target '*.gradle', '*.md', '.gitignore' + + // define the steps to apply to those files + trimTrailingWhitespace() + indentWithTabs() // or spaces. Takes an integer argument if you don't like 4 + endWithNewline() + } + java { + // Use the default importOrder configuration + + // don't need to set target, it is inferred from java + + // apply a specific flavor of google-java-format + googleJavaFormat('1.17.0') + // fix formatting of type annotations + formatAnnotations() + // make sure every file has the following copyright header. + // optionally, Spotless can set copyright years by digging + // through git history (see "license" section below) + licenseHeaderFile(project(":").file("config/spotless/java.license")).named('hildr').onlyIfContentMatches('/*\n' + + ' * Copyright 2023 281165273grape@gmail.com') + licenseHeaderFile(project(":").file("config/spotless/besu.license")).named('besu').onlyIfContentMatches('/*\n' + + ' * Copyright ConsenSys AG') + + importOrder() + + removeUnusedImports() + } +} + +checkstyleMain + .exclude('io/optimism/rpc/handler/TimeoutHandler.java') + .exclude('io/optimism/rpc/handler/JsonRpcExecutorHandler.java') + .exclude('io/optimism/rpc/handler/JsonRpcParseHandler.java') + .exclude('io/optimism/rpc/methods/JsonRpcMethod.java') + .exclude('io/optimism/rpc/methods/JsonRpcMethodsFactory.java') + .exclude('io/optimism/rpc/methods/JsonRpcProcessor.java') + .exclude('io/optimism/rpc/execution/LoggedJsonRpcProcessor.java') + .exclude('io/optimism/rpc/internal/JsonRpcRequest.java') + .exclude('io/optimism/rpc/internal/JsonRpcRequestContext.java') + .exclude('io/optimism/rpc/internal/JsonRpcRequestId.java') + .exclude('io/optimism/rpc/internal/response/JsonRpcResponse.java') + .exclude('io/optimism/rpc/internal/response/JsonRpcErrorResponse.java') + .exclude('io/optimism/rpc/internal/response/JsonRpcError.java') + .exclude('io/optimism/rpc/internal/response/JsonRpcResponseType.java') + .exclude('io/optimism/rpc/internal/response/JsonRpcSuccessResponse.java') + .exclude('io/optimism/rpc/internal/response/JsonRpcNoResponse.java') + .exclude('io/optimism/rpc/execution/BaseJsonRpcProcessor.java') + .exclude('io/optimism/rpc/execution/JsonRpcProcessor.java') +//testSets { +// integrationTest +//} + +tasks.named('test') { + // Use JUnit Platform for unit tests. + useJUnitPlatform() +} + +check { + dependsOn += jacocoTestCoverageVerification +// dependsOn += integrationTest +} + +tasks.withType(Test).configureEach { + def outputDir = reports.junitXml.outputLocation + jvmArgumentProviders << ({ + [ + "-Djunit.platform.reporting.open.xml.enabled=true", + "-Djunit.platform.reporting.output.dir=${outputDir.get().asFile.absolutePath}", + "--enable-preview" + ] + } as CommandLineArgumentProvider) +} + +java { + withJavadocJar() + withSourcesJar() +} + + +javadoc { + if (JavaVersion.current().isJava9Compatible()) { + options.addBooleanOption('html5', true) + } + options.addBooleanOption('-enable-preview', true) + options.addStringOption('-release', '20') + options.addStringOption('-add-modules', 'jdk.incubator.concurrent') +} + +jar { + manifest { + attributes "Main-Class": "io.optimism.HildrBatcher" + } + + duplicatesStrategy = DuplicatesStrategy.WARN + from { + configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } + } { + exclude 'META-INF/*.DSA', 'META-INF/*.RSA', 'META-INF/*.SF' + } +} diff --git a/hildr-batcher/src/main/java/io/optimism/HildrBatcher.java b/hildr-batcher/src/main/java/io/optimism/HildrBatcher.java new file mode 100644 index 00000000..8c9af866 --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/HildrBatcher.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 q315xia@163.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.optimism; + +/** + * Batcher main method. + * + * @author thinkAfCod + * @since 0.1.1 + */ +public class HildrBatcher { + + /** Constructor of HildrBatcher. */ + public HildrBatcher() {} + + /** + * Main method of HildrBatcher. + * + * @param args Starts arguments + */ + public static void main(String[] args) { + // todo start batcherSubmitter + // todo start server + // todo start metrics server + // todo listen close signal + } +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressor.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressor.java new file mode 100644 index 00000000..571c3122 --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressor.java @@ -0,0 +1,64 @@ +/* + * Copyright 2023 q315xia@163.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.optimism.batcher.compressor; + +import java.io.Closeable; +import java.io.Flushable; + +/** + * Tx data bytes compressor interface. + * + * @author thinkAfCod + * @since 0.1.1 + */ +public interface Compressor extends Closeable, Readable, Flushable { + + /** + * write uncompressed data which will be compressed. Should return CompressorFullException if the + * compressor is full and no more data should be written. + * + * @param p uncompressed data + * @return length of compressed data + */ + int write(byte[] p); + + /** + * read compressed data; should only be called after Close. + * + * @param p read buffer bytes to this byte array + * @return length of read compressed data. + */ + int read(byte[] p); + + /** reset all written data. */ + void reset(); + + /** + * returns an estimate of the current length of the compressed data; calling Flush will. increase + * the accuracy at the expense of a poorer compression ratio. + * + * @return an estimate of the current length of the compressed data + */ + int length(); + + /** + * returns CompressorFullException if the compressor is known to be full. Note that calls to Write + * will fail if an error is returned from this method, but calls to Write can still return + * CompressorFullErr even if this does not. + */ + void fullErr(); +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/CompressorFactory.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/CompressorFactory.java new file mode 100644 index 00000000..fc80e7f2 --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/CompressorFactory.java @@ -0,0 +1,28 @@ +/* + * Copyright 2023 q315xia@163.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.optimism.batcher.compressor; + +/** + * Compressor Factory. + * + * @author thinkAfCod + * @since 0.1.1 + */ +public class CompressorFactory { + + private CompressorFactory() {} +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressors.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressors.java new file mode 100644 index 00000000..345b907b --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressors.java @@ -0,0 +1,39 @@ +/* + * Copyright 2023 q315xia@163.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.optimism.batcher.compressor; + +/** + * Compressor create tool. + * + * @author thinkAfCod + * @since 0.1.1 + */ +public interface Compressors { + + /** Kind type of ratio. */ + String RatioKind = "ratio"; + + /** Kind type of shadow. */ + String ShadowKind = "shadow"; + + /** + * Create Compressor by kind. + * + * @param kind Type of Compressor + */ + default void create(String kind) {} +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Config.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Config.java new file mode 100644 index 00000000..1501386d --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Config.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 q315xia@163.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.optimism.batcher.compressor; + +/** + * Compressor Config. + * + * @author thinkAfCod + * @since 0.1.1 + * @param targetFrameSize To target when creating channel frames. Note that if the realized + * compression ratio is worse than the approximate, more frames may actually be created. This + * also depends on how close the target is to the max frame size. + * @param targetNumFrame To create in this channel. If the realized compression ratio is worse than + * approxComprRatio, additional leftover frame(s) might get created. + * @param approxComprRatio ApproxComprRatio to assume. Should be slightly smaller than average from + * experiments to avoid the chances of creating a small additional leftover frame. + * @param kind Kind of compressor to use. Must be one of KindKeys. If unset, NewCompressor will + * default to RatioKind. + */ +public record Config( + long targetFrameSize, int targetNumFrame, double approxComprRatio, String kind) {} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/RatioCompressor.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/RatioCompressor.java new file mode 100644 index 00000000..0122d86a --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/RatioCompressor.java @@ -0,0 +1,66 @@ +/* + * Copyright 2023 q315xia@163.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.optimism.batcher.compressor; + +import java.io.IOException; +import java.nio.CharBuffer; +import org.jetbrains.annotations.NotNull; + +/** + * RatioCompressor class. + * + * @author thinkAfCod + * @since 0.1.1 + */ +public class RatioCompressor implements Compressor { + + RatioCompressor() { + // todo plan to use java.util.zip.Deflater + } + + @Override + public int write(byte[] p) { + return 0; + } + + @Override + public int read(byte[] p) { + return 0; + } + + @Override + public int read(@NotNull CharBuffer cb) throws IOException { + return 0; + } + + @Override + public void reset() {} + + @Override + public int length() { + return 0; + } + + @Override + public void fullErr() {} + + @Override + public void close() throws IOException {} + + @Override + public void flush() throws IOException {} +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ShadowCompressor.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ShadowCompressor.java new file mode 100644 index 00000000..9550fbf1 --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ShadowCompressor.java @@ -0,0 +1,65 @@ +/* + * Copyright 2023 q315xia@163.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.optimism.batcher.compressor; + +import java.io.IOException; +import java.nio.CharBuffer; +import org.jetbrains.annotations.NotNull; + +/** + * ShadowCompressor class. + * + * @author thinkAfCod + * @since 0.1.1 + */ +public class ShadowCompressor implements Compressor { + + /** Constructor of ShadowCompressor. */ + public ShadowCompressor() {} + + @Override + public int write(byte[] p) { + return 0; + } + + @Override + public int read(byte[] p) { + return 0; + } + + @Override + public int read(@NotNull CharBuffer cb) throws IOException { + return 0; + } + + @Override + public void reset() {} + + @Override + public int length() { + return 0; + } + + @Override + public void fullErr() {} + + @Override + public void close() throws IOException {} + + @Override + public void flush() throws IOException {} +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ex/CompressorFullException.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ex/CompressorFullException.java new file mode 100644 index 00000000..5f321196 --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ex/CompressorFullException.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 q315xia@163.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.optimism.batcher.compressor.ex; + +/** + * If the compressor is full and no more data should be written or the compressor is known to be + * full. + * + * @author thinkAfCod + * @since 0.1.1 + */ +public class CompressorFullException extends RuntimeException { + + /** + * Constructor of CompressorFullException. + * + * @param message error message + */ + public CompressorFullException(String message) { + super(message); + } +} diff --git a/settings.gradle b/settings.gradle index 4509c03b..b610e942 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,3 +16,4 @@ pluginManagement { rootProject.name = 'hildr' include('hildr-node') +include('hildr-batcher')