From 8cbd757091b0709a0e1e0326e3ffe1f5df870213 Mon Sep 17 00:00:00 2001 From: thinkAfCod Date: Mon, 24 Jul 2023 20:54:07 +0800 Subject: [PATCH] batcher commit --- README.md | 2 +- build.gradle | 566 +++++++++--------- hildr-batcher/build.gradle | 10 +- .../main/java/io/optimism/HildrBatcher.java | 1 - .../batcher/compressor/Compressor.java | 39 ++ .../batcher/compressor/CompressorFactory.java | 22 + .../batcher/compressor/Compressors.java | 13 + .../optimism/batcher/compressor/Config.java | 17 + .../batcher/compressor/RatioCompressor.java | 48 ++ .../batcher/compressor/ShadowCompressor.java | 44 ++ .../ex/CompressorFullException.java | 15 + settings.gradle | 8 +- 12 files changed, 489 insertions(+), 296 deletions(-) create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressor.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/CompressorFactory.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressors.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/Config.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/RatioCompressor.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/ShadowCompressor.java create mode 100644 hildr-batcher/src/main/java/io/optimism/batcher/compressor/ex/CompressorFullException.java diff --git a/README.md b/README.md index a9250a49..0cf218b2 100644 --- a/README.md +++ b/README.md @@ -67,4 +67,4 @@ docker compose up -d The docker setup contains a Grafana dashboard. To view sync progress, you can check the dashboard at `http://localhost:3000` with the username `hildr` and password `passw0rd`. Alternatively, you can view Hildr's logs by running `docker logs hildr-node --follow`. ## Contribution -To help hildr grow, follow [Contributing to hildr](CONTRIBUTING.md). \ No newline at end of file +To help hildr grow, follow [Contributing to hildr](CONTRIBUTING.md). diff --git a/build.gradle b/build.gradle index a11e094d..e62f9360 100644 --- a/build.gradle +++ b/build.gradle @@ -1,295 +1,295 @@ plugins { - id 'java' - id 'application' - id 'checkstyle' - id "com.diffplug.spotless" version "6.19.0" - id "net.ltgt.errorprone" version "3.1.0" - id 'org.graalvm.buildtools.native' version '0.9.22' + id 'java' + id 'application' + id 'checkstyle' + id "com.diffplug.spotless" version "6.19.0" + id "net.ltgt.errorprone" version "3.1.0" + id 'org.graalvm.buildtools.native' version '0.9.22' } allprojects { - apply plugin: 'java' - apply plugin: 'application' - apply plugin: 'checkstyle' - apply plugin: 'jacoco' - apply plugin: 'net.ltgt.errorprone' - apply plugin: 'com.diffplug.spotless' - - repositories { - // Use Maven Central for resolving dependencies. - mavenCentral() - } - - dependencies { - // This dependency is used by the application. - implementation 'com.google.guava:guava:31.1-jre' - implementation 'com.github.rholder:guava-retrying:2.0.0' - - // define any required OkHttp artifacts without version - implementation("com.squareup.okhttp3:okhttp:5.0.0-alpha.2") - implementation("com.squareup.okhttp3:logging-interceptor:5.0.0-alpha.2") - - implementation('org.web3j:core:4.9.8') { - exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on' - exclude group: 'com.squareup.okhttp3', module: 'okhttp' - exclude group: 'com.squareup.okhttp3', module: 'logging-interceptor' - } - implementation('net.osslabz.evm:evm-abi-decoder:0.0.6') - implementation 'com.github.gestalt-config:gestalt-core:0.20.4' - implementation 'com.github.gestalt-config:gestalt-toml:0.20.4' - - implementation 'com.fasterxml.jackson:jackson-bom:2.15.2' - implementation 'com.fasterxml.jackson.core:jackson-core' - - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml' - implementation 'org.jctools:jctools-core:4.0.1' - - implementation 'io.jsonwebtoken:jjwt-api:0.11.5' - implementation 'io.jsonwebtoken:jjwt-impl:0.11.5' - implementation 'io.jsonwebtoken:jjwt-jackson:0.11.5' - - //jsonrpc - implementation('io.vertx:vertx-auth-jwt:4.4.2') - implementation('io.vertx:vertx-core:4.4.2') - implementation('io.vertx:vertx-web:4.4.2') - - implementation 'io.micrometer:micrometer-registry-prometheus:1.11.0' - implementation platform('io.micrometer:micrometer-tracing-bom:1.1.1') - implementation 'io.micrometer:micrometer-tracing' - implementation 'io.micrometer:micrometer-tracing-bridge-otel' - - // Logback - implementation 'ch.qos.logback:logback-core:1.4.7' - implementation 'ch.qos.logback:logback-classic:1.4.7' - implementation 'org.slf4j:slf4j-api:2.0.7' - - implementation platform("io.opentelemetry:opentelemetry-bom-alpha:1.26.0-alpha") - // OpenTelemetry core - implementation(platform("io.opentelemetry:opentelemetry-bom:1.26.0")) - implementation 'io.opentelemetry:opentelemetry-api' - implementation 'io.opentelemetry:opentelemetry-sdk' - implementation 'io.opentelemetry:opentelemetry-sdk-logs' - - // OpenTelemetry log4j appenders - implementation platform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha:1.26.0-alpha") - runtimeOnly 'io.opentelemetry.instrumentation:opentelemetry-logback-mdc-1.0' - - - implementation 'info.picocli:picocli:4.7.3' - annotationProcessor 'info.picocli:picocli-codegen:4.7.3' - - errorprone("com.google.errorprone:error_prone_core:2.18.0") - } - - tasks.withType(JavaCompile).configureEach { - options.annotationProcessorPath = configurations.annotationProcessor - options.compilerArgs += "--enable-preview" - options.compilerArgs += "-Xlint:preview" - options.compilerArgs += ["--add-modules", "jdk.incubator.concurrent"] - options.compilerArgs += ["-Aproject=${project.group}/${project.name}"] - } - - tasks.withType(Test).configureEach { - jvmArgs += "--enable-preview" - jvmArgs += ["--add-modules", "jdk.incubator.concurrent"] - } - - tasks.withType(JavaExec).configureEach { - jvmArgs += "--enable-preview" - jvmArgs += ["--add-modules", "jdk.incubator.concurrent"] - } - - java { - toolchain { - languageVersion = JavaLanguageVersion.of(20) - } - } - - test { - useJUnitPlatform() - testLogging { - events "passed", "skipped", "failed" - } - finalizedBy jacocoTestReport - } - - jacoco { - toolVersion = "0.8.9" - } - - jacocoTestReport { - dependsOn test - - reports { - csv.required = true - } - } - - jacocoTestCoverageVerification { - - violationRules { - rule { - limit { - minimum = 0 - } - } - } - } - - checkstyle { - toolVersion = '10.10.0' - // default checkstyle config -- specific to your team agreement - configFile = project(":").file("config/checkstyle/google_checks.xml") - // Google style (idiosyncratic to Google): - // configFile = project(":").file("config/checkstyle/google_checks.xml") - // SUN style (closest to modern Java styles) -- the basis for this project: - // configFile = project(":").file("config/checkstyle/sun_checks.xml") + apply plugin: 'java' + apply plugin: 'application' + apply plugin: 'checkstyle' + apply plugin: 'jacoco' + apply plugin: 'net.ltgt.errorprone' + apply plugin: 'com.diffplug.spotless' + + repositories { + // Use Maven Central for resolving dependencies. + mavenCentral() + } + + dependencies { + // This dependency is used by the application. + implementation 'com.google.guava:guava:31.1-jre' + implementation 'com.github.rholder:guava-retrying:2.0.0' + + // define any required OkHttp artifacts without version + implementation("com.squareup.okhttp3:okhttp:5.0.0-alpha.2") + implementation("com.squareup.okhttp3:logging-interceptor:5.0.0-alpha.2") + + implementation('org.web3j:core:4.9.8') { + exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on' + exclude group: 'com.squareup.okhttp3', module: 'okhttp' + exclude group: 'com.squareup.okhttp3', module: 'logging-interceptor' + } + implementation('net.osslabz.evm:evm-abi-decoder:0.0.6') + implementation 'com.github.gestalt-config:gestalt-core:0.20.4' + implementation 'com.github.gestalt-config:gestalt-toml:0.20.4' + + implementation 'com.fasterxml.jackson:jackson-bom:2.15.2' + implementation 'com.fasterxml.jackson.core:jackson-core' + + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml' + implementation 'org.jctools:jctools-core:4.0.1' + + implementation 'io.jsonwebtoken:jjwt-api:0.11.5' + implementation 'io.jsonwebtoken:jjwt-impl:0.11.5' + implementation 'io.jsonwebtoken:jjwt-jackson:0.11.5' + + //jsonrpc + implementation('io.vertx:vertx-auth-jwt:4.4.2') + implementation('io.vertx:vertx-core:4.4.2') + implementation('io.vertx:vertx-web:4.4.2') + + implementation 'io.micrometer:micrometer-registry-prometheus:1.11.0' + implementation platform('io.micrometer:micrometer-tracing-bom:1.1.1') + implementation 'io.micrometer:micrometer-tracing' + implementation 'io.micrometer:micrometer-tracing-bridge-otel' + + // Logback + implementation 'ch.qos.logback:logback-core:1.4.7' + implementation 'ch.qos.logback:logback-classic:1.4.7' + implementation 'org.slf4j:slf4j-api:2.0.7' + + implementation platform("io.opentelemetry:opentelemetry-bom-alpha:1.26.0-alpha") + // OpenTelemetry core + implementation(platform("io.opentelemetry:opentelemetry-bom:1.26.0")) + implementation 'io.opentelemetry:opentelemetry-api' + implementation 'io.opentelemetry:opentelemetry-sdk' + implementation 'io.opentelemetry:opentelemetry-sdk-logs' + + // OpenTelemetry log4j appenders + implementation platform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha:1.26.0-alpha") + runtimeOnly 'io.opentelemetry.instrumentation:opentelemetry-logback-mdc-1.0' + + + implementation 'info.picocli:picocli:4.7.3' + annotationProcessor 'info.picocli:picocli-codegen:4.7.3' + + errorprone("com.google.errorprone:error_prone_core:2.18.0") + } + + tasks.withType(JavaCompile).configureEach { + options.annotationProcessorPath = configurations.annotationProcessor + options.compilerArgs += "--enable-preview" + options.compilerArgs += "-Xlint:preview" + options.compilerArgs += ["--add-modules", "jdk.incubator.concurrent"] + options.compilerArgs += ["-Aproject=${project.group}/${project.name}"] + } + + tasks.withType(Test).configureEach { + jvmArgs += "--enable-preview" + jvmArgs += ["--add-modules", "jdk.incubator.concurrent"] + } + + tasks.withType(JavaExec).configureEach { + jvmArgs += "--enable-preview" + jvmArgs += ["--add-modules", "jdk.incubator.concurrent"] + } + + java { + toolchain { + languageVersion = JavaLanguageVersion.of(20) + } + } + + test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } + finalizedBy jacocoTestReport + } + + jacoco { + toolVersion = "0.8.9" + } + + jacocoTestReport { + dependsOn test + + reports { + csv.required = true + } + } + + jacocoTestCoverageVerification { + + violationRules { + rule { + limit { + minimum = 0 + } + } + } + } + + checkstyle { + toolVersion = '10.10.0' + // default checkstyle config -- specific to your team agreement + configFile = project(":").file("config/checkstyle/google_checks.xml") + // Google style (idiosyncratic to Google): + // configFile = project(":").file("config/checkstyle/google_checks.xml") + // SUN style (closest to modern Java styles) -- the basis for this project: + // configFile = project(":").file("config/checkstyle/sun_checks.xml") // ignoreFailures = false // maxWarnings = 0 - } + } - spotless { - // optional: limit format enforcement to just the files changed by this feature branch + spotless { + // optional: limit format enforcement to just the files changed by this feature branch // ratchetFrom 'origin/main' - format 'misc', { - // define the files to apply `misc` to - target '*.gradle', '*.md', '.gitignore' - - // define the steps to apply to those files - trimTrailingWhitespace() - indentWithTabs() // or spaces. Takes an integer argument if you don't like 4 - endWithNewline() - } - java { - // Use the default importOrder configuration - - // don't need to set target, it is inferred from java - - // apply a specific flavor of google-java-format - googleJavaFormat('1.17.0') - // fix formatting of type annotations - formatAnnotations() - // make sure every file has the following copyright header. - // optionally, Spotless can set copyright years by digging - // through git history (see "license" section below) - licenseHeaderFile(project(":").file("config/spotless/java.license")).named('hildr').onlyIfContentMatches('/*\n' + - ' * Copyright 2023 281165273grape@gmail.com') - licenseHeaderFile(project(":").file("config/spotless/besu.license")).named('besu').onlyIfContentMatches('/*\n' + - ' * Copyright ConsenSys AG') - - importOrder() - - removeUnusedImports() - } - } - - tasks.named('test') { - // Use JUnit Platform for unit tests. - useJUnitPlatform() - } - - check { - dependsOn += jacocoTestCoverageVerification - } - - tasks.withType(Test).configureEach { - def outputDir = reports.junitXml.outputLocation - jvmArgumentProviders << ({ - [ - "-Djunit.platform.reporting.open.xml.enabled=true", - "-Djunit.platform.reporting.output.dir=${outputDir.get().asFile.absolutePath}", - "--enable-preview" - ] - } as CommandLineArgumentProvider) - } - - java { - withJavadocJar() - withSourcesJar() - } - - javadoc { - if (JavaVersion.current().isJava9Compatible()) { - options.addBooleanOption('html5', true) - } - options.addBooleanOption('-enable-preview', true) - options.addStringOption('-release', '20') - options.addStringOption('-add-modules', 'jdk.incubator.concurrent') - } - - task copyJarToBinaryDir { - dependsOn jar - def buildBinaryDir = "build/binary" - doLast { - new File(buildBinaryDir).mkdirs() - copy { - from "build/libs/${project.name}-${project.version}.jar" - into buildBinaryDir - rename "${project.name}-${project.version}.jar", "${project.name}.jar" - } - } - } - - task buildBinary { - dependsOn copyJarToBinaryDir - def buildBinaryDir = "build/binary" - def out = new ByteArrayOutputStream() - doLast { - exec { - workingDir buildBinaryDir - executable "sh" - args "-c", "native-image -jar ${project.name}.jar --no-fallback --enable-http --enable-https --enable-preview --add-modules jdk.incubator.concurrent -H:EnableURLProtocols=http,https --initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback.core.CoreConstants,ch.qos.logback.core.util.Loader,ch.qos.logback.core.util.StatusPrinter,ch.qos.logback.core.status.InfoStatus,ch.qos.logback.classic.Logger,ch.qos.logback.core.rolling.helper.FileNamePattern,ch.qos.logback.classic.Level,ch.qos.logback.core.status.StatusBase,io.opentelemetry.api.trace.ArrayBasedTraceStateBuilder,io.opentelemetry.context.LazyStorage,ch.qos.logback.core.util.FileSize,ch.qos.logback.core.rolling.helper.RollingCalendar,io.opentelemetry.api.internal.ImmutableSpanContext,io.opentelemetry.api.internal.OtelEncodingUtils,ch.qos.logback.classic.PatternLayout,io.opentelemetry.context.ThreadLocalContextStorage,io.opentelemetry.api.trace.PropagatedSpan,io.opentelemetry.context.ContextStorageWrappers,ch.qos.logback.core.rolling.helper.Compressor\$1,io.opentelemetry.api.trace.ImmutableTraceFlags,ch.qos.logback.core.rolling.helper.RollingCalendar\$1,ch.qos.logback.classic.model.ConfigurationModel,ch.qos.logback.core.model.processor.DefaultProcessor\$1,ch.qos.logback.core.model.processor.ImplicitModelHandler\$1,ch.qos.logback.core.subst.Token,ch.qos.logback.core.pattern.parser.Parser,ch.qos.logback.core.subst.Parser\$1,ch.qos.logback.core.util.Duration,ch.qos.logback.core.model.processor.ChainedModelFilter\$1,ch.qos.logback.classic.model.processor.ConfigurationModelHandler,ch.qos.logback.classic.model.processor.LogbackClassicDefaultNestedComponentRules,ch.qos.logback.core.subst.NodeToStringTransformer\$1,ch.qos.logback.core.pattern.parser.TokenStream\$1,ch.qos.logback.core.subst.Tokenizer\$1 --initialize-at-run-time=io.netty.channel.AbstractChannel,io.netty.channel.socket.nio.SelectorProviderUtil,io.netty.util.concurrent.DefaultPromise,io.netty,org.slf4j.MDC,org.github.gestalt.config ${project.name}" - standardOutput out - } - } - println(out.toString()) - } - - task buildBinaryStaticNoG1GC { - dependsOn copyJarToBinaryDir - def buildBinaryDir = "build/binary" - def out = new ByteArrayOutputStream() - doLast { - exec { - workingDir buildBinaryDir - executable "sh" - args "-c", "native-image -jar ${project.name}.jar --static --libc=musl --no-fallback --enable-http --enable-https --enable-preview --add-modules jdk.incubator.concurrent -H:EnableURLProtocols=http,https --initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback.core.CoreConstants,ch.qos.logback.core.util.Loader,ch.qos.logback.core.util.StatusPrinter,ch.qos.logback.core.status.InfoStatus,ch.qos.logback.classic.Logger,ch.qos.logback.core.rolling.helper.FileNamePattern,ch.qos.logback.classic.Level,ch.qos.logback.core.status.StatusBase,io.opentelemetry.api.trace.ArrayBasedTraceStateBuilder,io.opentelemetry.context.LazyStorage,ch.qos.logback.core.util.FileSize,ch.qos.logback.core.rolling.helper.RollingCalendar,io.opentelemetry.api.internal.ImmutableSpanContext,io.opentelemetry.api.internal.OtelEncodingUtils,ch.qos.logback.classic.PatternLayout,io.opentelemetry.context.ThreadLocalContextStorage,io.opentelemetry.api.trace.PropagatedSpan,io.opentelemetry.context.ContextStorageWrappers,ch.qos.logback.core.rolling.helper.Compressor\$1,io.opentelemetry.api.trace.ImmutableTraceFlags,ch.qos.logback.core.rolling.helper.RollingCalendar\$1,ch.qos.logback.classic.model.ConfigurationModel,ch.qos.logback.core.model.processor.DefaultProcessor\$1,ch.qos.logback.core.model.processor.ImplicitModelHandler\$1,ch.qos.logback.core.subst.Token,ch.qos.logback.core.pattern.parser.Parser,ch.qos.logback.core.subst.Parser\$1,ch.qos.logback.core.util.Duration,ch.qos.logback.core.model.processor.ChainedModelFilter\$1,ch.qos.logback.classic.model.processor.ConfigurationModelHandler,ch.qos.logback.classic.model.processor.LogbackClassicDefaultNestedComponentRules,ch.qos.logback.core.subst.NodeToStringTransformer\$1,ch.qos.logback.core.pattern.parser.TokenStream\$1,ch.qos.logback.core.subst.Tokenizer\$1 --initialize-at-run-time=io.netty.channel.AbstractChannel,io.netty.channel.socket.nio.SelectorProviderUtil,io.netty.util.concurrent.DefaultPromise,io.netty,org.slf4j.MDC,org.github.gestalt.config ${project.name}" - standardOutput out - } - } - println(out.toString()) - } - - task buildBinaryStatic { - dependsOn copyJarToBinaryDir - def buildBinaryDir = "build/binary" - def out = new ByteArrayOutputStream() - doLast { - exec { - workingDir buildBinaryDir - executable "sh" - args "-c", "native-image -jar ${project.name}.jar --gc=G1 --static --libc=musl --no-fallback --enable-http --enable-https --enable-preview --add-modules jdk.incubator.concurrent -H:EnableURLProtocols=http,https --initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback.core.CoreConstants,ch.qos.logback.core.util.Loader,ch.qos.logback.core.util.StatusPrinter,ch.qos.logback.core.status.InfoStatus,ch.qos.logback.classic.Logger,ch.qos.logback.core.rolling.helper.FileNamePattern,ch.qos.logback.classic.Level,ch.qos.logback.core.status.StatusBase,io.opentelemetry.api.trace.ArrayBasedTraceStateBuilder,io.opentelemetry.context.LazyStorage,ch.qos.logback.core.util.FileSize,ch.qos.logback.core.rolling.helper.RollingCalendar,io.opentelemetry.api.internal.ImmutableSpanContext,io.opentelemetry.api.internal.OtelEncodingUtils,ch.qos.logback.classic.PatternLayout,io.opentelemetry.context.ThreadLocalContextStorage,io.opentelemetry.api.trace.PropagatedSpan,io.opentelemetry.context.ContextStorageWrappers,ch.qos.logback.core.rolling.helper.Compressor\$1,io.opentelemetry.api.trace.ImmutableTraceFlags,ch.qos.logback.core.rolling.helper.RollingCalendar\$1,ch.qos.logback.classic.model.ConfigurationModel,ch.qos.logback.core.model.processor.DefaultProcessor\$1,ch.qos.logback.core.model.processor.ImplicitModelHandler\$1,ch.qos.logback.core.subst.Token,ch.qos.logback.core.pattern.parser.Parser,ch.qos.logback.core.subst.Parser\$1,ch.qos.logback.core.util.Duration,ch.qos.logback.core.model.processor.ChainedModelFilter\$1,ch.qos.logback.classic.model.processor.ConfigurationModelHandler,ch.qos.logback.classic.model.processor.LogbackClassicDefaultNestedComponentRules,ch.qos.logback.core.subst.NodeToStringTransformer\$1,ch.qos.logback.core.pattern.parser.TokenStream\$1,ch.qos.logback.core.subst.Tokenizer\$1 --initialize-at-run-time=io.netty.channel.AbstractChannel,io.netty.channel.socket.nio.SelectorProviderUtil,io.netty.util.concurrent.DefaultPromise,io.netty,org.slf4j.MDC,org.github.gestalt.config ${project.name}" - standardOutput out - } - } - println(out.toString()) - } - - task buildBinaryG1GC { - dependsOn copyJarToBinaryDir - def buildBinaryDir = "build/binary" - def out = new ByteArrayOutputStream() - doLast { - exec { - workingDir buildBinaryDir - executable "sh" - args "-c", "native-image -jar ${project.name}.jar --gc=G1 --no-fallback --enable-http --enable-https --enable-preview --add-modules jdk.incubator.concurrent -H:EnableURLProtocols=http,https --initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback.core.CoreConstants,ch.qos.logback.core.util.Loader,ch.qos.logback.core.util.StatusPrinter,ch.qos.logback.core.status.InfoStatus,ch.qos.logback.classic.Logger,ch.qos.logback.core.rolling.helper.FileNamePattern,ch.qos.logback.classic.Level,ch.qos.logback.core.status.StatusBase,io.opentelemetry.api.trace.ArrayBasedTraceStateBuilder,io.opentelemetry.context.LazyStorage,ch.qos.logback.core.util.FileSize,ch.qos.logback.core.rolling.helper.RollingCalendar,io.opentelemetry.api.internal.ImmutableSpanContext,io.opentelemetry.api.internal.OtelEncodingUtils,ch.qos.logback.classic.PatternLayout,io.opentelemetry.context.ThreadLocalContextStorage,io.opentelemetry.api.trace.PropagatedSpan,io.opentelemetry.context.ContextStorageWrappers,ch.qos.logback.core.rolling.helper.Compressor\$1,io.opentelemetry.api.trace.ImmutableTraceFlags,ch.qos.logback.core.rolling.helper.RollingCalendar\$1,ch.qos.logback.classic.model.ConfigurationModel,ch.qos.logback.core.model.processor.DefaultProcessor\$1,ch.qos.logback.core.model.processor.ImplicitModelHandler\$1,ch.qos.logback.core.subst.Token,ch.qos.logback.core.pattern.parser.Parser,ch.qos.logback.core.subst.Parser\$1,ch.qos.logback.core.util.Duration,ch.qos.logback.core.model.processor.ChainedModelFilter\$1,ch.qos.logback.classic.model.processor.ConfigurationModelHandler,ch.qos.logback.classic.model.processor.LogbackClassicDefaultNestedComponentRules,ch.qos.logback.core.subst.NodeToStringTransformer\$1,ch.qos.logback.core.pattern.parser.TokenStream\$1,ch.qos.logback.core.subst.Tokenizer\$1 --initialize-at-run-time=io.netty.channel.AbstractChannel,io.netty.channel.socket.nio.SelectorProviderUtil,io.netty.util.concurrent.DefaultPromise,io.netty,org.slf4j.MDC,org.github.gestalt.config ${project.name}" - standardOutput out - } - } - println(out.toString()) - } + format 'misc', { + // define the files to apply `misc` to + target '*.gradle', '*.md', '.gitignore' + + // define the steps to apply to those files + trimTrailingWhitespace() + indentWithTabs() // or spaces. Takes an integer argument if you don't like 4 + endWithNewline() + } + java { + // Use the default importOrder configuration + + // don't need to set target, it is inferred from java + + // apply a specific flavor of google-java-format + googleJavaFormat('1.17.0') + // fix formatting of type annotations + formatAnnotations() + // make sure every file has the following copyright header. + // optionally, Spotless can set copyright years by digging + // through git history (see "license" section below) + licenseHeaderFile(project(":").file("config/spotless/java.license")).named('hildr').onlyIfContentMatches('/*\n' + + ' * Copyright 2023 281165273grape@gmail.com') + licenseHeaderFile(project(":").file("config/spotless/besu.license")).named('besu').onlyIfContentMatches('/*\n' + + ' * Copyright ConsenSys AG') + + importOrder() + + removeUnusedImports() + } + } + + tasks.named('test') { + // Use JUnit Platform for unit tests. + useJUnitPlatform() + } + + check { + dependsOn += jacocoTestCoverageVerification + } + + tasks.withType(Test).configureEach { + def outputDir = reports.junitXml.outputLocation + jvmArgumentProviders << ({ + [ + "-Djunit.platform.reporting.open.xml.enabled=true", + "-Djunit.platform.reporting.output.dir=${outputDir.get().asFile.absolutePath}", + "--enable-preview" + ] + } as CommandLineArgumentProvider) + } + + java { + withJavadocJar() + withSourcesJar() + } + + javadoc { + if (JavaVersion.current().isJava9Compatible()) { + options.addBooleanOption('html5', true) + } + options.addBooleanOption('-enable-preview', true) + options.addStringOption('-release', '20') + options.addStringOption('-add-modules', 'jdk.incubator.concurrent') + } + + task copyJarToBinaryDir { + dependsOn jar + def buildBinaryDir = "build/binary" + doLast { + new File(buildBinaryDir).mkdirs() + copy { + from "build/libs/${project.name}-${project.version}.jar" + into buildBinaryDir + rename "${project.name}-${project.version}.jar", "${project.name}.jar" + } + } + } + + task buildBinary { + dependsOn copyJarToBinaryDir + def buildBinaryDir = "build/binary" + def out = new ByteArrayOutputStream() + doLast { + exec { + workingDir buildBinaryDir + executable "sh" + args "-c", "native-image -jar ${project.name}.jar --no-fallback --enable-http --enable-https --enable-preview --add-modules jdk.incubator.concurrent -H:EnableURLProtocols=http,https --initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback.core.CoreConstants,ch.qos.logback.core.util.Loader,ch.qos.logback.core.util.StatusPrinter,ch.qos.logback.core.status.InfoStatus,ch.qos.logback.classic.Logger,ch.qos.logback.core.rolling.helper.FileNamePattern,ch.qos.logback.classic.Level,ch.qos.logback.core.status.StatusBase,io.opentelemetry.api.trace.ArrayBasedTraceStateBuilder,io.opentelemetry.context.LazyStorage,ch.qos.logback.core.util.FileSize,ch.qos.logback.core.rolling.helper.RollingCalendar,io.opentelemetry.api.internal.ImmutableSpanContext,io.opentelemetry.api.internal.OtelEncodingUtils,ch.qos.logback.classic.PatternLayout,io.opentelemetry.context.ThreadLocalContextStorage,io.opentelemetry.api.trace.PropagatedSpan,io.opentelemetry.context.ContextStorageWrappers,ch.qos.logback.core.rolling.helper.Compressor\$1,io.opentelemetry.api.trace.ImmutableTraceFlags,ch.qos.logback.core.rolling.helper.RollingCalendar\$1,ch.qos.logback.classic.model.ConfigurationModel,ch.qos.logback.core.model.processor.DefaultProcessor\$1,ch.qos.logback.core.model.processor.ImplicitModelHandler\$1,ch.qos.logback.core.subst.Token,ch.qos.logback.core.pattern.parser.Parser,ch.qos.logback.core.subst.Parser\$1,ch.qos.logback.core.util.Duration,ch.qos.logback.core.model.processor.ChainedModelFilter\$1,ch.qos.logback.classic.model.processor.ConfigurationModelHandler,ch.qos.logback.classic.model.processor.LogbackClassicDefaultNestedComponentRules,ch.qos.logback.core.subst.NodeToStringTransformer\$1,ch.qos.logback.core.pattern.parser.TokenStream\$1,ch.qos.logback.core.subst.Tokenizer\$1 --initialize-at-run-time=io.netty.channel.AbstractChannel,io.netty.channel.socket.nio.SelectorProviderUtil,io.netty.util.concurrent.DefaultPromise,io.netty,org.slf4j.MDC,org.github.gestalt.config ${project.name}" + standardOutput out + } + } + println(out.toString()) + } + + task buildBinaryStaticNoG1GC { + dependsOn copyJarToBinaryDir + def buildBinaryDir = "build/binary" + def out = new ByteArrayOutputStream() + doLast { + exec { + workingDir buildBinaryDir + executable "sh" + args "-c", "native-image -jar ${project.name}.jar --static --libc=musl --no-fallback --enable-http --enable-https --enable-preview --add-modules jdk.incubator.concurrent -H:EnableURLProtocols=http,https --initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback.core.CoreConstants,ch.qos.logback.core.util.Loader,ch.qos.logback.core.util.StatusPrinter,ch.qos.logback.core.status.InfoStatus,ch.qos.logback.classic.Logger,ch.qos.logback.core.rolling.helper.FileNamePattern,ch.qos.logback.classic.Level,ch.qos.logback.core.status.StatusBase,io.opentelemetry.api.trace.ArrayBasedTraceStateBuilder,io.opentelemetry.context.LazyStorage,ch.qos.logback.core.util.FileSize,ch.qos.logback.core.rolling.helper.RollingCalendar,io.opentelemetry.api.internal.ImmutableSpanContext,io.opentelemetry.api.internal.OtelEncodingUtils,ch.qos.logback.classic.PatternLayout,io.opentelemetry.context.ThreadLocalContextStorage,io.opentelemetry.api.trace.PropagatedSpan,io.opentelemetry.context.ContextStorageWrappers,ch.qos.logback.core.rolling.helper.Compressor\$1,io.opentelemetry.api.trace.ImmutableTraceFlags,ch.qos.logback.core.rolling.helper.RollingCalendar\$1,ch.qos.logback.classic.model.ConfigurationModel,ch.qos.logback.core.model.processor.DefaultProcessor\$1,ch.qos.logback.core.model.processor.ImplicitModelHandler\$1,ch.qos.logback.core.subst.Token,ch.qos.logback.core.pattern.parser.Parser,ch.qos.logback.core.subst.Parser\$1,ch.qos.logback.core.util.Duration,ch.qos.logback.core.model.processor.ChainedModelFilter\$1,ch.qos.logback.classic.model.processor.ConfigurationModelHandler,ch.qos.logback.classic.model.processor.LogbackClassicDefaultNestedComponentRules,ch.qos.logback.core.subst.NodeToStringTransformer\$1,ch.qos.logback.core.pattern.parser.TokenStream\$1,ch.qos.logback.core.subst.Tokenizer\$1 --initialize-at-run-time=io.netty.channel.AbstractChannel,io.netty.channel.socket.nio.SelectorProviderUtil,io.netty.util.concurrent.DefaultPromise,io.netty,org.slf4j.MDC,org.github.gestalt.config ${project.name}" + standardOutput out + } + } + println(out.toString()) + } + + task buildBinaryStatic { + dependsOn copyJarToBinaryDir + def buildBinaryDir = "build/binary" + def out = new ByteArrayOutputStream() + doLast { + exec { + workingDir buildBinaryDir + executable "sh" + args "-c", "native-image -jar ${project.name}.jar --gc=G1 --static --libc=musl --no-fallback --enable-http --enable-https --enable-preview --add-modules jdk.incubator.concurrent -H:EnableURLProtocols=http,https --initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback.core.CoreConstants,ch.qos.logback.core.util.Loader,ch.qos.logback.core.util.StatusPrinter,ch.qos.logback.core.status.InfoStatus,ch.qos.logback.classic.Logger,ch.qos.logback.core.rolling.helper.FileNamePattern,ch.qos.logback.classic.Level,ch.qos.logback.core.status.StatusBase,io.opentelemetry.api.trace.ArrayBasedTraceStateBuilder,io.opentelemetry.context.LazyStorage,ch.qos.logback.core.util.FileSize,ch.qos.logback.core.rolling.helper.RollingCalendar,io.opentelemetry.api.internal.ImmutableSpanContext,io.opentelemetry.api.internal.OtelEncodingUtils,ch.qos.logback.classic.PatternLayout,io.opentelemetry.context.ThreadLocalContextStorage,io.opentelemetry.api.trace.PropagatedSpan,io.opentelemetry.context.ContextStorageWrappers,ch.qos.logback.core.rolling.helper.Compressor\$1,io.opentelemetry.api.trace.ImmutableTraceFlags,ch.qos.logback.core.rolling.helper.RollingCalendar\$1,ch.qos.logback.classic.model.ConfigurationModel,ch.qos.logback.core.model.processor.DefaultProcessor\$1,ch.qos.logback.core.model.processor.ImplicitModelHandler\$1,ch.qos.logback.core.subst.Token,ch.qos.logback.core.pattern.parser.Parser,ch.qos.logback.core.subst.Parser\$1,ch.qos.logback.core.util.Duration,ch.qos.logback.core.model.processor.ChainedModelFilter\$1,ch.qos.logback.classic.model.processor.ConfigurationModelHandler,ch.qos.logback.classic.model.processor.LogbackClassicDefaultNestedComponentRules,ch.qos.logback.core.subst.NodeToStringTransformer\$1,ch.qos.logback.core.pattern.parser.TokenStream\$1,ch.qos.logback.core.subst.Tokenizer\$1 --initialize-at-run-time=io.netty.channel.AbstractChannel,io.netty.channel.socket.nio.SelectorProviderUtil,io.netty.util.concurrent.DefaultPromise,io.netty,org.slf4j.MDC,org.github.gestalt.config ${project.name}" + standardOutput out + } + } + println(out.toString()) + } + + task buildBinaryG1GC { + dependsOn copyJarToBinaryDir + def buildBinaryDir = "build/binary" + def out = new ByteArrayOutputStream() + doLast { + exec { + workingDir buildBinaryDir + executable "sh" + args "-c", "native-image -jar ${project.name}.jar --gc=G1 --no-fallback --enable-http --enable-https --enable-preview --add-modules jdk.incubator.concurrent -H:EnableURLProtocols=http,https --initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback.core.CoreConstants,ch.qos.logback.core.util.Loader,ch.qos.logback.core.util.StatusPrinter,ch.qos.logback.core.status.InfoStatus,ch.qos.logback.classic.Logger,ch.qos.logback.core.rolling.helper.FileNamePattern,ch.qos.logback.classic.Level,ch.qos.logback.core.status.StatusBase,io.opentelemetry.api.trace.ArrayBasedTraceStateBuilder,io.opentelemetry.context.LazyStorage,ch.qos.logback.core.util.FileSize,ch.qos.logback.core.rolling.helper.RollingCalendar,io.opentelemetry.api.internal.ImmutableSpanContext,io.opentelemetry.api.internal.OtelEncodingUtils,ch.qos.logback.classic.PatternLayout,io.opentelemetry.context.ThreadLocalContextStorage,io.opentelemetry.api.trace.PropagatedSpan,io.opentelemetry.context.ContextStorageWrappers,ch.qos.logback.core.rolling.helper.Compressor\$1,io.opentelemetry.api.trace.ImmutableTraceFlags,ch.qos.logback.core.rolling.helper.RollingCalendar\$1,ch.qos.logback.classic.model.ConfigurationModel,ch.qos.logback.core.model.processor.DefaultProcessor\$1,ch.qos.logback.core.model.processor.ImplicitModelHandler\$1,ch.qos.logback.core.subst.Token,ch.qos.logback.core.pattern.parser.Parser,ch.qos.logback.core.subst.Parser\$1,ch.qos.logback.core.util.Duration,ch.qos.logback.core.model.processor.ChainedModelFilter\$1,ch.qos.logback.classic.model.processor.ConfigurationModelHandler,ch.qos.logback.classic.model.processor.LogbackClassicDefaultNestedComponentRules,ch.qos.logback.core.subst.NodeToStringTransformer\$1,ch.qos.logback.core.pattern.parser.TokenStream\$1,ch.qos.logback.core.subst.Tokenizer\$1 --initialize-at-run-time=io.netty.channel.AbstractChannel,io.netty.channel.socket.nio.SelectorProviderUtil,io.netty.util.concurrent.DefaultPromise,io.netty,org.slf4j.MDC,org.github.gestalt.config ${project.name}" + standardOutput out + } + } + println(out.toString()) + } } jar { enabled = false } diff --git a/hildr-batcher/build.gradle b/hildr-batcher/build.gradle index 20ea73f6..8d192037 100644 --- a/hildr-batcher/build.gradle +++ b/hildr-batcher/build.gradle @@ -1,15 +1,11 @@ -plugins { - id 'java' -} - group = 'io.optimism' version = '0.1.0' application { - // Define the main class for the application. - mainClass = 'io.optimism.HildrBatcher' + // Define the main class for the application. + mainClass = 'io.optimism.HildrBatcher' } test { - useJUnitPlatform() + useJUnitPlatform() } diff --git a/hildr-batcher/src/main/java/io/optimism/HildrBatcher.java b/hildr-batcher/src/main/java/io/optimism/HildrBatcher.java index 355d6287..53e07ef9 100644 --- a/hildr-batcher/src/main/java/io/optimism/HildrBatcher.java +++ b/hildr-batcher/src/main/java/io/optimism/HildrBatcher.java @@ -17,4 +17,3 @@ public static void main(String[] args) { System.out.println("i = " + i); } } - diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressor.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressor.java new file mode 100644 index 00000000..a3b9d53a --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressor.java @@ -0,0 +1,39 @@ +package io.optimism.batcher.compressor; + +import java.io.Closeable; +import java.io.Flushable; + +/** + * @author thinkAfCod + * @since 2023.07 + */ +public interface Compressor extends Closeable, Readable, Flushable { + + /** + * write uncompressed data which will be compressed. Should return CompressorFullException if the + * compressor is full and no more data should be written. + */ + int write(byte[] p); + + // read compressed data; should only be called after Close. + int read(byte[] p); + + // reset all written data. + void reset(); + + // returns an estimate of the current length of the compressed data; calling Flush will + // increase the accuracy at the expense of a poorer compression ratio. + int length(); + + // returns CompressorFullException if the compressor is known to be full. Note that + // calls to Write will fail if an error is returned from this method, but calls to Write + // can still return CompressorFullErr even if this does not. + void fullErr(); + + //// Closer Close function should be called before reading any data. + // io.Closer + //// Flush flushes any uncompressed data to the compression buffer. This will result in a + //// non-optimal compression ratio. + // Flush() error + +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/CompressorFactory.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/CompressorFactory.java new file mode 100644 index 00000000..bda7d2dd --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/CompressorFactory.java @@ -0,0 +1,22 @@ +package io.optimism.batcher.compressor; + +/** + * @author thinkAfCod + * @since 2023.07 + */ +public class CompressorFactory { + + // var Kinds = map[string]FactoryFunc{ + // RatioKind: NewRatioCompressor, + // ShadowKind: NewShadowCompressor, + // } + // + // var KindKeys []string + // + // func init() { + // for k := range Kinds { + // KindKeys = append(KindKeys, k) + // } + // } + +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressors.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressors.java new file mode 100644 index 00000000..6ba6bc9c --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Compressors.java @@ -0,0 +1,13 @@ +package io.optimism.batcher.compressor; + +/** + * @author thinkAfCod + * @since 2023.07 + */ +public interface Compressors { + + String RatioKind = "ratio"; + String ShadowKind = "shadow"; + + default void create(String kind) {} +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Config.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Config.java new file mode 100644 index 00000000..1516bad6 --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/Config.java @@ -0,0 +1,17 @@ +package io.optimism.batcher.compressor; + +/** + * @author thinkAfCod + * @since 2023.07 + * @param targetFrameSize to target when creating channel frames. Note that if the realized + * compression ratio is worse than the approximate, more frames may actually be created. This + * also depends on how close the target is to the max frame size. + * @param targetNumFrame to create in this channel. If the realized compression ratio is worse than + * approxComprRatio, additional leftover frame(s) might get created. + * @param approxComprRatio ApproxComprRatio to assume. Should be slightly smaller than average from + * experiments to avoid the chances of creating a small additional leftover frame. + * @param kind Kind of compressor to use. Must be one of KindKeys. If unset, NewCompressor will + * default to RatioKind. + */ +public record Config( + long targetFrameSize, int targetNumFrame, double approxComprRatio, String kind) {} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/RatioCompressor.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/RatioCompressor.java new file mode 100644 index 00000000..0c1ad427 --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/RatioCompressor.java @@ -0,0 +1,48 @@ +package io.optimism.batcher.compressor; + +import java.io.IOException; +import java.nio.CharBuffer; +import org.jetbrains.annotations.NotNull; + +/** + * @author thinkAfCod + * @since 2023.07 + */ +public class RatioCompressor implements Compressor { + + RatioCompressor() { + // todo 看看有没有更高性能的工具，如果没有就使用java.util.zip.Deflater + } + + @Override + public int write(byte[] p) { + return 0; + } + + @Override + public int read(byte[] p) { + return 0; + } + + @Override + public void reset() {} + + @Override + public int length() { + return 0; + } + + @Override + public void fullErr() {} + + @Override + public void close() throws IOException {} + + @Override + public void flush() throws IOException {} + + @Override + public int read(@NotNull CharBuffer cb) throws IOException { + return 0; + } +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ShadowCompressor.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ShadowCompressor.java new file mode 100644 index 00000000..70de0f9b --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ShadowCompressor.java @@ -0,0 +1,44 @@ +package io.optimism.batcher.compressor; + +import java.io.IOException; +import java.nio.CharBuffer; +import org.jetbrains.annotations.NotNull; + +/** + * @author thinkAfCod + * @since 2023.07 + */ +public class ShadowCompressor implements Compressor { + + @Override + public int write(byte[] p) { + return 0; + } + + @Override + public int read(byte[] p) { + return 0; + } + + @Override + public void reset() {} + + @Override + public int length() { + return 0; + } + + @Override + public void fullErr() {} + + @Override + public void close() throws IOException {} + + @Override + public void flush() throws IOException {} + + @Override + public int read(@NotNull CharBuffer cb) throws IOException { + return 0; + } +} diff --git a/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ex/CompressorFullException.java b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ex/CompressorFullException.java new file mode 100644 index 00000000..04637f25 --- /dev/null +++ b/hildr-batcher/src/main/java/io/optimism/batcher/compressor/ex/CompressorFullException.java @@ -0,0 +1,15 @@ +package io.optimism.batcher.compressor.ex; + +/** + * if the compressor is full and no more data should be written or the compressor is known to be + * full. + * + * @author thinkAfCod + * @since 2023.07 + */ +public class CompressorFullException extends RuntimeException { + + public CompressorFullException(String message) { + super(message); + } +} diff --git a/settings.gradle b/settings.gradle index b610e942..00f4de30 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,10 +8,10 @@ */ pluginManagement { - repositories { - mavenCentral() - gradlePluginPortal() - } + repositories { + mavenCentral() + gradlePluginPortal() + } } rootProject.name = 'hildr'