diff --git a/hildr-node/src/main/java/io/optimism/config/Config.java b/hildr-node/src/main/java/io/optimism/config/Config.java index 15b27784..4dc932ba 100644 --- a/hildr-node/src/main/java/io/optimism/config/Config.java +++ b/hildr-node/src/main/java/io/optimism/config/Config.java @@ -40,10 +40,11 @@ * @param l2RpcUrl L2 chain rpc url. * @param l2EngineUrl L2 engine API url. * @param jwtSecret L2 engine API jwt secret. - * @param chainConfig The chain config. + * @param checkpointSyncUrl The checkpoint sync url. * @param rpcPort The rpc port. * @param devnet The flag of devnet. - * @param checkpointSyncUrl The checkpoint sync url. + * @param syncMode The sync mode + * @param chainConfig The chain config. * @author grapebaba * @since 0.1.0 */ @@ -143,6 +144,7 @@ private static MapConfigSource getMapConfigSource() { * @param jwtSecret L2 engine API jwt secret. * @param checkpointSyncUrl The checkpoint sync url. * @param rpcPort The rpc port. + * @param syncMode The sync mode. * @param devnet The devnet flag. */ public record CliConfig( @@ -670,6 +672,10 @@ public enum SyncMode { */ ExecutionLayer; + /** + * is execution layer sync mode + * @return true if execution layer sync mode, otherwise false. + */ public boolean isEl() { return this == ExecutionLayer; } diff --git a/hildr-node/src/main/java/io/optimism/derive/stages/Batches.java b/hildr-node/src/main/java/io/optimism/derive/stages/Batches.java index 2a758fbc..97ac5261 100644 --- a/hildr-node/src/main/java/io/optimism/derive/stages/Batches.java +++ b/hildr-node/src/main/java/io/optimism/derive/stages/Batches.java @@ -336,8 +336,11 @@ private BatchStatus spanBatchStatus(final Batch batchWrapper) { // check batch timestamp if (spanEndTimestamp.compareTo(nextTimestamp) < 0) { - LOGGER.warn("past batch: nextTimestamp = l2SafeHead({}) + blockTime({}), spanEndTimestamp({})", - l2SafeHead.timestamp(), this.config.chainConfig().blockTime(), spanEndTimestamp); + LOGGER.warn( + "past batch: nextTimestamp = l2SafeHead({}) + blockTime({}), spanEndTimestamp({})", + l2SafeHead.timestamp(), + this.config.chainConfig().blockTime(), + spanEndTimestamp); return BatchStatus.Drop; } if (spanStartTimestamp.compareTo(nextTimestamp) > 0) { diff --git a/hildr-node/src/main/java/io/optimism/driver/Driver.java b/hildr-node/src/main/java/io/optimism/driver/Driver.java index ac80a10b..67703150 100644 --- a/hildr-node/src/main/java/io/optimism/driver/Driver.java +++ b/hildr-node/src/main/java/io/optimism/driver/Driver.java @@ -116,6 +116,7 @@ public class Driver extends AbstractExecutionThreadService { * * @param engineDriver the engine driver * @param pipeline the pipeline + * @param l2Fetcher the L2 HeadInfo fetcher * @param state the state * @param chainWatcher the chain watcher * @param unsafeBlockQueue the unsafe block queue @@ -227,8 +228,8 @@ public static Driver from(Config config, CountDownLatch latch) l2Refs = io.optimism.derive.State.initL2Refs(finalizedHead.number(), config.chainConfig(), l2Provider); } var l2Fetcher = Driver.l2Fetcher(l2Provider); - AtomicReference state = new AtomicReference<>(io.optimism.derive.State.create( - l2Refs, l2Fetcher, finalizedHead, finalizedEpoch, config)); + AtomicReference state = new AtomicReference<>( + io.optimism.derive.State.create(l2Refs, l2Fetcher, finalizedHead, finalizedEpoch, config)); EngineDriver engineDriver = new EngineDriver<>(finalizedHead, finalizedEpoch, l2Provider, config); @@ -242,7 +243,16 @@ public static Driver from(Config config, CountDownLatch latch) l2Provider.shutdown(); return new Driver<>( - engineDriver, pipeline, l2Fetcher, state, watcher, unsafeBlockQueue, rpcServer, latch, config, opStackNetwork); + engineDriver, + pipeline, + l2Fetcher, + state, + watcher, + unsafeBlockQueue, + rpcServer, + latch, + config, + opStackNetwork); } /** @@ -381,7 +391,6 @@ protected void shutDown() { this.engineDriver.stop(); LOGGER.info("engineDriver shut down."); this.rpcServer.stop(); - Web3jProvider.stop(); LOGGER.info("driver stopped."); if (this.opStackNetwork != null && this.isP2PNetworkStarted.compareAndExchange(true, false)) { this.opStackNetwork.stop(); @@ -668,15 +677,15 @@ private void fetchAndUpdateFinalizedHead() { if (this.engineDriver.getFinalizedHead().number().compareTo(BigInteger.ZERO) == 0) { blockParameter = FINALIZED; } else { - blockParameter = DefaultBlockParameter.valueOf(this.engineDriver.getFinalizedHead().number()); + blockParameter = DefaultBlockParameter.valueOf( + this.engineDriver.getFinalizedHead().number()); } Tuple2 finalizedHead = l2Fetcher.apply(blockParameter, true); this.engineDriver.updateFinalized(finalizedHead.component1(), finalizedHead.component2()); } - - - private static BiFunction> l2Fetcher(final Web3j l2Provider) { + private static BiFunction> l2Fetcher( + final Web3j l2Provider) { return (blockParameter, returnFull) -> { try (var scope = new StructuredTaskScope.ShutdownOnFailure()) { StructuredTaskScope.Subtask blockTask = scope.fork(TracerTaskWrapper.wrap(() -> l2Provider @@ -698,8 +707,6 @@ private static BiFunction) body -> body.contains("method\":\"engine_"); - this.web3jService = (HttpService) Web3jProvider.create(baseUrl, filter).component2(); + this.web3jService = (HttpService) Web3jProvider.create(baseUrl).component2(); } /** diff --git a/hildr-node/src/main/java/io/optimism/engine/ExecutionPayload.java b/hildr-node/src/main/java/io/optimism/engine/ExecutionPayload.java index 63326ce7..eb190f47 100644 --- a/hildr-node/src/main/java/io/optimism/engine/ExecutionPayload.java +++ b/hildr-node/src/main/java/io/optimism/engine/ExecutionPayload.java @@ -60,6 +60,11 @@ public record ExecutionPayload( BigInteger excessBlobGas, String parentBeaconBlockRoot) { + /** + * Converts the ExecutionPayload to an L2BlockRef. + * @param config the chain config + * @return the L2BlockRef + */ public L2BlockRef toL2BlockInfo(Config.ChainConfig config) { final Epoch l1GenesisEpoch = config.l1StartEpoch(); final BlockInfo l2GenesisInfo = config.l2Genesis(); diff --git a/hildr-node/src/main/java/io/optimism/l1/BeaconBlobFetcher.java b/hildr-node/src/main/java/io/optimism/l1/BeaconBlobFetcher.java index 0c34aaf7..62dca736 100644 --- a/hildr-node/src/main/java/io/optimism/l1/BeaconBlobFetcher.java +++ b/hildr-node/src/main/java/io/optimism/l1/BeaconBlobFetcher.java @@ -15,6 +15,7 @@ import java.util.concurrent.StructuredTaskScope; import java.util.stream.Collectors; import okhttp3.Call; +import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -75,6 +76,12 @@ public BeaconBlobFetcher(String beaconUrl) { * @param beaconArchiverUrl L1 beacon archiver client url */ public BeaconBlobFetcher(String beaconUrl, String beaconArchiverUrl) { + if (beaconUrl.endsWith("/")) { + beaconUrl = beaconUrl.replaceAll("/+$", ""); + } + if (beaconArchiverUrl.endsWith("/")) { + beaconArchiverUrl = beaconArchiverUrl.replaceAll("/+$", ""); + } this.genesisMethod = GENESIS_METHOD_FORMAT.formatted(beaconUrl); this.specMethod = SPEC_METHOD_FORMAT.formatted(beaconUrl); this.sidecarsMethod = SIDECARS_METHOD_PREFIX_FORMAT.formatted(beaconUrl); @@ -138,16 +145,21 @@ public List getBlobSidecards(String blockId, final List return res.getData(); } if (this.archiverSidecarsMethod != null) { - LOGGER.debug( - "blob sidecars may be pruned, try blob archiver sidecars method: blockId = {}, indices = {}", + LOGGER.info( + "blob sidecars may be pruned, try blob archiver sidecars method: blockId = {}, indices = {}, url = {}", blockId, - indices); + indices, + "%s/%s".formatted(this.archiverSidecarsMethod, postfix)); var archiverRes = getBlobSidecars("%s/%s".formatted(this.archiverSidecarsMethod, postfix)); + LOGGER.info( + "archiverUrl: {}; archiverRes return data: {}", + this.archiverSidecarsMethod, + archiverRes.getData() != null && !archiverRes.getData().isEmpty()); if (archiverRes.getData() != null && !archiverRes.getData().isEmpty()) { return archiverRes.getData(); } } else { - LOGGER.debug( + LOGGER.info( "blob archiver sidecars method is empty, skip retry: block Id = {}, indices = {}", blockId, indices); @@ -157,7 +169,7 @@ public List getBlobSidecards(String blockId, final List } private BeaconApiResponse> getBlobSidecars(String url) { - var req = new Request.Builder().get().url(url).build(); + var req = new Request.Builder().get().url(HttpUrl.parse(url)).build(); return this.send(req, new TypeReference>>() {}); } diff --git a/hildr-node/src/main/java/io/optimism/runner/Runner.java b/hildr-node/src/main/java/io/optimism/runner/Runner.java index 41d5b8b1..7c0b6c3d 100644 --- a/hildr-node/src/main/java/io/optimism/runner/Runner.java +++ b/hildr-node/src/main/java/io/optimism/runner/Runner.java @@ -298,6 +298,11 @@ public void checkpointSync() throws ExecutionException, InterruptedException { waitDriverRunning(); } + /** + * snap sync. + * + * @throws InterruptedException the interrupted exception + */ public void executionLayerSync() throws InterruptedException { LOGGER.info("execution layer sync"); waitDriverRunning(); diff --git a/hildr-node/src/test/java/io/optimism/engine/EngineApiTest.java b/hildr-node/src/test/java/io/optimism/engine/EngineApiTest.java index 395989dc..9b1f8177 100644 --- a/hildr-node/src/test/java/io/optimism/engine/EngineApiTest.java +++ b/hildr-node/src/test/java/io/optimism/engine/EngineApiTest.java @@ -17,7 +17,6 @@ import io.optimism.engine.ForkChoiceUpdate.ForkChoiceUpdateRes; import io.optimism.engine.ForkChoiceUpdate.ForkchoiceState; import io.optimism.type.Epoch; -import io.optimism.utilities.rpc.Web3jProvider; import java.io.IOException; import java.math.BigInteger; import java.security.Key; @@ -178,32 +177,4 @@ void testJwts() { - jwt.getBody().getIssuedAt().toInstant().getEpochSecond(), 60L); } - - @Test - void testLogging() throws IOException, InterruptedException { - TestConstants.createConfig(); - if (!TestConstants.isConfiguredApiKeyEnv) { - return; - } - EngineApi engineApi = new EngineApi( - TestConstants.createConfig(), - "http://127.0.0.1:8552", - "bf549f5188556ce0951048ef467ec93067bc4ea21acebe46ef675cd4e8e015ff"); - ForkchoiceState forkchoiceState = new ForkchoiceState("123", "123", "!@3"); - PayloadAttributes payloadAttributes = new PayloadAttributes( - new BigInteger("123123"), - "123123", - "123", - List.of(""), - null, - true, - new BigInteger("1"), - new Epoch(new BigInteger("12"), "123", new BigInteger("1233145"), BigInteger.ZERO), - new BigInteger("1334"), - new BigInteger("321"), - null); - engineApi.forkchoiceUpdated(forkchoiceState, payloadAttributes); - Web3jProvider.stop(); - Thread.sleep(3000); - } } diff --git a/hildr-utilities/src/main/java/io/optimism/type/BeaconBlockHeader.java b/hildr-utilities/src/main/java/io/optimism/type/BeaconBlockHeader.java index 32027e79..02766903 100644 --- a/hildr-utilities/src/main/java/io/optimism/type/BeaconBlockHeader.java +++ b/hildr-utilities/src/main/java/io/optimism/type/BeaconBlockHeader.java @@ -25,8 +25,19 @@ public class BeaconBlockHeader { @JsonAlias("body_root") private String bodyRoot; + /** + * Instantiates a new Beacon block header. + */ public BeaconBlockHeader() {} + /** + * Instantiates a new Beacon block header. + * @param slot the slot + * @param proposerIndex the proposer index + * @param parentRoot the parent root + * @param stateRoot the state root + * @param bodyRoot the body root + */ public BeaconBlockHeader(String slot, String proposerIndex, String parentRoot, String stateRoot, String bodyRoot) { this.slot = slot; this.proposerIndex = proposerIndex; @@ -35,42 +46,92 @@ public BeaconBlockHeader(String slot, String proposerIndex, String parentRoot, S this.bodyRoot = bodyRoot; } + /** + * Gets slot. + * + * @return the slot + */ public String getSlot() { return slot; } + /** + * Sets slot value. + * + * @param slot the slot + */ public void setSlot(String slot) { this.slot = slot; } + /** + * Gets proposer index. + * + * @return the proposer index + */ public String getProposerIndex() { return proposerIndex; } + /** + * Sets proposer index value. + * + * @param proposerIndex the proposer index + */ public void setProposerIndex(String proposerIndex) { this.proposerIndex = proposerIndex; } + /** + * Gets beacon parent root. + * + * @return the beacon parent root + */ public String getParentRoot() { return parentRoot; } + /** + * Sets parent root value. + * + * @param parentRoot the parent root + */ public void setParentRoot(String parentRoot) { this.parentRoot = parentRoot; } + /** + * Gets state root. + * + * @return the state root + */ public String getStateRoot() { return stateRoot; } + /** + * Sets state root value. + * + * @param stateRoot the state root + */ public void setStateRoot(String stateRoot) { this.stateRoot = stateRoot; } + /** + * Gets body root. + * + * @return the body root + */ public String getBodyRoot() { return bodyRoot; } + /** + * Sets body root value. + * + * @param bodyRoot the body root + */ public void setBodyRoot(String bodyRoot) { this.bodyRoot = bodyRoot; } diff --git a/hildr-utilities/src/main/java/io/optimism/type/BlobSidecar.java b/hildr-utilities/src/main/java/io/optimism/type/BlobSidecar.java index c059fcec..fffc2d98 100644 --- a/hildr-utilities/src/main/java/io/optimism/type/BlobSidecar.java +++ b/hildr-utilities/src/main/java/io/optimism/type/BlobSidecar.java @@ -41,6 +41,7 @@ public BlobSidecar() {} * @param signedBlockHeader signed blob block header info * @param kzgCommitment the kzg commitment info * @param kzgProof the kzg proofs + * @param kzgCommitmentInclusionProof the kzg commitment inclusion proofs */ public BlobSidecar( String index, @@ -57,54 +58,119 @@ public BlobSidecar( this.kzgCommitmentInclusionProof = kzgCommitmentInclusionProof; } + /** + * Gets index. + * + * @return the index + */ public String getIndex() { return index; } + /** + * Sets index value. + * + * @param index the index + */ public void setIndex(String index) { this.index = index; } + /** + * Gets blob. + * + * @return the blob + */ public String getBlob() { return blob; } + /** + * Sets blob value. + * + * @param blob the blob + */ public void setBlob(String blob) { this.blob = blob; } + /** + * Gets signed block header. + * + * @return the signed block header + */ public BeaconSignedBlockHeader getSignedBlockHeader() { return signedBlockHeader; } + /** + * Sets signed block header value. + * + * @param signedBlockHeader the signed block header + */ public void setSignedBlockHeader(BeaconSignedBlockHeader signedBlockHeader) { this.signedBlockHeader = signedBlockHeader; } + /** + * Gets kzg commitment. + * + * @return the kzg commitment + */ public String getKzgCommitment() { return kzgCommitment; } + /** + * Sets kzg commitment value. + * + * @param kzgCommitment the kzg commitment + */ public void setKzgCommitment(String kzgCommitment) { this.kzgCommitment = kzgCommitment; } + /** + * Gets kzg proof. + * + * @return the kzg proof + */ public String getKzgProof() { return kzgProof; } + /** + * Sets kzg proof value. + * + * @param kzgProof the kzg proof + */ public void setKzgProof(String kzgProof) { this.kzgProof = kzgProof; } + /** + * Gets kzg commitment inclusion proof. + * + * @return the kzg commitment inclusion proof + */ public List getKzgCommitmentInclusionProof() { return kzgCommitmentInclusionProof; } + /** + * Sets kzg commitment inclusion proof value. + * + * @param kzgCommitmentInclusionProof the kzg commitment inclusion proof + */ public void setKzgCommitmentInclusionProof(List kzgCommitmentInclusionProof) { this.kzgCommitmentInclusionProof = kzgCommitmentInclusionProof; } + /** + * Gets versioned hash. + * + * @return the versioned hash + */ public String getVersionedHash() { var hash = Hash.sha256(Numeric.hexStringToByteArray(this.kzgCommitment)); hash[0] = 1; diff --git a/hildr-utilities/src/main/java/io/optimism/type/L1BlockInfo.java b/hildr-utilities/src/main/java/io/optimism/type/L1BlockInfo.java index fc475a0b..39d1693f 100644 --- a/hildr-utilities/src/main/java/io/optimism/type/L1BlockInfo.java +++ b/hildr-utilities/src/main/java/io/optimism/type/L1BlockInfo.java @@ -18,6 +18,8 @@ * @param batcherAddr batcher address * @param l1FeeOverhead l1 fee overhead * @param l1FeeScalar l1 fee scalar + * @param blobBaseFeeScalar blob base fee scalar + * @param blobBaseFee blob base fee * @author thinkAfCod * @since 0.1.1 */ diff --git a/hildr-utilities/src/main/java/io/optimism/type/SpecConfig.java b/hildr-utilities/src/main/java/io/optimism/type/SpecConfig.java index f6c97002..ff805800 100644 --- a/hildr-utilities/src/main/java/io/optimism/type/SpecConfig.java +++ b/hildr-utilities/src/main/java/io/optimism/type/SpecConfig.java @@ -12,13 +12,22 @@ */ public class SpecConfig { + /** + * The seconds per slot. + */ @JsonAlias("SECONDS_PER_SLOT") public String secondsPerSlot; + /** + * The SpecConfig constructor. + */ public BigInteger getSecondsPerSlot() { return new BigInteger(secondsPerSlot); } + /** + * The SpecConfig constructor. + */ public void setSecondsPerSlot(String secondsPerSlot) { this.secondsPerSlot = secondsPerSlot; } diff --git a/hildr-utilities/src/main/java/io/optimism/type/enums/SyncStatus.java b/hildr-utilities/src/main/java/io/optimism/type/enums/SyncStatus.java index d9b6f506..72566717 100644 --- a/hildr-utilities/src/main/java/io/optimism/type/enums/SyncStatus.java +++ b/hildr-utilities/src/main/java/io/optimism/type/enums/SyncStatus.java @@ -34,10 +34,20 @@ public enum SyncStatus { this.code = code; } + /** + * Get the sync status code. + * + * @return the sync status code + */ public int getCode() { return code; } + /** + * Check if the engine is syncing. + * + * @return true if the engine is syncing, false otherwise + */ public boolean isEngineSyncing() { return code > 1 && code < 5; } diff --git a/hildr-utilities/src/main/java/io/optimism/utilities/derive/UpgradeDepositSource.java b/hildr-utilities/src/main/java/io/optimism/utilities/derive/UpgradeDepositSource.java deleted file mode 100644 index 0dd9f93a..00000000 --- a/hildr-utilities/src/main/java/io/optimism/utilities/derive/UpgradeDepositSource.java +++ /dev/null @@ -1,35 +0,0 @@ -package io.optimism.utilities.derive; - -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import org.web3j.crypto.Hash; -import org.web3j.utils.Numeric; - -/** - * The UpgradeDepositSource class. - * @author thinkAfCod - * @since 0.2.7 - */ -public class UpgradeDepositSource { - - private static final BigInteger UPGRADE_DEPOSIT_SOURCE_DOMAIN = BigInteger.TWO; - - private final String intent; - - /** - * The UpgradeDepositSource constructor. - * @param intent The intent identifies the upgrade-tx uniquely, in a human-readable way. - */ - public UpgradeDepositSource(String intent) { - this.intent = intent; - } - - public String sourceHash() { - byte[] domainInput = new byte[32 * 2]; - byte[] paddedDomain = Numeric.toBytesPadded(UPGRADE_DEPOSIT_SOURCE_DOMAIN, 8); - System.arraycopy(paddedDomain, 0, domainInput, 24, 8); - byte[] intentHash = Hash.sha3(this.intent.getBytes(StandardCharsets.UTF_8)); - System.arraycopy(intentHash, 0, domainInput, 32, 32); - return Numeric.toHexString(Hash.sha3(domainInput)); - } -} diff --git a/hildr-utilities/src/main/java/io/optimism/utilities/gas/GasCalculator.java b/hildr-utilities/src/main/java/io/optimism/utilities/gas/GasCalculator.java index f53fc86c..a0d7788f 100644 --- a/hildr-utilities/src/main/java/io/optimism/utilities/gas/GasCalculator.java +++ b/hildr-utilities/src/main/java/io/optimism/utilities/gas/GasCalculator.java @@ -44,11 +44,11 @@ private GasCalculator() {} /** * The constant INIT_CODE_WORD_GAS. */ - public static final long INIT_CODE_WORD_GAS = 2L; + private static final long INIT_CODE_WORD_GAS = 2L; - public static final BigInteger MIN_BLOB_GAS_PRICE = BigInteger.ONE; + private static final BigInteger MIN_BLOB_GAS_PRICE = BigInteger.ONE; - public static final BigInteger BLOB_GAS_PRICE_UPDATE_FRACTION = new BigInteger("3338477"); + private static final BigInteger BLOB_GAS_PRICE_UPDATE_FRACTION = new BigInteger("3338477"); /** * Calculator gas fee but exclude effective of AccessList. diff --git a/hildr-utilities/src/main/java/io/optimism/utilities/rpc/JsonRpcRequestBodyLoggingInterceptor.java b/hildr-utilities/src/main/java/io/optimism/utilities/rpc/JsonRpcRequestBodyLoggingInterceptor.java deleted file mode 100644 index 232d387b..00000000 --- a/hildr-utilities/src/main/java/io/optimism/utilities/rpc/JsonRpcRequestBodyLoggingInterceptor.java +++ /dev/null @@ -1,196 +0,0 @@ -package io.optimism.utilities.rpc; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.nio.file.StandardOpenOption.APPEND; -import static java.nio.file.StandardOpenOption.CREATE; - -import com.google.common.util.concurrent.AbstractExecutionThreadService; -import io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueue; -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.StructuredTaskScope; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; -import okhttp3.Interceptor; -import okhttp3.Request; -import okhttp3.Response; -import okio.Buffer; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class JsonRpcRequestBodyLoggingInterceptor extends AbstractExecutionThreadService implements Interceptor { - - private static final Logger LOGGER = LoggerFactory.getLogger(JsonRpcRequestBodyLoggingInterceptor.class); - - private MpscArrayQueue queue; - - private int fileId = 0; - - private String outputParentPath = "./json_raw/"; - - int dataLength = 0; - List requestCache = new ArrayList<>(); - - private final String outputFileNamePostfix = "_request.debug.json"; - - private final String outputFileNameFormat = "%d" + outputFileNamePostfix; - - private final Function shouldLog; - - private AtomicBoolean shutdown = new AtomicBoolean(false); - - public JsonRpcRequestBodyLoggingInterceptor(Function shouldLog) { - queue = new MpscArrayQueue<>(10000); - File outputParent = Path.of(outputParentPath).toFile(); - if (outputParent.isFile()) { - throw new IllegalArgumentException("Output parent path is a file"); - } - if (!outputParent.exists()) { - try { - Files.createDirectories(outputParent.toPath()); - } catch (IOException e) { - throw new IllegalStateException("Failed to create output parent directory"); - } - } - AtomicInteger fileIdAtomic = new AtomicInteger(0); - File[] unused = outputParent.listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - if (name.endsWith(outputFileNamePostfix)) { - String idStr = name.replace(outputFileNamePostfix, ""); - int id = Integer.parseInt(idStr); - if (id > fileIdAtomic.get()) { - fileIdAtomic.set(id); - } - } - return false; - } - }); - this.fileId = fileIdAtomic.get(); - this.shouldLog = shouldLog; - this.startAsync(); - } - - @NotNull @Override - public Response intercept(@NotNull final Chain chain) throws IOException { - Request request = chain.request(); - Buffer buffer = new Buffer(); - if (request.body() == null) { - return chain.proceed(request); - } - request.body().writeTo(buffer); - String requestBody = buffer.readUtf8(); - if (!shouldLog.apply(requestBody)) { - return chain.proceed(request); - } - queue.offer(requestBody); - return chain.proceed(request); - } - - @Override - protected void run() throws Exception { - for (; ; ) { - if (shutdown.get() && dataLength == 0) { - break; - } - readRequest(); - Thread.sleep(250); - } - LOGGER.info("logging service break"); - } - - @Override - protected void triggerShutdown() { - super.triggerShutdown(); - shutdown.compareAndExchange(false, true); - LOGGER.info("logging service shutdown, but will write file before"); - try { - writeReq(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private void readRequest() throws IOException { - for (var body = this.queue.poll(); body != null; body = this.queue.poll()) { - dataLength += body.getBytes(UTF_8).length; - requestCache.add(body); - LOGGER.info("add body to queue: cache size: {}, dateLength: {}", requestCache.size(), dataLength); - if (dataLength > 200 * 1024) { - writeReq(); - } - } - } - - private void writeReq() throws IOException { - LOGGER.info("will write cache to file: dateLength: {}", dataLength); - writeToFile(requestCache); - requestCache = new ArrayList<>(); - dataLength = 0; - } - - private void writeToFile(List bodyies) throws IOException { - var outputFilePath = Path.of(outputParentPath, String.format(outputFileNameFormat, fileId)); - var outputFile = outputFilePath.toFile(); - if (outputFile.length() > 1024 * 1024 * 500) { - compressFile(outputFile, true); - fileId += 1; - outputFilePath = Path.of(outputParentPath, String.format(outputFileNameFormat, fileId)); - } - try (BufferedWriter writer = Files.newBufferedWriter(outputFilePath, UTF_8, CREATE, APPEND)) { - for (int i = 0; i < bodyies.size(); i++) { - writer.write(bodyies.get(i)); - writer.newLine(); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - public static void compressFile(final File file, final boolean deleteOriginalFile) { - try (var scope = new StructuredTaskScope.ShutdownOnFailure()) { - scope.fork(() -> { - Path zipFilePath = Path.of(file.getParent(), "zipped", file.getName() + ".zip"); - LOGGER.info("will create json raw zip file: {}", zipFilePath); - if (!zipFilePath.getParent().toFile().exists()) { - Files.createDirectories(zipFilePath.getParent()); - } - Files.deleteIfExists(zipFilePath); - FileOutputStream fos = new FileOutputStream(zipFilePath.toString()); - ZipOutputStream zos = new ZipOutputStream(fos); - zos.setLevel(9); - File fileToZip = new File(file.getPath()); - - FileInputStream fis = new FileInputStream(fileToZip); - ZipEntry zipEntry = new ZipEntry(fileToZip.getName()); - zos.putNextEntry(zipEntry); - - byte[] bytes = new byte[4096]; - int length; - while ((length = fis.read(bytes)) >= 0) { - zos.write(bytes, 0, length); - } - zos.closeEntry(); - fis.close(); - zos.close(); - fos.close(); - if (deleteOriginalFile) { - file.delete(); - } - return null; - }); - scope.join(); - scope.throwIfFailed(); - } catch (ExecutionException | InterruptedException e) { - throw new IllegalStateException("Failed to compress file", e); - } - } -} diff --git a/hildr-utilities/src/main/java/io/optimism/utilities/rpc/Web3jProvider.java b/hildr-utilities/src/main/java/io/optimism/utilities/rpc/Web3jProvider.java index ece7d171..22f641b4 100644 --- a/hildr-utilities/src/main/java/io/optimism/utilities/rpc/Web3jProvider.java +++ b/hildr-utilities/src/main/java/io/optimism/utilities/rpc/Web3jProvider.java @@ -1,12 +1,8 @@ package io.optimism.utilities.rpc; import ch.qos.logback.classic.Level; -import com.google.common.util.concurrent.AbstractExecutionThreadService; import java.net.ConnectException; -import java.util.ArrayList; -import java.util.List; import java.util.function.Consumer; -import java.util.function.Function; import okhttp3.OkHttpClient; import okhttp3.logging.HttpLoggingInterceptor; import org.apache.commons.lang3.StringUtils; @@ -31,8 +27,6 @@ public class Web3jProvider { private Web3jProvider() {} - private static final List services = new ArrayList<>(); - /** * create web3j client. * @@ -51,10 +45,6 @@ public static Web3j createClient(String url) { * @return web3j client and web3j service */ public static Tuple2 create(String url) { - return create(url, null); - } - - public static Tuple2 create(String url, Function logFilter) { Web3jService web3Srv; if (Web3jProvider.isHttp(url)) { var okHttpClientBuilder = new OkHttpClient.Builder(); @@ -62,11 +52,6 @@ public static Tuple2 create(String url, Function create(String url, Function(Web3j.build(web3Srv), web3Srv); } - public static void stop() { - services.forEach(AbstractExecutionThreadService::stopAsync); - } - private static void wsConnect(final WebSocketService wss) { final Consumer onError = t -> { if (t instanceof WebsocketNotConnectedException) { diff --git a/hildr-utilities/src/main/java/io/optimism/utilities/rpc/response/BeaconApiResponse.java b/hildr-utilities/src/main/java/io/optimism/utilities/rpc/response/BeaconApiResponse.java index 9ffbfda8..de56294e 100644 --- a/hildr-utilities/src/main/java/io/optimism/utilities/rpc/response/BeaconApiResponse.java +++ b/hildr-utilities/src/main/java/io/optimism/utilities/rpc/response/BeaconApiResponse.java @@ -7,9 +7,13 @@ * * @author thinkAfCod * @since 0.3.0 + * @param the beacon api response data type. */ public class BeaconApiResponse { + /** + * The response inner data. + */ public T data; /**