-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix(ingestion-web) sorting and filtering uses api (#11844)
- Loading branch information
Showing
11 changed files
with
212 additions
and
80 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
...in/java/com/linkedin/datahub/upgrade/config/BackfillIngestionSourceInfoIndicesConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| package com.linkedin.datahub.upgrade.config; | ||
|
|
||
| import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; | ||
| import com.linkedin.datahub.upgrade.system.ingestion.BackfillIngestionSourceInfoIndices; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import org.springframework.beans.factory.annotation.Value; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Conditional; | ||
| import org.springframework.context.annotation.Configuration; | ||
|
|
||
| @Configuration | ||
| @Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class) | ||
| public class BackfillIngestionSourceInfoIndicesConfig { | ||
|
|
||
| @Bean | ||
| public NonBlockingSystemUpgrade backfillIngestionSourceInfoIndices( | ||
| final OperationContext opContext, | ||
| final EntityService<?> entityService, | ||
| final AspectDao aspectDao, | ||
| @Value("${systemUpdate.ingestionIndices.enabled}") final boolean enabled, | ||
| @Value("${systemUpdate.ingestionIndices.batchSize}") final Integer batchSize, | ||
| @Value("${systemUpdate.ingestionIndices.delayMs}") final Integer delayMs, | ||
| @Value("${systemUpdate.ingestionIndices.limit}") final Integer limit) { | ||
| return new BackfillIngestionSourceInfoIndices( | ||
| opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit); | ||
| } | ||
| } |
43 changes: 43 additions & 0 deletions
43
...ava/com/linkedin/datahub/upgrade/system/ingestion/BackfillIngestionSourceInfoIndices.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package com.linkedin.datahub.upgrade.system.ingestion; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import java.util.List; | ||
| import javax.annotation.Nonnull; | ||
|
|
||
| public class BackfillIngestionSourceInfoIndices implements NonBlockingSystemUpgrade { | ||
|
|
||
| private final List<UpgradeStep> _steps; | ||
|
|
||
| public BackfillIngestionSourceInfoIndices( | ||
| @Nonnull OperationContext opContext, | ||
| EntityService<?> entityService, | ||
| AspectDao aspectDao, | ||
| boolean enabled, | ||
| Integer batchSize, | ||
| Integer batchDelayMs, | ||
| Integer limit) { | ||
| if (enabled) { | ||
| _steps = | ||
| ImmutableList.of( | ||
| new BackfillIngestionSourceInfoIndicesStep( | ||
| opContext, entityService, aspectDao, batchSize, batchDelayMs, limit)); | ||
| } else { | ||
| _steps = ImmutableList.of(); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return getClass().getSimpleName(); | ||
| } | ||
|
|
||
| @Override | ||
| public List<UpgradeStep> steps() { | ||
| return _steps; | ||
| } | ||
| } |
56 changes: 56 additions & 0 deletions
56
...com/linkedin/datahub/upgrade/system/ingestion/BackfillIngestionSourceInfoIndicesStep.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| package com.linkedin.datahub.upgrade.system.ingestion; | ||
|
|
||
| import static com.linkedin.metadata.Constants.*; | ||
|
|
||
| import com.linkedin.common.urn.Urn; | ||
| import com.linkedin.datahub.upgrade.system.AbstractMCLStep; | ||
| import com.linkedin.metadata.boot.BootstrapStep; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import javax.annotation.Nonnull; | ||
| import javax.annotation.Nullable; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| @Slf4j | ||
| public class BackfillIngestionSourceInfoIndicesStep extends AbstractMCLStep { | ||
|
|
||
| private static final String UPGRADE_ID = BackfillIngestionSourceInfoIndices.class.getSimpleName(); | ||
| private static final Urn UPGRADE_ID_URN = BootstrapStep.getUpgradeUrn(UPGRADE_ID); | ||
|
|
||
| public BackfillIngestionSourceInfoIndicesStep( | ||
| @Nonnull OperationContext opContext, | ||
| EntityService<?> entityService, | ||
| AspectDao aspectDao, | ||
| Integer batchSize, | ||
| Integer batchDelayMs, | ||
| Integer limit) { | ||
| super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit); | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return UPGRADE_ID; | ||
| } | ||
|
|
||
| @Nonnull | ||
| @Override | ||
| protected String getAspectName() { | ||
| return INGESTION_INFO_ASPECT_NAME; | ||
| } | ||
|
|
||
| @Nullable | ||
| @Override | ||
| protected String getUrnLike() { | ||
| return "urn:li:" + INGESTION_SOURCE_ENTITY_NAME + ":%"; | ||
| } | ||
|
|
||
| /** | ||
| * Returns whether the upgrade should proceed if the step fails after exceeding the maximum | ||
| * retries. | ||
| */ | ||
| @Override | ||
| public boolean isOptional() { | ||
| return true; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.