Kamu UI 323 support etheriumlogs as a source kind in UI (#326)

* update schema * added ethereum logs fragment * Added EthereumLogs to metadata block * Added EthereumLogs to the metadata tab * Added Ethereum Logs step * Changed CHANGELOG.md * Fixed typo * changed CHANGELOG.md * Added min validator * Added condition for unregistered user
kamu-data · Jun 24, 2024 · 82ab3c5 · 82ab3c5
1 parent 083da14
commit 82ab3c5
Show file tree

Hide file tree

Showing 17 changed files with 194 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Closed security error with initial drag & drop area for data displayed to anonymous users
 
 
+## [0.21.0] - 2024-06-24
+### Added
+- Added `EthereumLogs` polling source and metadata block support 
+
 ## [0.21.0] - 2024-06-24
 ### Added
 - Added ingest via file upload

diff --git a/resources/schema.graphql b/resources/schema.graphql
@@ -760,7 +760,7 @@ type ExecuteTransformInput {
   newOffset: Int
 }
 
-union FetchStep = FetchStepUrl | FetchStepFilesGlob | FetchStepContainer | FetchStepMqtt
+union FetchStep = FetchStepUrl | FetchStepFilesGlob | FetchStepContainer | FetchStepMqtt | FetchStepEthereumLogs
 
 type FetchStepContainer {
   image: String!
@@ -769,6 +769,13 @@ type FetchStepContainer {
   env: [EnvVar!]
 }
 
+type FetchStepEthereumLogs {
+  chainId: Int
+  nodeUrl: String
+  filter: String
+  signature: String
+}
+
 type FetchStepFilesGlob {
   path: String!
   eventTime: EventTimeSource

diff --git a/src/app/api/gql/fragments/events/fragment-set-polling-source.graphql b/src/app/api/gql/fragments/events/fragment-set-polling-source.graphql
@@ -4,6 +4,7 @@ fragment SetPollingSourceEvent on SetPollingSource {
     ...FetchStepFilesGlobData
     ...FetchStepContainerData
     ...FetchStepMqttData
+    ...FetchStepEthereumLogsData
   }
   read {
     ...ReadStepCsvData

diff --git a/...pp/api/gql/fragments/events/sources-steps/fetch/fragment-fetch-step-ethereum-logs.graphql b/...pp/api/gql/fragments/events/sources-steps/fetch/fragment-fetch-step-ethereum-logs.graphql
@@ -0,0 +1,6 @@
+fragment FetchStepEthereumLogsData on FetchStepEthereumLogs {
+  chainId
+  nodeUrl
+  filter
+  signature
+}
diff --git a/src/app/api/kamu.graphql.interface.ts b/src/app/api/kamu.graphql.interface.ts
@@ -833,7 +833,7 @@ export type ExecuteTransformInput = {
     prevOffset?: Maybe<Scalars["Int"]>;
 };
 
-export type FetchStep = FetchStepContainer | FetchStepFilesGlob | FetchStepMqtt | FetchStepUrl;
+export type FetchStep = FetchStepContainer | FetchStepEthereumLogs | FetchStepFilesGlob | FetchStepMqtt | FetchStepUrl;
 
 export type FetchStepContainer = {
     __typename?: "FetchStepContainer";
@@ -843,6 +843,14 @@ export type FetchStepContainer = {
     image: Scalars["String"];
 };
 
+export type FetchStepEthereumLogs = {
+    __typename?: "FetchStepEthereumLogs";
+    chainId?: Maybe<Scalars["Int"]>;
+    filter?: Maybe<Scalars["String"]>;
+    nodeUrl?: Maybe<Scalars["String"]>;
+    signature?: Maybe<Scalars["String"]>;
+};
+
 export type FetchStepFilesGlob = {
     __typename?: "FetchStepFilesGlob";
     cache?: Maybe<SourceCaching>;
@@ -2032,6 +2040,7 @@ export type DatasetConnectionDataFragment = {
                     __typename?: "SetPollingSource";
                     fetch:
                         | ({ __typename?: "FetchStepContainer" } & FetchStepContainerDataFragment)
+                        | { __typename?: "FetchStepEthereumLogs" }
                         | ({ __typename?: "FetchStepFilesGlob" } & FetchStepFilesGlobDataFragment)
                         | { __typename?: "FetchStepMqtt" }
                         | ({ __typename?: "FetchStepUrl" } & FetchStepUrlDataFragment);
@@ -2577,6 +2586,7 @@ export type GetDatasetListFlowsQuery = {
                           __typename?: "SetPollingSource";
                           fetch:
                               | ({ __typename?: "FetchStepContainer" } & FetchStepContainerDataFragment)
+                              | { __typename?: "FetchStepEthereumLogs" }
                               | ({ __typename?: "FetchStepFilesGlob" } & FetchStepFilesGlobDataFragment)
                               | { __typename?: "FetchStepMqtt" }
                               | ({ __typename?: "FetchStepUrl" } & FetchStepUrlDataFragment);
@@ -2934,6 +2944,7 @@ export type SetPollingSourceEventFragment = {
     __typename?: "SetPollingSource";
     fetch:
         | ({ __typename?: "FetchStepContainer" } & FetchStepContainerDataFragment)
+        | ({ __typename?: "FetchStepEthereumLogs" } & FetchStepEthereumLogsDataFragment)
         | ({ __typename?: "FetchStepFilesGlob" } & FetchStepFilesGlobDataFragment)
         | ({ __typename?: "FetchStepMqtt" } & FetchStepMqttDataFragment)
         | ({ __typename?: "FetchStepUrl" } & FetchStepUrlDataFragment);
@@ -2972,6 +2983,14 @@ export type FetchStepContainerDataFragment = {
     env?: Array<{ __typename?: "EnvVar"; name: string; value?: string | null }> | null;
 };
 
+export type FetchStepEthereumLogsDataFragment = {
+    __typename?: "FetchStepEthereumLogs";
+    chainId?: number | null;
+    nodeUrl?: string | null;
+    filter?: string | null;
+    signature?: string | null;
+};
+
 export type FetchStepFilesGlobDataFragment = {
     __typename?: "FetchStepFilesGlob";
     path: string;
@@ -3099,6 +3118,7 @@ export type CurrentSourceFetchUrlFragment = {
         __typename?: "SetPollingSource";
         fetch:
             | { __typename?: "FetchStepContainer" }
+            | { __typename?: "FetchStepEthereumLogs" }
             | { __typename?: "FetchStepFilesGlob" }
             | { __typename?: "FetchStepMqtt"; host: string; port: number }
             | { __typename?: "FetchStepUrl"; url: string };
@@ -4103,6 +4123,14 @@ export const FetchStepMqttDataFragmentDoc = gql`
         }
     }
 `;
+export const FetchStepEthereumLogsDataFragmentDoc = gql`
+    fragment FetchStepEthereumLogsData on FetchStepEthereumLogs {
+        chainId
+        nodeUrl
+        filter
+        signature
+    }
+`;
 export const ReadStepCsvDataFragmentDoc = gql`
     fragment ReadStepCsvData on ReadStepCsv {
         schema
@@ -4203,6 +4231,7 @@ export const SetPollingSourceEventFragmentDoc = gql`
             ...FetchStepFilesGlobData
             ...FetchStepContainerData
             ...FetchStepMqttData
+            ...FetchStepEthereumLogsData
         }
         read {
             ...ReadStepCsvData
@@ -4230,6 +4259,7 @@ export const SetPollingSourceEventFragmentDoc = gql`
     ${FetchStepFilesGlobDataFragmentDoc}
     ${FetchStepContainerDataFragmentDoc}
     ${FetchStepMqttDataFragmentDoc}
+    ${FetchStepEthereumLogsDataFragmentDoc}
     ${ReadStepCsvDataFragmentDoc}
     ${ReadStepJsonDataFragmentDoc}
     ${ReadStepNdJsonDataFragmentDoc}

diff --git a/src/app/common/app.values.ts b/src/app/common/app.values.ts
@@ -9,6 +9,7 @@ export default class AppValues {
     public static readonly DEFAULT_USER_DISPLAY_NAME = "anonymous";
     public static readonly DEFAULT_AVATAR_URL = "https://avatars.githubusercontent.com/u/11951648?v=4";
     public static readonly URL_PATTERN = /^(http:\/\/)|(https:\/\/)/i;
+    public static readonly ZERO_OR_POSITIVE_PATTERN = /^[0-9]*$/i;
     public static readonly SCHEMA_NAME_PATTERN = /^[a-zA-Z0-9]+[a-zA-Z0-9\s(_)]*$/i;
     public static readonly SPLIT_ARGUMENTS_PATTERN = /[^\s"']+|"([^"]*)"+|'([^']*)'/g;
 

diff --git a/src/app/common/data.helpers.ts b/src/app/common/data.helpers.ts
@@ -137,6 +137,9 @@ export class DataHelpers {
             case "MergeStrategySnapshot": {
                 return "Snapshot";
             }
+            case "FetchStepEthereumLogs": {
+                return "Ethereum Logs";
+            }
             default:
                 return "Unknown type";
         }
@@ -244,6 +247,10 @@ export const getValidators = (validators: JsonFormValidators): ValidatorFn[] =>
                     validatorsToAdd.push(Validators.maxLength(value as number));
                 }
                 break;
+            case "min": {
+                validatorsToAdd.push(Validators.min(value as number));
+                break;
+            }
             default:
                 break;
         }

diff --git a/src/app/common/tooltips/tooltips.text.ts b/src/app/common/tooltips/tooltips.text.ts
@@ -27,4 +27,11 @@ export class SetPollingSourceTooltipsTexts {
     public static readonly MQTT_USERNAME = "Username to use for auth with the broker.";
     public static readonly MQTT_PASSWORD = "Password to use for auth with the broker (can be templated).";
     public static readonly MQTT_TOPICS = "List of topic subscription parameters.";
+    public static readonly FROM_ETHEREUM_LOGS = "Connects to an Ethereum node to stream transaction logs.";
+    public static readonly ETHEREUM_LOGS_CHAIN_ID = "Identifier of the chain to scan logs from.";
+    public static readonly ETHEREUM_LOGS_NODE_URL = "Url of the node.";
+    public static readonly ETHEREUM_LOGS_FILTER =
+        "An SQL WHERE clause that can be used to pre-filter the logs before fetching them from the ETH node.";
+    public static readonly ETHEREUM_LOGS_SIGNATURE =
+        "Solidity log event signature to use for decoding. Using this field adds event to the output containing decoded log as JSON.";
 }
diff --git a/...ents/event-details/components/set-polling-source-event/set-polling-source-event.source.ts b/...ents/event-details/components/set-polling-source-event/set-polling-source-event.source.ts
@@ -102,6 +102,46 @@ export const SET_POLLING_SOURCE_DESCRIPTORS: EventRowDescriptorsByField = {
         dataTestId: "setPollingSource-FetchStepMqtt-topics",
     },
 
+    "SetPollingSource.FetchStepEthereumLogs.__typename": {
+        label: "Type:",
+        tooltip: SetPollingSourceTooltipsTexts.FROM_ETHEREUM_LOGS,
+        presentationComponent: StepTypePropertyComponent,
+        separateRowForValue: false,
+        dataTestId: "setPollingSource-FetchStepEthereumLogs-__typename",
+    },
+
+    "SetPollingSource.FetchStepEthereumLogs.chainId": {
+        label: "Chain ID:",
+        tooltip: SetPollingSourceTooltipsTexts.ETHEREUM_LOGS_CHAIN_ID,
+        presentationComponent: SimplePropertyComponent,
+        separateRowForValue: false,
+        dataTestId: "setPollingSource-FetchStepEthereumLogs-chainId",
+    },
+
+    "SetPollingSource.FetchStepEthereumLogs.filter": {
+        label: "Filter:",
+        tooltip: SetPollingSourceTooltipsTexts.ETHEREUM_LOGS_FILTER,
+        presentationComponent: SimplePropertyComponent,
+        separateRowForValue: false,
+        dataTestId: "setPollingSource-FetchStepEthereumLogs-filter",
+    },
+
+    "SetPollingSource.FetchStepEthereumLogs.signature": {
+        label: "Signature:",
+        tooltip: SetPollingSourceTooltipsTexts.ETHEREUM_LOGS_SIGNATURE,
+        presentationComponent: SimplePropertyComponent,
+        separateRowForValue: false,
+        dataTestId: "setPollingSource-FetchStepEthereumLogs-signature",
+    },
+
+    "SetPollingSource.FetchStepEthereumLogs.nodeUrl": {
+        label: "Node url:",
+        tooltip: SetPollingSourceTooltipsTexts.ETHEREUM_LOGS_NODE_URL,
+        presentationComponent: SimplePropertyComponent,
+        separateRowForValue: false,
+        dataTestId: "setPollingSource-FetchStepEthereumLogs-nodeUrl",
+    },
+
     "SetPollingSource.FetchStepFilesGlob.__typename": {
         label: "Type:",
         tooltip: SetPollingSourceTooltipsTexts.FROM_FILES_GLOB,

diff --git a/...ts/metadata-component/components/form-components/number-field/number-field.component.html b/...ts/metadata-component/components/form-components/number-field/number-field.component.html
@@ -17,5 +17,16 @@
                 Field is required</span
             >
         </div>
+        <div *ngIf="form?.get(controlName)?.errors?.min" class="mt-1">
+            <span class="text-danger fs-12" attr.data-test-id="error-{{ controlName }}-min">
+                Minimum field value {{ form.get(controlName)?.errors?.min.min }}</span
+            >
+        </div>
+        <pre>{{ form.get(controlName)?.errors | json }}</pre>
+        <div *ngIf="form?.get(controlName)?.errors?.pattern" class="mt-1">
+            <span class="text-danger fs-12" attr.data-test-id="error-{{ controlName }}-pattern">
+                Field must be an integer</span
+            >
+        </div>
     </div>
 </div>
diff --git a/...ta-component/components/source-events/add-polling-source/add-polling-source-form.types.ts b/...ta-component/components/source-events/add-polling-source/add-polling-source-form.types.ts
@@ -5,6 +5,7 @@ export interface JsonFormValidators {
     minLength?: number;
     maxLength?: number;
     pattern?: string | RegExp;
+    min?: number;
 }
 
 interface JsonFormControlOptions {
@@ -61,6 +62,7 @@ export enum FetchKind {
     FILES_GLOB = "FilesGlob",
     CONTAINER = "Container",
     MQTT = "Mqtt",
+    ETHEREUM_LOGS = "EthereumLogs",
 }
 
 export enum ReadKind {
@@ -118,6 +120,10 @@ export interface AddPollingSourceEditFormType {
         url?: string;
         order?: string;
         topics?: TopicsType[];
+        chainId?: number;
+        nodeUrl?: string;
+        filter?: string;
+        signature?: string;
     };
     read: {
         kind: ReadKind;

diff --git a/...nts/metadata-component/components/source-events/add-polling-source/form-control.source.ts b/...nts/metadata-component/components/source-events/add-polling-source/form-control.source.ts
@@ -39,6 +39,13 @@ export const FETCH_STEP_RADIO_CONTROLS: RadioControlType[] = [
         label: "Sync from MQTT",
         tooltip: SetPollingSourceTooltipsTexts.FROM_MQTT,
     },
+    {
+        id: "from-ethereumLogs",
+        value: FetchKind.ETHEREUM_LOGS,
+        icon: "diamond",
+        label: "Sync from Ethereum logs",
+        tooltip: SetPollingSourceTooltipsTexts.FROM_ETHEREUM_LOGS,
+    },
 ];
 
 export const READ_STEP_RADIO_CONTROLS: RadioControlType[] = [

diff --git a/...onal-components/metadata-component/components/source-events/steps/data/fetch-form-data.ts b/...onal-components/metadata-component/components/source-events/steps/data/fetch-form-data.ts
@@ -218,4 +218,46 @@ export const FETCH_FORM_DATA: JsonFormData = {
             },
         ],
     },
+    EthereumLogs: {
+        controls: [
+            {
+                name: "chainId",
+                label: "Chain ID",
+                value: "",
+                type: ControlType.NUMBER,
+                placeholder: "Enter id",
+                tooltip: SetPollingSourceTooltipsTexts.ETHEREUM_LOGS_CHAIN_ID,
+                validators: { min: 0, pattern: AppValues.ZERO_OR_POSITIVE_PATTERN },
+            },
+            {
+                name: "nodeUrl",
+                label: "Node url",
+                value: "",
+                type: ControlType.TEXT,
+                placeholder: "Enter url",
+                tooltip: SetPollingSourceTooltipsTexts.ETHEREUM_LOGS_NODE_URL,
+                validators: {
+                    pattern: AppValues.URL_PATTERN,
+                },
+            },
+            {
+                name: "filter",
+                label: "Filter",
+                value: "",
+                type: ControlType.TEXT,
+                placeholder: "Enter filter",
+                tooltip: SetPollingSourceTooltipsTexts.ETHEREUM_LOGS_FILTER,
+                validators: {},
+            },
+            {
+                name: "signature",
+                label: "Signature",
+                value: "",
+                type: ControlType.TEXT,
+                placeholder: "Enter signature",
+                tooltip: SetPollingSourceTooltipsTexts.ETHEREUM_LOGS_SIGNATURE,
+                validators: {},
+            },
+        ],
+    },
 };
diff --git a/src/app/dataset-view/additional-components/metadata-component/metadata.component.html b/src/app/dataset-view/additional-components/metadata-component/metadata.component.html
@@ -120,6 +120,15 @@
                                 <span>{{ currentPollingSource.fetch.port }}</span>
                             </app-block-row-data>
                         </div>
+
+                        <div class="mx-4" *ngIf="currentPollingSource.fetch.__typename === 'FetchStepEthereumLogs'">
+                            <app-block-row-data
+                                [tooltip]="'Connects to an Ethereum node to stream transaction logs.'"
+                                [label]="'Type:'"
+                            >
+                                <span>Ethereum Logs</span>
+                            </app-block-row-data>
+                        </div>
                     </div>
 
                     <ng-container

diff --git a/...pp/dataset-view/additional-components/metadata-component/services/process-form.service.ts b/...pp/dataset-view/additional-components/metadata-component/services/process-form.service.ts
@@ -30,11 +30,19 @@ export class ProcessFormService {
             this.processEventTimeControl(formGroup);
             this.processPipeCommandControl(formGroup);
             this.removePollingSourceEmptyControls(formGroup);
+            this.changeTypeChainIdControl(formGroup);
             return;
         }
         this.removePushSourceEmptyControls(formGroup);
     }
 
+    private changeTypeChainIdControl(formGroup: FormGroup): void {
+        const form = formGroup.value as AddPollingSourceEditFormType;
+        if (form.fetch.chainId) {
+            form.fetch.chainId = +form.fetch.chainId;
+        }
+    }
+
     private transformSchema(formGroup: FormGroup): void {
         const form = formGroup.value as SchemaControlType;
         if (form.read.schema?.length && typeof form.read.schema[0] !== "string") {
@@ -67,7 +75,10 @@ export class ProcessFormService {
 
     private processEventTimeControl(formGroup: FormGroup): void {
         const form = formGroup.value as OrderControlType;
-        if (form.fetch.eventTime && [FetchKind.CONTAINER, FetchKind.MQTT].includes(form.fetch.kind)) {
+        if (
+            form.fetch.eventTime &&
+            [FetchKind.CONTAINER, FetchKind.MQTT, FetchKind.ETHEREUM_LOGS].includes(form.fetch.kind)
+        ) {
             delete form.fetch.eventTime;
         }
         if (form.fetch.eventTime?.kind !== EventTimeSourceKind.FROM_PATH) {