diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index 0f0436563..a3a94303a 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -5,9 +5,10 @@ on: [pull_request] jobs: test-and-build: runs-on: it - steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 + with: + fetch-depth: 0 - uses: actions/setup-node@v3 with: cache: 'yarn' @@ -33,15 +34,23 @@ jobs: - name: build fusion docker image run: | docker build . --tag=nexus-web:fresh - # TODO: The following have been commented out because cypress tests are failing in the CI. Uncomment these when 3911 is fixed. - # - name: Start services - # run: docker-compose -f ci/docker-compose.yml up -d && sleep 60 - # - name: Copy nexus-web into Cypress container - # # avoids permission issue where cypress writes screenshots to host with root as user - # # which we can't then delete easily - # run: docker cp ./. cypress:/e2e - # - name: e2e tests - # run: echo | docker exec -e DEBUG=cypress:launcher:browsers -t cypress cypress run --config-file cypress.config.ts --browser chrome --record --key ${{ secrets.CYPRESS_RECORD_KEY }} + - name: Start services + run: docker-compose -f ci/docker-compose.yml up -d && sleep 60 + - name: Copy nexus-web into Cypress container + # avoids permission issue where cypress writes screenshots to host with root as user + # which we can't then delete easily + run: docker cp ./. cypress:/e2e + - name: e2e tests + run: >- + echo | timeout --verbose 20m docker exec + -e 'DEBUG=cypress:launcher:browsers' + -t + cypress + cypress run + --config-file cypress.config.ts + --browser chrome + --record + --key ${{ secrets.CYPRESS_RECORD_KEY }} - name: Cleanup Docker Containers if: ${{ always() }} run: docker-compose -f ci/docker-compose.yml down --rmi "local" --volumes diff --git a/.gitignore b/.gitignore index 14f873d4f..5a6fb5838 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ cypress.env.json *.sops* .sops-*.* *.key +*.asdcas diff --git a/README.md b/README.md index 03444080f..5652807a0 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,23 @@ Run unit tests: yarn test ``` +Run end to end tests: + +1. Ensure you have the `cypress.env.json` file locally. +2. + +a. To run the tests in headed mode: + +``` +yarn cy:open +``` + +b. To run the tests in headless mode: + +``` +yarn cy:run +``` + ## Build for production Compile app in `dist/` folder. diff --git a/ci/config/delta-postgres.conf b/ci/config/delta-postgres.conf index cd926de84..193dec17c 100644 --- a/ci/config/delta-postgres.conf +++ b/ci/config/delta-postgres.conf @@ -1,6 +1,6 @@ app { http { - base-uri = "http://delta:8080/v1" + base-uri = "http://delta:8098/v1" interface = 0.0.0.0 } diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index c8da3601e..9c9fb8b6f 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -1,14 +1,13 @@ version: '3.3' services: cypress: - image: 'cypress/included:10.3.1-typescript' + image: 'cypress/included:12.17.0' depends_on: - fusion - delta environment: - - CYPRESS_NEXUS_API_URL=http://delta.test:8080/v1 + - CYPRESS_NEXUS_API_URL=http://delta.test:8098/v1 - CYPRESS_BASE_URL=http://fusion.test:8000 - working_dir: /e2e container_name: cypress entrypoint: ['bash'] @@ -19,11 +18,13 @@ services: - delta environment: ANALYSIS_PLUGIN_SHOW_ON_TYPES: 'Analysis,DetailedCircuit,AnalysisReport,Report' - API_ENDPOINT: 'http://delta.test:8080/v1' + API_ENDPOINT: 'http://delta.test:8098/v1' CLIENT_ID: 'fusion' image: nexus-web:fresh expose: - 8000 + ports: + - 8000:8000 networks: default: aliases: @@ -56,9 +57,12 @@ services: './bin/delta-app', '-Xmx4G', '-Dapp.cluster.remote-interface=delta', + '-Dapp.http.port=8098', ] expose: - - 8080 + - 8098 + ports: + - 8098:8098 volumes: - ./config:/config - /tmp:/default-volume @@ -79,6 +83,7 @@ services: - --hostname-port=8080 - --hostname-strict=false - --hostname-strict-backchannel=false + - --log-level=DEBUG ports: - 8080:8080 networks: diff --git a/cypress.config.ts b/cypress.config.ts index 572391c83..a7e1f4afa 100644 --- a/cypress.config.ts +++ b/cypress.config.ts @@ -4,7 +4,7 @@ import { createNexusOrgAndProject, createResource, deprecateNexusOrgAndProject, -} from './cypress/plugins/nexus'; +} from './cypress/support/Utils/nexus'; import { uuidv4 } from './src/shared/utils'; import setup, { TestUsers } from './cypress/support/setupRealmsAndUsers'; @@ -14,14 +14,14 @@ export default defineConfig({ projectId: '1iihco', viewportWidth: 1200, video: true, + screenshotOnRunFailure: false, e2e: { baseUrl: 'http://localhost:8000', fileServerFolder: '/cypress', defaultCommandTimeout: 50000, specPattern: 'cypress/e2e/**/*.cy.{js,jsx,ts,tsx}', - // @ts-ignore - experimentalSessionAndOrigin: true, - // testIsolation: false, + chromeWebSecurity: false, + env: { DEBUG: 'cypress:launcher:browsers', ELECTRON_DISABLE_GPU: 'true', @@ -29,8 +29,19 @@ export default defineConfig({ }, setupNodeEvents(on, config) { on('before:browser:launch', (browser, launchOptions) => { + launchOptions.args.push('--window-size=1920,1080'); if (browser.name == 'chrome') { launchOptions.args.push('--disable-gpu'); + launchOptions.args.push('--no-sandbox'); + + launchOptions.args.push('--disable-web-security'); + launchOptions.args.push( + '--unsafely-treat-insecure-origin-as-secure=http://keycloak.test:8080' + ); + if (!browser.isHeaded) { + console.log('Pushing Headless New'); + launchOptions.args.push('--headless=new'); + } } return launchOptions; }), @@ -65,7 +76,7 @@ export default defineConfig({ const projectLabel = `${projectLabelBase}-${uuidv4()}`; const projectDescription = 'An project used for Cypress automated tests'; - + console.log('Auth TOKEN', authToken); try { const nexus = createNexusClient({ uri: nexusApiUrl, diff --git a/cypress/e2e/AnalysisPlugin.cy-ignore.ts b/cypress/e2e/AnalysisPlugin.cy.ts similarity index 100% rename from cypress/e2e/AnalysisPlugin.cy-ignore.ts rename to cypress/e2e/AnalysisPlugin.cy.ts diff --git a/cypress/e2e/Studios.cy-ignore.ts b/cypress/e2e/Studios.cy.ts similarity index 86% rename from cypress/e2e/Studios.cy-ignore.ts rename to cypress/e2e/Studios.cy.ts index 0ac14fe9e..10526bce5 100644 --- a/cypress/e2e/Studios.cy-ignore.ts +++ b/cypress/e2e/Studios.cy.ts @@ -44,14 +44,14 @@ describe('Studios', () => { studioDetailsPage = new StudioDetailsPage(); }); - // after(function() { - // cy.task('project:teardown', { - // nexusApiUrl: Cypress.env('NEXUS_API_URL'), - // authToken: this.nexusToken, - // orgLabel: Cypress.env('ORG_LABEL'), - // projectLabel: this.projectLabel, - // }); - // }); + after(function() { + cy.task('project:teardown', { + nexusApiUrl: Cypress.env('NEXUS_API_URL'), + authToken: this.nexusToken, + orgLabel: Cypress.env('ORG_LABEL'), + projectLabel: this.projectLabel, + }); + }); it('user can create a studio with a workspace and dashboard', function() { cy.visit( @@ -70,9 +70,9 @@ describe('Studios', () => { cy.findByText('Enable Filter').click(); - return cy.findByRole('button', { name: /Save/ }).click(); - }) - .then(() => { + cy.findByRole('button', { name: /Save/ }).click({ force: true }); + // cy.wait('@saveDashboardRequest'); + studioDetailsPage.openEditDashboard(); cy.findByLabelText(/Enable Filter/i).should('be.checked'); diff --git a/cypress/support/Studios/StudioDetails.ts b/cypress/support/Studios/StudioDetails.ts index 1884306b3..add70c463 100644 --- a/cypress/support/Studios/StudioDetails.ts +++ b/cypress/support/Studios/StudioDetails.ts @@ -17,10 +17,10 @@ export class StudioDetailsPage extends StudioListPage { } openEditDashboard() { - cy.findByRole('button', { name: /Dashboard/ }).click(); + cy.findByRole('button', { name: /Dashboard/ }).click({ force: true }); cy.get('button[data-testid="edit-dashboard"]') .contains(/Edit/) - .click(); + .click({ force: true }); cy.findByRole('dialog').findByText('Edit'); } @@ -42,14 +42,16 @@ export class StudioDetailsPage extends StudioListPage { cy.findByRole('button', { name: /Add/ }).click(); cy.findByPlaceholderText(/Name/i).type(dashboardName); cy.findByRole('combobox', { name: /View/ }).click(); - cy.findByTitle( - 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex' - ).click(); + cy.get( + '[data-testid="https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex"]' + ); + cy.get( + '[data-testid="https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex"]' + ).trigger('click'); + cy.findByRole('checkbox', { name: /Enable Sort/i }).click(); cy.findByRole('button', { name: /Save/ }).click(); - cy.findByRole('menuitem', { name: new RegExp(workspaceName, 'i') }).click(); - cy.get('ul') - .contains(new RegExp(dashboardName, 'i')) - .click(); + console.log('Saved'); + cy.findByText(new RegExp(dashboardName, 'i')).should('exist'); } } diff --git a/cypress/support/Studios/StudioList.ts b/cypress/support/Studios/StudioList.ts index 1618b18a4..b6546e146 100644 --- a/cypress/support/Studios/StudioList.ts +++ b/cypress/support/Studios/StudioList.ts @@ -3,7 +3,6 @@ export class StudioListPage { cy.findByRole('button', { name: /Create Studio/i }).click(); cy.findByRole('textbox', { name: /Label/ }).type(name); cy.findByRole('button', { name: /Save/ }).click(); - const studioElement = cy.findByRole('heading', { name }); studioElement.contains(name); return studioElement; diff --git a/cypress/support/Utils/nexus.ts b/cypress/support/Utils/nexus.ts new file mode 100644 index 000000000..d27b30acd --- /dev/null +++ b/cypress/support/Utils/nexus.ts @@ -0,0 +1,67 @@ +import { NexusClient, ResourcePayload } from '@bbp/nexus-sdk'; + +export const createNexusOrgAndProject = async ({ + nexus, + orgLabel, + orgDescription, + projectLabel, + projectDescription, +}: { + nexus: NexusClient; + orgLabel: string; + orgDescription: string; + projectLabel: string; + projectDescription: string; +}) => { + // Create Org if does not exist + try { + const org = await nexus.Organization.get(orgLabel); + + if (org._deprecated) { + throw new Error('Org deprecated'); + } + } catch (e) { + await nexus.Organization.create(orgLabel, { + description: orgDescription, + }); + } + + // Create project if does not exist + try { + await nexus.Project.get(orgLabel, projectLabel); + } catch (e) { + await nexus.Project.create(orgLabel, projectLabel, { + description: projectDescription, + }); + } +}; + +export const deprecateNexusOrgAndProject = async ({ + nexus, + orgLabel, + projectLabel, +}: { + nexus: NexusClient; + orgLabel: string; + projectLabel: string; +}) => { + // Deprecate project + try { + const project = await nexus.Project.get(orgLabel, projectLabel); + await nexus.Project.deprecate(orgLabel, projectLabel, project._rev); + } catch (e) { + console.error('Encountered an error whilst trying to deprecate project', e); + } +}; + +export const createResource = async ({ + nexus, + orgLabel, + projectLabel, + resource, +}: { + nexus: NexusClient; + orgLabel: string; + projectLabel: string; + resource: ResourcePayload; +}) => await nexus.Resource.create(orgLabel, projectLabel, resource); diff --git a/cypress/support/setupRealmsAndUsers.ts b/cypress/support/setupRealmsAndUsers.ts index 85a325fc8..8d6b5b1c4 100644 --- a/cypress/support/setupRealmsAndUsers.ts +++ b/cypress/support/setupRealmsAndUsers.ts @@ -3,7 +3,7 @@ const fetch = require('node-fetch'); const fs = require('fs'); const keycloakUrl = 'http://keycloak.test:8080'; -const deltaBaseUrl = 'http://delta.test:8080/v1'; +const deltaBaseUrl = 'http://delta.test:8098/v1'; // /* these are used for administrating keycloak */ const keycloakAdmin = { diff --git a/docs/development.md b/docs/development.md index c735dba1a..4feac8cbc 100644 --- a/docs/development.md +++ b/docs/development.md @@ -250,3 +250,63 @@ yarn cy:run --e2e --browser chrome ``` All of the tests will run in headless mode. + +## Debugging End-to-End test failures in CI + +There are a few differences between how our e2e tests run in the ci compared to how we run it locally. We use different instances of delta, postgress, keycloak etc in these 2 environments. Also, the version of chrome you may have locally might differ from the version of chrome being used in the ci. Finally, the machine cpu, memory are also most likely different. This can make debugging test failures in ci difficult. + +The videos and screenshots of tests are uploaded to [cypress cloud](https://cloud.cypress.io/). If these are not sufficient to debug the issue, follow along. + +1. Remove your node_modules folder locally and reinstall the dependencies to be sure that they are exactly the same as in ci: + +``` +rm -rf node_modules +yarn install --frozen-lockfile +``` + +2. Create the image for fusion: + +``` +sudo docker build . --tag=nexus-web:fresh +``` + +You may need to run the above command with sudo privileges + +3. [Optional] You may want to mount your local code inside the container. This will allow you to make changes and see their results quickly, as opposed to make changes, copy the changes into the container, and then see their results. You'll need to update the `cypress` service section inside your `docker-compose.yml` for this: + +``` + cypress: + image: 'cypress/included:12.17.0' + volumes: # This is the change + - ../:/e2e + user: ${CYPRESS_USER} # You can also set a user here to avoid issues with file permissions. This is optional. +``` + +4. Start services defined in docker-compose file: + +If you followed step 3: + +``` +CYPRESS_USER=$UID sudo --preserve-env=CYPRESS_USER docker-compose -f ci/docker-compose.yml up -d +``` + +Otherwise: + +``` +sudo docker-compose -f ci/docker-compose.yml up -d +sudo docker cp ./. cypress:/e2e +``` + +5. Run the tests as they would in ci: + +``` +yarn run cy:ci +``` + +6. You can also navigate to http://fusion.test:8000 in your browser to see the same instance of fusion that cypress is testing. + +7. Remember to stop the services once done: + +``` +sudo docker-compose -f ci/docker-compose.yml down --rmi "local" --volumes +``` diff --git a/package.json b/package.json index e1a1e38df..a94597307 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,10 @@ "start": "NODE_ENV=development DEBUG=* webpack --mode development --config-name server && node dist/server.js", "build": "NODE_ENV=production NODE_OPTIONS=--max_old_space_size=8192 webpack --mode production", "test": "jest", - "test:watch": "jest --watch", + "test:watch": "jest --watch --maxWorkers=4", "cy:open": "cypress open", "cy:run": "cypress run", + "cy:ci": "echo | sudo docker exec --user=$UID -t cypress cypress run --config-file cypress.config.ts --browser chrome", "test-ui": "API_ENDPOINT=http://test start-server-and-test start http://localhost:8000 cy:run", "codecov": "codecov", "lint": "tslint --project tsconfig.json", @@ -114,7 +115,7 @@ "@storybook/addon-links": "^5.0.1", "@storybook/addons": "^5.0.1", "@storybook/react": "^5.0.10", - "@testing-library/cypress": "^8.0.3", + "@testing-library/cypress": "9.0.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "12.1.2", "@testing-library/react-hooks": "^8.0.1", @@ -151,7 +152,7 @@ "babel-loader": "^8.0.4", "copy-webpack-plugin": "^5.1.1", "css-loader": "^2.1.1", - "cypress": "^10.3.0", + "cypress": "^12.17.0", "cypress-file-upload": "^5.0.8", "enzyme": "^3.9.0", "enzyme-adapter-react-16": "^1.10.0", @@ -211,7 +212,10 @@ ], "globals": { "FUSION_VERSION": "1.0.0", - "COMMIT_HASH": "9013fa343" + "COMMIT_HASH": "9013fa343", + "ts-jest": { + "isolatedModules": true + } }, "watchPathIgnorePatterns": [ "node_modules" diff --git a/src/__mocks__/handlers/DataExplorer/handlers.ts b/src/__mocks__/handlers/DataExplorer/handlers.ts index 523e8b574..5e1ea3c6b 100644 --- a/src/__mocks__/handlers/DataExplorer/handlers.ts +++ b/src/__mocks__/handlers/DataExplorer/handlers.ts @@ -4,6 +4,7 @@ import { Resource } from '@bbp/nexus-sdk'; import { AggregatedBucket, AggregationsResult, + NexusMultiFetchResponse, } from 'subapps/dataExplorer/DataExplorerUtils'; export const getCompleteResources = ( @@ -16,26 +17,78 @@ export const dataExplorerPageHandler = ( partialResources: Resource[] = defaultPartialResources, total: number = 300 ) => { - return rest.get(deltaPath(`/resources`), (req, res, ctx) => { - if (req.url.searchParams.has('aggregations')) { - return res(ctx.status(200), ctx.json(mockAggregationsResult())); + return [ + rest.get(deltaPath(`/resources`), (req, res, ctx) => { + if (req.url.searchParams.has('aggregations')) { + return res(ctx.status(200), ctx.json(mockAggregationsResult())); + } + const passedType = req.url.searchParams.get('type'); + const mockResponse = { + '@context': [ + 'https://bluebrain.github.io/nexus/contexts/metadata.json', + 'https://bluebrain.github.io/nexus/contexts/search.json', + 'https://bluebrain.github.io/nexus/contexts/search-metadata.json', + ], + _total: total, + _results: passedType + ? partialResources.filter(res => res['@type'] === passedType) + : partialResources, + _next: + 'https://bbp.epfl.ch/nexus/v1/resources?size=50&sort=@id&after=%5B1687269183553,%22https://bbp.epfl.ch/neurosciencegraph/data/31e22529-2c36-44f0-9158-193eb50526cd%22%5D', + }; + return res(ctx.status(200), ctx.json(mockResponse)); + }), + rest.post(deltaPath('/multi-fetch/resources'), (req, res, ctx) => { + const requestedIds = ((req.body as any)?.resources).map( + (res: { id: string }) => res.id + ); + const response: NexusMultiFetchResponse = { + format: 'compacted', + resources: partialResources + .filter(res => requestedIds.includes(res['@id'])) + .map(r => ({ value: { ...r, ...propertiesOnlyInSource } })), + }; + return res(ctx.status(200), ctx.json(response)); + }), + ]; +}; + +export const graphAnalyticsTypeHandler = () => { + return rest.get( + deltaPath('/graph-analytics/:org/:project/properties/:type'), + (req, res, ctx) => { + const mockResponse = { + '@context': + 'https://bluebrain.github.io/nexus/contexts/properties.json', + '@id': 'http://schema.org/Dataset', + _count: 50, + _name: 'Dataset', + _properties: [ + { + '@id': 'https://neuroshapes.org/tag__apical', + _count: 30, + _name: 'author', + }, + { + '@id': 'https://neuroshapes.org/nr__reconstruction_type', + _count: 30, + _name: 'propertyAlwaysThere', + }, + { + '@id': 'https://neuroshapes.org/createdBy', + _count: 30, + _name: '_createdBy', + }, + { + '@id': 'https://neuroshapes.org/edition', + _count: 30, + _name: 'edition', + }, + ], + }; + return res(ctx.status(200), ctx.json(mockResponse)); } - const passedType = req.url.searchParams.get('type'); - const mockResponse = { - '@context': [ - 'https://bluebrain.github.io/nexus/contexts/metadata.json', - 'https://bluebrain.github.io/nexus/contexts/search.json', - 'https://bluebrain.github.io/nexus/contexts/search-metadata.json', - ], - _total: total, - _results: passedType - ? partialResources.filter(res => res['@type'] === passedType) - : partialResources, - _next: - 'https://bbp.epfl.ch/nexus/v1/resources?size=50&sort=@id&after=%5B1687269183553,%22https://bbp.epfl.ch/neurosciencegraph/data/31e22529-2c36-44f0-9158-193eb50526cd%22%5D', - }; - return res(ctx.status(200), ctx.json(mockResponse)); - }); + ); }; const propertiesOnlyInSource = { userProperty1: { subUserProperty1: 'bar' } }; @@ -109,6 +162,30 @@ export const filterByProjectHandler = ( }); }; +export const elasticSearchQueryHandler = (resources: Resource[]) => { + return rest.post( + deltaPath('/graph-analytics/:org/:project/_search'), + (req, res, ctx) => { + const esResponse = { + hits: { + hits: resources.map(resource => ({ + _id: resource['@id'], + _source: { + '@id': resource['@id'], + _project: resource._project, + }, + })), + max_score: 0, + total: { + value: 479, + }, + }, + }; + return res(ctx.status(200), ctx.json(esResponse)); + } + ); +}; + const mockAggregationsResult = ( bucketForTypes: AggregatedBucket[] = defaultBucketForTypes ): AggregationsResult => { @@ -205,6 +282,8 @@ export const getMockResource = ( '@type': type, _constrainedBy: 'https://bluebrain.github.io/nexus/schemas/unconstrained.json', + propertyAlwaysThere: + 'Mock property in all test resources in DataExplorer spec', _createdAt: '2023-06-21T09:39:47.217Z', _createdBy: 'https://bbp.epfl.ch/nexus/v1/realms/bbp/users/antonel', _deprecated: false, diff --git a/src/shared/components/EditTableForm.tsx b/src/shared/components/EditTableForm.tsx index 13caf971d..3283ba687 100644 --- a/src/shared/components/EditTableForm.tsx +++ b/src/shared/components/EditTableForm.tsx @@ -567,7 +567,11 @@ const EditTableForm: React.FC<{ > {availableViews && availableViews.map(view => ( - ))} diff --git a/src/shared/molecules/TypeSelector/TypeSelector.tsx b/src/shared/molecules/TypeSelector/TypeSelector.tsx index c8077137d..bfec67e33 100644 --- a/src/shared/molecules/TypeSelector/TypeSelector.tsx +++ b/src/shared/molecules/TypeSelector/TypeSelector.tsx @@ -166,7 +166,6 @@ const TypeSelector = ({ ]; const selectCallback = useCallback((data: TTypeAggregationsResult) => { - console.log('@@selectCallback', data); const options = ( data.aggregations.types?.buckets ?? ([] as TTypesAggregatedBucket[]) ).map(item => typesOptionsBuilder(item)); diff --git a/src/subapps/dataExplorer/DataExplorer-utils.spec.tsx b/src/subapps/dataExplorer/DataExplorer-utils.spec.tsx index f651285a6..9fa3add32 100644 --- a/src/subapps/dataExplorer/DataExplorer-utils.spec.tsx +++ b/src/subapps/dataExplorer/DataExplorer-utils.spec.tsx @@ -1,3 +1,7 @@ +import { + GraphAnalyticsProperty, + getUniquePathsForProperties, +} from './DataExplorerUtils'; import { doesResourceContain, getAllPaths, @@ -680,4 +684,100 @@ describe('DataExplorerSpec-Utils', () => { checkPathExistence(resource, '@context.@vocab', 'does-not-exist') ).toEqual(true); }); + + it('can get paths for propeties with no nesting', () => { + const mockProperties: GraphAnalyticsProperty[] = [ + { + _name: 'prop1', + _properties: [], + }, + { + _name: 'prop2', + _properties: [], + }, + { + _name: 'prop3', + _properties: [], + }, + ]; + + const actualPaths = getUniquePathsForProperties(mockProperties); + expect(actualPaths).toEqual(['prop1', 'prop2', 'prop3']); + }); + + it('gets paths for 1 level deep nested properties', () => { + const mockProperties: GraphAnalyticsProperty[] = [ + { + _name: 'prop1', + _properties: [{ _name: 'sub1', _properties: [] }], + }, + { + _name: 'prop2', + _properties: [], + }, + { + _name: 'prop3', + _properties: [{ _name: 'sub1', _properties: [] }], + }, + ]; + + const actualPaths = getUniquePathsForProperties(mockProperties); + expect(actualPaths).toEqual([ + 'prop1', + 'prop1.sub1', + 'prop2', + 'prop3', + 'prop3.sub1', + ]); + }); + + it('gets paths for multi level nested properties', () => { + const mockProperties: GraphAnalyticsProperty[] = [ + { + _name: 'who', + _properties: [ + { + _name: 'let', + _properties: [ + { + _name: 'the', + _properties: [ + { + _name: 'dogs', + _properties: [{ _name: 'out?', _properties: [] }], + }, + ], + }, + ], + }, + ], + }, + { + _name: 'who', + _properties: [ + { _name: 'who', _properties: [{ _name: 'who?', _properties: [] }] }, + ], + }, + { + _name: 'by', + _properties: [ + { _name: 'baha', _properties: [{ _name: 'men', _properties: [] }] }, + ], + }, + ]; + + const actualPaths = getUniquePathsForProperties(mockProperties); + expect(actualPaths).toEqual([ + 'who', + 'who.let', + 'who.let.the', + 'who.let.the.dogs', + 'who.let.the.dogs.out?', + 'who.who', + 'who.who.who?', + 'by', + 'by.baha', + 'by.baha.men', + ]); + }); }); diff --git a/src/subapps/dataExplorer/DataExplorer.spec.tsx b/src/subapps/dataExplorer/DataExplorer.spec.tsx index 799e23eb7..1c6bec8f9 100644 --- a/src/subapps/dataExplorer/DataExplorer.spec.tsx +++ b/src/subapps/dataExplorer/DataExplorer.spec.tsx @@ -6,9 +6,11 @@ import userEvent from '@testing-library/user-event'; import { UserEvent } from '@testing-library/user-event/dist/types/setup/setup'; import { dataExplorerPageHandler, + elasticSearchQueryHandler, filterByProjectHandler, getCompleteResources, getMockResource, + graphAnalyticsTypeHandler, sourceResourceHandler, } from '__mocks__/handlers/DataExplorer/handlers'; import { deltaPath } from '__mocks__/handlers/handlers'; @@ -23,6 +25,7 @@ import { DOES_NOT_CONTAIN, DOES_NOT_EXIST, EXISTS, + FRONTEND_PREDICATE_WARNING, getAllPaths, } from './PredicateSelector'; import { createMemoryHistory } from 'history'; @@ -35,17 +38,19 @@ window.scrollTo = jest.fn(); describe('DataExplorer', () => { const defaultTotalResults = 500_123; + const ALWAYS_PRESENT_RESOURCE_PROPERTY = 'propertyAlwaysThere'; const mockResourcesOnPage1: Resource[] = getCompleteResources(); const mockResourcesForPage2: Resource[] = [ - getMockResource('self1', { author: 'piggy', edition: 1 }), - getMockResource('self2', { author: ['iggy', 'twinky'] }), - getMockResource('self3', { year: 2013 }), + getMockResource('self1', { author: 'piggy', edition: 1 }, 'unhcr', 'file'), + getMockResource('self2', { author: ['iggy', 'twinky'] }, 'unhcr', 'file'), + getMockResource('self3', { year: 2013 }, 'unhcr', 'file'), ]; const server = setupServer( - dataExplorerPageHandler(undefined, defaultTotalResults), + ...dataExplorerPageHandler(undefined, defaultTotalResults), sourceResourceHandler(), - filterByProjectHandler() + filterByProjectHandler(), + graphAnalyticsTypeHandler() ); const history = createMemoryHistory({}); @@ -61,7 +66,12 @@ describe('DataExplorer', () => { fetch, uri: deltaPath(), }); - const store = configureStore(history, { nexus }, {}); + + const store = configureStore( + history, + { nexus }, + { config: { apiEndpoint: 'https://localhost:3000' } } + ); dataExplorerPage = ( @@ -135,6 +145,7 @@ describe('DataExplorer', () => { selector: 'th .ant-table-column-title', exact: false, }); + expect(header).toBeInTheDocument(); return header; }; @@ -230,7 +241,7 @@ describe('DataExplorer', () => { ) => { server.use( sourceResourceHandler(resources), - dataExplorerPageHandler(resources, total) + ...dataExplorerPageHandler(resources, total) ); const pageInput = await screen.getByRole('listitem', { name: '2' }); @@ -265,7 +276,23 @@ describe('DataExplorer', () => { return menuDropdown; }; - const selectPath = async (path: string) => { + const selectPath = async ( + path: string, + project: string = 'unhcr', + type: string = 'file' + ) => { + // Select `project` project if it is not already selected + const projectInput = await getInputForLabel(ProjectMenuLabel); + if (!projectInput.value.match(new RegExp(project, 'i'))) { + await selectOptionFromMenu(ProjectMenuLabel, project); + } + + // Select `type` type if it is not already selected + const typeInput = await getSelectedValueInMenu(TypeMenuLabel); + if (!typeInput?.match(new RegExp(type, 'i'))) { + await selectOptionFromMenu(TypeMenuLabel, type, TypeOptionSelector); + } + await selectOptionFromMenu(PathMenuLabel, path, CustomOptionSelector); }; @@ -330,6 +357,18 @@ describe('DataExplorer', () => { await expectRowCountToBe(10); await getRowsForNextPage(resources); await expectRowCountToBe(resources.length); + server.use(filterByProjectHandler(mockResourcesForPage2)); + }; + + const mockElasticSearchHits = ( + path: string, + predicate: typeof EXISTS | typeof DOES_NOT_EXIST, + resources: Resource[] + ) => { + const matchingResources = resources.filter(res => + predicate === EXISTS ? res[path] : !res[path] + ); + server.use(elasticSearchQueryHandler(matchingResources)); }; const getResetProjectButton = async () => { @@ -365,8 +404,6 @@ describe('DataExplorer', () => { expect(pro).toBeVisible(); const type = await getInputForLabel(TypeMenuLabel); expect(type).toBeVisible(); - const predicate = await getInputForLabel(PathMenuLabel); - expect(predicate).toBeVisible(); const totalResultsCount = await getTotalSizeOfDataset('500,123'); expect(totalResultsCount).toBeVisible(); const metadataSwitch = await showMetadataSwitch(); @@ -492,7 +529,7 @@ describe('DataExplorer', () => { mock100Resources.push(getMockResource(`self${i}`, {})); } - server.use(dataExplorerPageHandler(mock100Resources)); + server.use(...dataExplorerPageHandler(mock100Resources)); const pageSizeChanger = await screen.getByRole('combobox', { name: 'Page Size', @@ -649,37 +686,17 @@ describe('DataExplorer', () => { ).rejects.toThrowError(); }); - it('shows paths as options in a select menu of path selector', async () => { - await expectRowCountToBe(10); - await openMenuFor('path-selector'); - - const pathOptions = getVisibleOptionsFromMenu(CustomOptionSelector); - - const expectedPaths = getAllPaths(mockResourcesOnPage1); - expect(expectedPaths.length).toBeGreaterThanOrEqual( - Object.keys(mockResourcesOnPage1[0]).length - ); - - pathOptions.forEach((path, index) => { - expect(path.innerHTML).toMatch( - new RegExp(`${expectedPaths[index]}$`, 'i') - ); - }); - - expect(pathOptions.length).toBeGreaterThanOrEqual(3); // Since antd options in a select menu are displayed in a virtual list (by default), not all expected options are in the DOM. - }); - it('shows resources that have path missing', async () => { - await updateResourcesShownInTable([ - getMockResource('self1', { author: 'piggy', edition: 1 }), - getMockResource('self2', { author: ['iggy', 'twinky'] }), - getMockResource('self3', { year: 2013 }), - ]); + await updateResourcesShownInTable(mockResourcesForPage2); + mockElasticSearchHits('author', DOES_NOT_EXIST, mockResourcesForPage2); await selectPath('author'); await selectOptionFromMenu(PredicateMenuLabel, DOES_NOT_EXIST); await expectRowCountToBe(1); + await resetPredicate(); + + mockElasticSearchHits('edition', DOES_NOT_EXIST, mockResourcesForPage2); await selectPath('edition'); await selectOptionFromMenu(PredicateMenuLabel, DOES_NOT_EXIST); await expectRowCountToBe(2); @@ -719,11 +736,8 @@ describe('DataExplorer', () => { }); it('shows resources that have a path when user selects exists predicate', async () => { - await updateResourcesShownInTable([ - getMockResource('self1', { author: 'piggy', edition: 1 }), - getMockResource('self2', { author: ['iggy', 'twinky'] }), - getMockResource('self3', { year: 2013 }), - ]); + await updateResourcesShownInTable(mockResourcesForPage2); + mockElasticSearchHits('author', EXISTS, mockResourcesForPage2); await selectPath('author'); await userEvent.click(container); @@ -799,24 +813,61 @@ describe('DataExplorer', () => { expect(totalFromFrontend).toEqual(null); }); - it('shows total filtered count if predicate is selected', async () => { + it('does not show total filtered count if backend predicate is selected', async () => { await expectRowCountToBe(10); - await updateResourcesShownInTable([ - getMockResource('self1', { author: 'piggy', edition: 1 }), - getMockResource('self2', { author: ['iggy', 'twinky'] }), - getMockResource('self3', { year: 2013 }), - ]); + await updateResourcesShownInTable(mockResourcesForPage2); + mockElasticSearchHits('author', EXISTS, mockResourcesForPage2); await selectPath('author'); await userEvent.click(container); await selectOptionFromMenu(PredicateMenuLabel, EXISTS); await expectRowCountToBe(2); - const totalFromFrontendAfter = await getFilteredResultsCount(2); - expect(totalFromFrontendAfter).toBeVisible(); + expect(await getFilteredResultsCount()).toBeFalsy(); + + mockElasticSearchHits('author', DOES_NOT_EXIST, mockResourcesForPage2); + await selectOptionFromMenu(PredicateMenuLabel, DOES_NOT_EXIST); + await expectRowCountToBe(1); + expect(await getFilteredResultsCount()).toBeFalsy(); + }); + + it('shows filtered count if frontend predicate is selected', async () => { + await updateResourcesShownInTable(mockResourcesForPage2); + + await selectPath('author'); + await selectOptionFromMenu(PredicateMenuLabel, CONTAINS); + const valueInput = await screen.getByPlaceholderText('Search for...'); + await userEvent.type(valueInput, 'twinky'); + await expectRowCountToBe(1); + + expect(await getFilteredResultsCount(1)).toBeVisible(); + }); + + it('shows predicate warning if frontend predicate is selected', async () => { + await updateResourcesShownInTable(mockResourcesForPage2); + + await selectPath('author'); + await selectOptionFromMenu(PredicateMenuLabel, CONTAINS); + + expect(await screen.getByText(FRONTEND_PREDICATE_WARNING)).toBeVisible(); + + await selectOptionFromMenu(PredicateMenuLabel, DOES_NOT_CONTAIN); + expect(await screen.getByText(FRONTEND_PREDICATE_WARNING)).toBeVisible(); + }); + + it('does not show predicate warning if backend predicate is selected', async () => { + await updateResourcesShownInTable(mockResourcesForPage2); + + await selectPath('author'); + await selectOptionFromMenu(PredicateMenuLabel, EXISTS); + + expect(await screen.queryByText(FRONTEND_PREDICATE_WARNING)).toBeFalsy(); + await selectOptionFromMenu(PredicateMenuLabel, EXISTS); + expect(await screen.queryByText(FRONTEND_PREDICATE_WARNING)).toBeFalsy(); }); it('shows column for metadata path even if toggle for show metadata is off', async () => { const metadataProperty = '_createdBy'; + mockElasticSearchHits(metadataProperty, EXISTS, mockResourcesOnPage1); await expectRowCountToBe(10); await expectColumHeaderToNotExist(metadataProperty); @@ -826,6 +877,8 @@ describe('DataExplorer', () => { await selectPath(metadataProperty); await selectOptionFromMenu(PredicateMenuLabel, EXISTS); + await expectRowCountToBe(10); + await expectColumHeaderToExist(metadataProperty); expect(getTotalColumns().length).toEqual(originalColumns + 1); @@ -835,13 +888,14 @@ describe('DataExplorer', () => { it('resets predicate fields when reset predicate clicked', async () => { await updateResourcesShownInTable(mockResourcesForPage2); + mockElasticSearchHits('author', EXISTS, mockResourcesForPage2); await selectPath('author'); + await selectPredicate(EXISTS); const selectedPathBefore = await getSelectedValueInMenu(PathMenuLabel); expect(selectedPathBefore).toMatch(/author/); - await expectRowCountToBe(2); await resetPredicate(); @@ -854,8 +908,9 @@ describe('DataExplorer', () => { it('only shows predicate menu if path is selected', async () => { await expectRowCountToBe(10); + expect(openMenuFor(PredicateMenuLabel)).rejects.toThrow(); - await selectPath('@type'); + await selectPath(ALWAYS_PRESENT_RESOURCE_PROPERTY); expect(openMenuFor(PredicateMenuLabel)).resolves.not.toThrow(); }); @@ -953,9 +1008,7 @@ describe('DataExplorer', () => { }); it('does not reset values in filters when header was hidden due to scroll', async () => { - await selectOptionFromMenu(ProjectMenuLabel, 'unhcr'); - await selectOptionFromMenu(TypeMenuLabel, 'file', TypeOptionSelector); - await selectPath('@type'); + await selectPath(ALWAYS_PRESENT_RESOURCE_PROPERTY); await scrollWindow(500); await waitForHeaderToBeHidden(); @@ -970,12 +1023,16 @@ describe('DataExplorer', () => { expect(typeInput).toMatch(new RegExp('file', 'i')); const pathInput = await getSelectedValueInMenu(PathMenuLabel); - expect(pathInput).toMatch(new RegExp('@type', 'i')); + expect(pathInput).toMatch( + new RegExp(ALWAYS_PRESENT_RESOURCE_PROPERTY, 'i') + ); }); it('resets predicate search term when different predicate verb is selected', async () => { await updateResourcesShownInTable(mockResourcesForPage2); + await selectPath('author'); + await selectPredicate(CONTAINS); const valueInput = await screen.getByPlaceholderText('Search for...'); await userEvent.type(valueInput, 'iggy'); @@ -987,4 +1044,18 @@ describe('DataExplorer', () => { const valueInputAfter = await screen.getByPlaceholderText('Search for...'); expect((valueInputAfter as HTMLInputElement).value).not.toEqual('iggy'); }); + + it('disables predicate selector if multiple types are selected', async () => { + await selectOptionFromMenu(ProjectMenuLabel, 'unhcr'); + await selectOptionFromMenu(TypeMenuLabel, 'file', TypeOptionSelector); + + expect(await getInputForLabel(PathMenuLabel)).toBeEnabled(); + + await selectOptionFromMenu( + TypeMenuLabel, + 'StudioDashboard', + TypeOptionSelector + ); + expect(await getInputForLabel(PathMenuLabel)).toBeDisabled(); + }); }); diff --git a/src/subapps/dataExplorer/DataExplorer.tsx b/src/subapps/dataExplorer/DataExplorer.tsx index 60d90f555..35febbfe8 100644 --- a/src/subapps/dataExplorer/DataExplorer.tsx +++ b/src/subapps/dataExplorer/DataExplorer.tsx @@ -10,6 +10,7 @@ import { columnFromPath, isUserColumn, sortColumns, + useGraphAnalyticsPath, usePaginatedExpandedResources, } from './DataExplorerUtils'; import { @@ -39,7 +40,8 @@ export interface DataExplorerConfiguration { offset: number; orgAndProject?: [string, string]; types?: TType[]; - predicate: ((resource: Resource) => boolean) | null; + frontendPredicate: ((resource: Resource) => boolean) | null; + backendPredicateQuery: Object | null; selectedPath: string | null; deprecated: boolean; columns: TColumn[]; @@ -121,6 +123,7 @@ export const columnsFromDataSource = ( ) .sort(sortColumns); }; + const DataExplorer: React.FC<{}> = () => { const history = useHistory(); const [showMetadataColumns, setShowMetadataColumns] = useState(false); @@ -136,7 +139,11 @@ const DataExplorer: React.FC<{}> = () => { pageSize, offset, orgAndProject, - predicate, + // NOTE: Right now, the `EXISTS` and `DOES_NOT_EXIST` predicates run on the backend and update the `backendPredicateQuery` parameter. + // `CONTAINS` and `DOES_NOT_CONTAIN` predicates on the other hand, only run on frontend and update `frontendPredicate` parameter. + // When we implement running all the predicates on backend, we should discard `frontendPredicate` parameter completely. + frontendPredicate, + backendPredicateQuery, types, selectedPath, deprecated, @@ -157,7 +164,8 @@ const DataExplorer: React.FC<{}> = () => { offset: 0, orgAndProject: undefined, types: [], - predicate: null, + frontendPredicate: null, + backendPredicateQuery: null, selectedPath: null, deprecated: false, columns: [], @@ -172,12 +180,13 @@ const DataExplorer: React.FC<{}> = () => { deprecated, typeOperator, types: types?.map(t => t.value), + predicateQuery: backendPredicateQuery, }); const currentPageDataSource: Resource[] = resources?._results || []; - const displayedDataSource = predicate - ? currentPageDataSource.filter(predicate) + const displayedDataSource = frontendPredicate + ? currentPageDataSource.filter(frontendPredicate) : currentPageDataSource; const buildColumns = useMemo(() => { @@ -223,6 +232,7 @@ const DataExplorer: React.FC<{}> = () => { updateTableConfiguration({ columns: newColumns }); updateSelectedColumnsCached(newColumns); }; + useEffect(() => { const selectedFilters = getSelectedFiltersCached(); if (selectedFilters) { @@ -284,6 +294,9 @@ const DataExplorer: React.FC<{}> = () => { }); return () => unlisten(); }, []); + + const shouldShowPredicateSelector = orgAndProject?.length === 2; + return (
{isLoading && } @@ -347,12 +360,16 @@ const DataExplorer: React.FC<{}> = () => { document.getElementById('data-explorer-filters')! } /> - + {shouldShowPredicateSelector ? ( + + ) : null}
= () => { types={types} nexusTotal={resources?._total ?? 0} totalOnPage={resources?._results?.length ?? 0} - totalFiltered={predicate ? displayedDataSource.length : undefined} + totalFiltered={ + frontendPredicate ? displayedDataSource.length : undefined + } />
{ const nexus = useNexusContext(); + const { apiEndpoint } = useSelector((state: RootState) => state.config); return useQuery({ queryKey: [ 'data-explorer', @@ -24,6 +29,7 @@ export const usePaginatedExpandedResources = ({ pageSize, offset, orgAndProject, + predicateQuery, ...(types?.length ? { types, @@ -34,6 +40,18 @@ export const usePaginatedExpandedResources = ({ ], retry: false, queryFn: async () => { + if (predicateQuery && orgAndProject) { + return getResultsForPredicateQuery( + nexus, + apiEndpoint, + orgAndProject[0], + orgAndProject[1], + predicateQuery, + pageSize, + offset + ); + } + const resultWithPartialResources = await nexus.Resource.list( orgAndProject?.[0], orgAndProject?.[1], @@ -48,39 +66,14 @@ export const usePaginatedExpandedResources = ({ } ); - // If we failed to fetch the expanded source for some resources, we can use the compact/partial resource as a fallback. - const fallbackResources: Resource[] = []; - const { results: expandedResources } = await PromisePool.withConcurrency( - 4 - ) - .for(resultWithPartialResources._results) - .handleError(async (err, partialResource) => { - console.log( - `@@error in fetching resource with id: ${partialResource['@id']}`, - err - ); - fallbackResources.push(partialResource); - return; - }) - .process(async partialResource => { - if (partialResource._project) { - const { org, project } = makeOrgProjectTuple( - partialResource._project - ); - - return (await nexus.Resource.get( - org, - project, - encodeURIComponent(partialResource['@id']), - { annotate: true } - )) as Resource; - } - - return partialResource; - }); + const expandedResources = await fetchMultipleResources( + nexus, + apiEndpoint, + resultWithPartialResources._results + ); return { ...resultWithPartialResources, - _results: [...expandedResources, ...fallbackResources], + _results: expandedResources, }; }, onError: error => { @@ -102,6 +95,63 @@ export const usePaginatedExpandedResources = ({ }); }; +export type NexusResourceFormats = + | 'source' + | 'compacted' + | 'expanded' + | 'n-triples' + | 'dot'; +export type NexusMultiFetchResponse = { + format: NexusResourceFormats; + resources: { value: Resource }[]; +}; + +type PartialResource = Pick; + +export const fetchMultipleResources = async ( + nexus: ReturnType, + apiEndpoint: string, + partialResources: PartialResource[], + orgAndProject?: string +): Promise => { + const resourceData = partialResources + .filter(resource => resource._project) + .map(resource => { + if (orgAndProject) { + return { + id: resource['@id'], + project: orgAndProject, + }; + } + + const { org, project } = makeOrgProjectTuple(resource._project); + + return { + id: resource['@id'], + project: `${org}/${project}`, + }; + }); + + if (resourceData.length === 0) { + return []; + } + + const multipleResources: NexusMultiFetchResponse = await nexus + .httpPost({ + path: `${apiEndpoint}/multi-fetch/resources`, + headers: { Accept: 'application/json' }, + body: JSON.stringify({ + format: 'compacted', + resources: resourceData, + }), + }) + .catch(() => { + return { format: 'compacted', value: [] }; + }); + + return multipleResources.resources.map(({ value }) => ({ ...value })); +}; + export const useAggregations = ( bucketName: 'projects' | 'types', orgAndProject?: string[] @@ -128,6 +178,123 @@ export const useAggregations = ( }); }; +export type GraphAnalyticsProperty = { + '@id'?: string; // TODO Make necessory + _name: string; + _count?: number; + _properties: GraphAnalyticsProperty[]; +}; + +type GraphAnalyticsResponse = { + _properties: GraphAnalyticsProperty[]; +}; + +export const useGraphAnalyticsPath = ( + org: string, + project: string, + types: string[] +) => { + const nexus = useNexusContext(); + return useQuery({ + queryKey: ['graph-analytics-paths', org, project, types], + retry: false, + staleTime: Infinity, + queryFn: async () => { + return (await nexus.GraphAnalytics.properties( + project, + org, + types[0] + )) as GraphAnalyticsResponse; + }, + select: data => { + return getPathsForProperties(data._properties); + }, + }); +}; + +const getResultsForPredicateQuery = async ( + nexus: ReturnType, + apiEndpoint: string, + org: string, + project: string, + query: Object, + pageSize: number, + offset: number +) => { + const searchResults: SearchResponse<{ + '@id': string; + _project: string; + }> = await nexus.httpPost({ + path: `${apiEndpoint}/graph-analytics/${org}/${project}/_search`, + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + body: JSON.stringify({ + from: offset, + size: pageSize, + ...query, + }), + }); + + const resourcesToFetch = searchResults.hits.hits.map(matching => ({ + '@id': matching._source['@id'], + _project: matching._source['_project'], + })); + console.log( + 'Requesting matching resources', + resourcesToFetch.map(r => r['@id']) + ); + const matchingResources = await fetchMultipleResources( + nexus, + apiEndpoint, + resourcesToFetch + ); + + return { + _results: matchingResources, + _total: searchResults.hits.total.value, + } as PaginatedList; +}; + +export const getUniquePathsForProperties = ( + properties: GraphAnalyticsProperty[], + paths: string[] = [], + pathSoFar?: string +): string[] => { + properties?.forEach(property => { + const name = pathSoFar ? `${pathSoFar}.${property._name}` : property._name; + paths.push(name); + getUniquePathsForProperties(property._properties ?? [], paths, name); + }); + + return Array.from(new Set(paths)); +}; + +export type PropertyPath = { + label: string; + value: string; +}; + +export const getPathsForProperties = ( + properties: GraphAnalyticsProperty[], + paths: PropertyPath[] = [], + pathSoFar?: string, + valueSoFar?: string +): PropertyPath[] => { + properties?.forEach(property => { + const label = pathSoFar ? `${pathSoFar}.${property._name}` : property._name; + const value = valueSoFar + ? `${valueSoFar} / ${property['@id']!}` + : property['@id']!; + paths.push({ label, value }); + getPathsForProperties(property._properties ?? [], paths, label, value); + }); + + const uniquePaths = new Set(paths.map(path => path.value)); + return paths.filter(path => uniquePaths.has(path.value)); +}; + export const sortColumns = (a: string, b: string) => { // Sorts paths alphabetically. Additionally all paths starting with an underscore are sorted at the end of the list (because they represent metadata). const columnA = columnFromPath(a); @@ -203,6 +370,7 @@ interface PaginatedResourcesParams { deprecated: boolean; types?: string[]; typeOperator: TTypeOperator; + predicateQuery: Object | null; } export const useTimeoutMessage = ({ diff --git a/src/subapps/dataExplorer/PredicateSelector.tsx b/src/subapps/dataExplorer/PredicateSelector.tsx index 9d3e05c6f..c4c0e3157 100644 --- a/src/subapps/dataExplorer/PredicateSelector.tsx +++ b/src/subapps/dataExplorer/PredicateSelector.tsx @@ -1,38 +1,46 @@ import { UndoOutlined } from '@ant-design/icons'; import { Resource } from '@bbp/nexus-sdk'; -import { Button, Form, Input, Select } from 'antd'; +import { Button, Form, Input, Select, Tooltip } from 'antd'; import { FormInstance } from 'antd/es/form'; import { DefaultOptionType } from 'antd/lib/cascader'; -import React, { useMemo, useRef } from 'react'; +import React, { useEffect, useRef } from 'react'; +import { TType } from 'shared/molecules/TypeSelector/types'; import { normalizeString } from '../../utils/stringUtils'; +import { TColumn } from './ColumnsSelector'; import { DataExplorerConfiguration } from './DataExplorer'; import { + PropertyPath, columnFromPath, isObject, isUserColumn, sortColumns, + useGraphAnalyticsPath, } from './DataExplorerUtils'; -import { TColumn } from './ColumnsSelector'; import './styles.less'; interface Props { columns: TColumn[]; - dataSource: Resource[]; onPredicateChange: React.Dispatch>; onResetCallback: (column: string, checked: boolean) => void; + org: string; + project: string; + types: TType[] | undefined; } export const PredicateSelector: React.FC = ({ columns, - dataSource, onPredicateChange, onResetCallback, + org, + project, + types, }: Props) => { const formRef = useRef(null); const pathRef = useRef<{ path: string; selected: boolean }>({ path: '', selected: false, }); + const predicateFilterOptions: PredicateFilterOptions[] = [ { value: EXISTS }, { value: DOES_NOT_EXIST }, @@ -40,64 +48,94 @@ export const PredicateSelector: React.FC = ({ { value: DOES_NOT_CONTAIN }, ]; - const allPathOptions = useMemo( - () => pathOptions([...getAllPaths(dataSource)]), - [dataSource] + const { data: paths, isLoading: arePathsLoading } = useGraphAnalyticsPath( + org, + project, + types?.map(t => t.value) ?? [] ); + // NOTE: Right now, the `EXISTS` and `DOES_NOT_EXIST` predicates run on the backend and update the `backendPredicateQuery` parameter. + // `CONTAINS` and `DOES_NOT_CONTAIN` predicates on the other hand, only run on frontend and update `frontendPredicate` parameter. + // When we implement running all the predicates on backend, we should discard `frontendPredicate` parameter completely. const predicateSelected = ( - path: string, + path: DefaultOptionType, predicate: PredicateFilterOptions['value'] | null, searchTerm: string | null ) => { if (!path || !predicate) { - onPredicateChange({ predicate: null, selectedPath: null }); + onPredicateChange({ + frontendPredicate: null, + selectedPath: null, + backendPredicateQuery: null, + }); } switch (predicate) { - case EXISTS: + case EXISTS: { onPredicateChange({ - predicate: (resource: Resource) => - checkPathExistence(resource, path, 'exists'), - selectedPath: path, + backendPredicateQuery: getPredicateQuery( + predicate, + types![0].value, + path.value as string + ), + frontendPredicate: null, + selectedPath: path.key, }); break; - case DOES_NOT_EXIST: + } + case DOES_NOT_EXIST: { onPredicateChange({ - predicate: (resource: Resource) => - checkPathExistence(resource, path, 'does-not-exist'), - selectedPath: path, + backendPredicateQuery: getPredicateQuery( + predicate, + types![0].value, + path.value as string + ), + frontendPredicate: null, + selectedPath: path.key, }); break; + } case CONTAINS: if (searchTerm) { onPredicateChange({ - predicate: (resource: Resource) => - doesResourceContain(resource, path, searchTerm, 'contains'), - selectedPath: path, + frontendPredicate: (resource: Resource) => + doesResourceContain(resource, path.key, searchTerm, 'contains'), + selectedPath: path.key, }); } else { - onPredicateChange({ predicate: null, selectedPath: null }); + onPredicateChange({ + frontendPredicate: null, + selectedPath: null, + backendPredicateQuery: null, + }); } break; case DOES_NOT_CONTAIN: if (searchTerm) { onPredicateChange({ - predicate: (resource: Resource) => + frontendPredicate: (resource: Resource) => doesResourceContain( resource, - path, + path.key, searchTerm, 'does-not-contain' ), - selectedPath: path, + selectedPath: path.key, }); } else { - onPredicateChange({ predicate: null, selectedPath: null }); + onPredicateChange({ + frontendPredicate: null, + selectedPath: null, + backendPredicateQuery: null, + }); } break; default: { - onPredicateChange({ predicate: null, selectedPath: null }); + onPredicateChange({ + frontendPredicate: null, + selectedPath: null, + backendPredicateQuery: null, + }); break; } } @@ -107,7 +145,10 @@ export const PredicateSelector: React.FC = ({ return formRef.current?.getFieldValue(fieldName) ?? ''; }; - const setFormField = (fieldName: string, fieldValue: string) => { + const setFormField = ( + fieldName: string, + fieldValue: string | DefaultOptionType + ) => { if (formRef.current) { formRef.current.setFieldValue(fieldName, fieldValue); } @@ -120,75 +161,104 @@ export const PredicateSelector: React.FC = ({ form.resetFields(); } pathRef.current = { path: '', selected: false }; - onPredicateChange({ predicate: null, selectedPath: null }); + onPredicateChange({ + frontendPredicate: null, + backendPredicateQuery: null, + selectedPath: null, + }); }; + useEffect(() => { + onReset(); + }, [types]); + const shouldShowValueInput = getFormFieldValue(PREDICATE_FIELD) === CONTAINS || getFormFieldValue(PREDICATE_FIELD) === DOES_NOT_CONTAIN; + const disablePredicateSelection = types?.length !== 1; + const isFrontendPredicateSelected = + getFormFieldValue(PREDICATE_FIELD) === CONTAINS || + getFormFieldValue(PREDICATE_FIELD) === DOES_NOT_CONTAIN; + return (
with - { + setFormField(PATH_FIELD, pathLabel); + predicateSelected( + pathLabel, + getFormFieldValue(PREDICATE_FIELD), + getFormFieldValue(SEARCH_TERM_FIELD) + ); + }} + disabled={disablePredicateSelection} + loading={arePathsLoading} + allowClear={true} + onClear={() => { + onReset(); + }} + virtual={true} + className="select-menu" + popupClassName="search-menu" + optionLabelProp="label" + aria-label="path-selector" + style={{ width: 200, minWidth: 'max-content' }} + dropdownMatchSelectWidth={false} // This ensures that the items in the dropdown list are always fully legible (ie they are not truncated) just because the input of select is too short. + /> + {getFormFieldValue(PATH_FIELD) && ( <> = - - { + setFormField(PREDICATE_FIELD, predicateLabel); + setFormField(SEARCH_TERM_FIELD, ''); + const selectedPath = getFormFieldValue(PATH_FIELD); + pathRef.current = { + path: selectedPath.key, + selected: + columns.find(column => column.value === selectedPath) + ?.selected ?? false, + }; + + predicateSelected(selectedPath, predicateLabel, ''); + }} + aria-label="predicate-selector" + className="select-menu reduced-width" + popupClassName="search-menu" + autoFocus={true} + allowClear={true} + onClear={() => { + predicateSelected(getFormFieldValue(PATH_FIELD), null, ''); + }} + /> + + {isFrontendPredicateSelected && ( + + {FRONTEND_PREDICATE_WARNING} + + )} +
)} @@ -230,6 +300,9 @@ export const EXISTS = 'Exists'; export const CONTAINS = 'Contains'; export const DOES_NOT_CONTAIN = 'Does not contain'; +export const FRONTEND_PREDICATE_WARNING = + 'This predicate will only run on the resources loaded in the current page.'; + const PATH_FIELD = 'path'; const PREDICATE_FIELD = 'predicate'; const SEARCH_TERM_FIELD = 'searchTerm'; @@ -245,23 +318,93 @@ type PredicateFilterOptions = { value: Exclude; }; +const getPredicateQuery = ( + predicateVerb: typeof EXISTS | typeof DOES_NOT_EXIST, + type: string, + path: string +) => { + if (predicateVerb === EXISTS) { + return { + query: { + bool: { + filter: [ + { + terms: { + '@type': [type], + }, + }, + { + term: { + _deprecated: false, + }, + }, + { + nested: { + path: 'properties', + query: { + term: { 'properties.path': path }, + }, + }, + }, + ], + }, + }, + }; + } + if (predicateVerb === DOES_NOT_EXIST) { + return { + query: { + bool: { + filter: [ + { + terms: { + '@type': [type], + }, + }, + { + term: { + _deprecated: false, + }, + }, + { + bool: { + must_not: { + nested: { + path: 'properties', + query: { + term: { 'properties.path': path }, + }, + }, + }, + }, + }, + ], + }, + }, + }; + } + + return null; +}; + // Creates