diff --git a/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_a_filesystem.md b/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_a_filesystem.md index 97fe9e5da2e6..a9e73e141c31 100644 --- a/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_a_filesystem.md +++ b/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_a_filesystem.md @@ -3,9 +3,6 @@ title: How to host and share Data Docs on a filesystem --- import Prerequisites from '../../connecting_to_your_data/components/prerequisites.jsx' import TechnicalTag from '@site/docs/term_tags/_tag.mdx'; -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - This guide will explain how to host and share on a filesystem. @@ -36,20 +33,9 @@ data_docs_sites: ### 2. Test that your configuration is correct by building the site -Use the following command: ``great_expectations docs build --site-name local_site``. If successful, the CLI will open your newly built Data Docs site and provide the path to the index page. - -```bash -> great_expectations docs build --site-name local_site - -The following Data Docs sites will be built: - - - local_site: file:///great_expectations/uncommitted/data_docs/local_site/index.html - -Would you like to proceed? [Y/n]: Y - -Building Data Docs... +Run the the following Python code to build and open your Data Docs: -Done building Data Docs +``` python name="tests/integration/docusaurus/reference/glossary/data_docs.py data_docs" ``` ## Additional notes diff --git a/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_amazon_s3.md b/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_amazon_s3.md index 5641304d87d6..6300d295e837 100644 --- a/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_amazon_s3.md +++ b/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_amazon_s3.md @@ -9,9 +9,6 @@ import ApplyThePolicy from './components_how_to_host_and_share_data_docs_on_amaz import AddANewS3SiteToTheDataDocsSitesSectionOfYourGreatExpectationsYml from './components_how_to_host_and_share_data_docs_on_amazon_s3/_add_a_new_s3_site_to_the_data_docs_sites_section_of_your_great_expectationsyml.mdx' import TestThatYourConfigurationIsCorrectByBuildingTheSite from './components_how_to_host_and_share_data_docs_on_amazon_s3/_test_that_your_configuration_is_correct_by_building_the_site.mdx' import AdditionalNotes from './components_how_to_host_and_share_data_docs_on_amazon_s3/_additional_notes.mdx' -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - diff --git a/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_azure_blob_storage.md b/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_azure_blob_storage.md index 6f971920b38b..39017e764948 100644 --- a/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_azure_blob_storage.md +++ b/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_azure_blob_storage.md @@ -3,9 +3,6 @@ title: How to host and share Data Docs on Azure Blob Storage --- import Prerequisites from '../../connecting_to_your_data/components/prerequisites.jsx' import TechnicalTag from '@site/docs/term_tags/_tag.mdx'; -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - This guide will explain how to host and share on Azure Blob Storage. Data Docs will be served using an Azure Blob Storage static website with restricted access. @@ -55,7 +52,7 @@ data_docs_sites: base_directory: uncommitted/data_docs/local_site/ site_index_builder: class_name: DefaultSiteIndexBuilder - az_site: # this is a user-selected name - you may select your own + new_site_name: # this is a user-selected name - you can select your own class_name: SiteBuilder store_backend: class_name: TupleAzureBlobStoreBackend @@ -97,26 +94,12 @@ The most common authentication methods are supported: ### 4. Build the Azure Blob Data Docs site You can create or modify an and this will build the Data Docs website. -Or you can use the following command: ``great_expectations docs build --site-name az_site``. - -```bash -> great_expectations docs build --site-name az_site - - The following Data Docs sites will be built: - - - az_site: https://.blob.core.windows.net/$web/index.html - Would you like to proceed? [Y/n]: y +Run the the following Python code to build and open your Data Docs: - Building Data Docs... - - Done building Data Docs +``` python name="tests/integration/docusaurus/reference/glossary/data_docs.py data_docs_site" ``` -If successful, the CLI will provide the object URL of the index page. -You may secure the access of your website using an IP filtering mechanism. - - ### 5. Limit the access to your company - On your Azure Storage Account Settings click on **Networking** diff --git a/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.md b/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.md index 4a652270a9cc..182a31f8a4fd 100644 --- a/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.md +++ b/docs/docusaurus/docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.md @@ -3,9 +3,6 @@ title: How to host and share Data Docs on GCS --- import Prerequisites from '../../connecting_to_your_data/components/prerequisites.jsx' import TechnicalTag from '@site/docs/term_tags/_tag.mdx'; -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - This guide will explain how to host and share on Google Cloud Storage. We recommend using IP-based access, which is achieved by deploying a simple Google App Engine app. Data Docs can also be served on Google Cloud Storage if the contents of the bucket are set to be publicly readable, but this is strongly discouraged. @@ -58,7 +55,7 @@ We recommend placing it in your project directory, for example ``great_expectati ### 4. Deploy your Google App Engine app -Issue the following command from within the app directory created above: +Run the following CLI command from within the app directory you created previously: ```bash name="tests/integration/docusaurus/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.py gcloud app deploy" ``` @@ -76,14 +73,9 @@ You may also replace the default ``local_site`` if you would only like to mainta ### 7. Build the GCS Data Docs site -Use the following CLI command: - -```bash name="tests/integration/docusaurus/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.py build data docs command" -``` - -If successful, the CLI will provide the object URL of the index page. Since the bucket is not public, this URL will be inaccessible. Rather, you will access the Data Docs site using the App Engine app configured above. +Run the the following Python code to build and open your Data Docs: -```bash name="tests/integration/docusaurus/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.py build data docs output" +``` python name="tests/integration/docusaurus/reference/glossary/data_docs.py data_docs_site" ``` ### 8. Test that everything was configured properly by launching your App Engine app diff --git a/docs/docusaurus/docs/guides/setup/installation/local.md b/docs/docusaurus/docs/guides/setup/installation/local.md index dbcfc19f4da8..6717c89b3370 100644 --- a/docs/docusaurus/docs/guides/setup/installation/local.md +++ b/docs/docusaurus/docs/guides/setup/installation/local.md @@ -9,9 +9,6 @@ import InstallGreatExpectations from './components_local/_install_great_expectat import VerifyGeInstallSucceeded from './components_local/_verify_ge_install_succeeded.mdx' import NextSteps from '/docs/guides/setup/components/install_nextsteps.md' import InstallCongratulations from '/docs/guides/setup/components/install_congrats.md' -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - diff --git a/docs/docusaurus/docs/guides/setup/optional_dependencies/cloud/how_to_set_up_gx_to_work_with_data_on_aws_s3.md b/docs/docusaurus/docs/guides/setup/optional_dependencies/cloud/how_to_set_up_gx_to_work_with_data_on_aws_s3.md index adfd8d189d9a..bbb456c02df9 100644 --- a/docs/docusaurus/docs/guides/setup/optional_dependencies/cloud/how_to_set_up_gx_to_work_with_data_on_aws_s3.md +++ b/docs/docusaurus/docs/guides/setup/optional_dependencies/cloud/how_to_set_up_gx_to_work_with_data_on_aws_s3.md @@ -9,9 +9,6 @@ keywords: [Great Expectations, Data Context, Filesystem, Amazon Web Services S3] import TechnicalTag from '/docs/term_tags/_tag.mdx'; import Prerequisites from '/docs/components/_prerequisites.jsx' -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - import PrereqInstalledAwsCli from '/docs/components/prerequisites/_aws_installed_the_aws_cli.mdx' diff --git a/docs/docusaurus/docs/guides/setup/optional_dependencies/cloud/how_to_set_up_gx_to_work_with_data_on_gcs.md b/docs/docusaurus/docs/guides/setup/optional_dependencies/cloud/how_to_set_up_gx_to_work_with_data_on_gcs.md index 6f21d2b92c95..7c83ddbce267 100644 --- a/docs/docusaurus/docs/guides/setup/optional_dependencies/cloud/how_to_set_up_gx_to_work_with_data_on_gcs.md +++ b/docs/docusaurus/docs/guides/setup/optional_dependencies/cloud/how_to_set_up_gx_to_work_with_data_on_gcs.md @@ -9,9 +9,6 @@ keywords: [Great Expectations, Data Context, Filesystem, GCS, Google Cloud Stora import TechnicalTag from '/docs/term_tags/_tag.mdx'; import Prerequisites from '/docs/components/_prerequisites.jsx' -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - import PrereqGcpServiceAccount from '/docs/components/prerequisites/_gcp_service_account.md' diff --git a/docs/docusaurus/docs/guides/validation/advanced/how_to_deploy_a_scheduled_checkpoint_with_cron.md b/docs/docusaurus/docs/guides/validation/advanced/how_to_deploy_a_scheduled_checkpoint_with_cron.md index 8fc831016182..ab78a2d40fb2 100644 --- a/docs/docusaurus/docs/guides/validation/advanced/how_to_deploy_a_scheduled_checkpoint_with_cron.md +++ b/docs/docusaurus/docs/guides/validation/advanced/how_to_deploy_a_scheduled_checkpoint_with_cron.md @@ -3,9 +3,6 @@ title: How to deploy a scheduled Checkpoint with cron --- import Prerequisites from '../../connecting_to_your_data/components/prerequisites.jsx' import TechnicalTag from '@site/docs/term_tags/_tag.mdx'; -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - This guide will help you deploy a scheduled with cron. @@ -22,10 +19,9 @@ This guide will help you deploy a scheduled [OpenLineage](https://openlineage.io) is an open framework for collection and analysis of data lineage. It tracks the movement of data over time, tracing relationships between datasets. Data engineers can use data lineage metadata to determine the root cause of failures, identify performance bottlenecks, and simulate the effects of planned changes. @@ -57,11 +54,10 @@ action_list: ### 3. Test your Action by Validating a Batch of data. -Run your Checkpoint to Validate a of data and emit lineage events to the OpenLineage backend. This can be done from the command line: +Run the following command to retrieve and run a Checkpoint to Validate a of data and then emit lineage events to the OpenLineage backend: - ```bash - % great_expectations checkpoint run - ``` +```python name="tests/integration/docusaurus/reference/glossary/checkpoints.py retrieve_and_run" +``` :::note Reminder Our [guide on how to Validate data by running a Checkpoint](../how_to_validate_data_by_running_a_checkpoint.md) has more detailed instructions for this step, including instructions on how to run a checkpoint from a Python script instead of from the . diff --git a/docs/docusaurus/docs/guides/validation/validation_actions/how_to_update_data_docs_as_a_validation_action.md b/docs/docusaurus/docs/guides/validation/validation_actions/how_to_update_data_docs_as_a_validation_action.md index e71357aa4dc9..f489b66505d5 100644 --- a/docs/docusaurus/docs/guides/validation/validation_actions/how_to_update_data_docs_as_a_validation_action.md +++ b/docs/docusaurus/docs/guides/validation/validation_actions/how_to_update_data_docs_as_a_validation_action.md @@ -4,9 +4,6 @@ title: How to update Data Docs after Validating a Checkpoint import Prerequisites from '../../../guides/connecting_to_your_data/components/prerequisites.jsx'; import TechnicalTag from '@site/docs/term_tags/_tag.mdx'; -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - This guide will explain how to use an to update sites with new from running a . @@ -73,7 +70,7 @@ The ``StoreValidationResultAction`` Action must appear before ``UpdateDataDocsA Test that your new Action is configured correctly: -Run the Checkpoint from your code or the and verify that no errors are thrown. +Run the following command to run the Checkpoint and verify that no errors are returned: ```python import great_expectations as gx @@ -81,9 +78,6 @@ context = gx.get_context() checkpoint_name = "your checkpoint name here" context.run_checkpoint(checkpoint_name=checkpoint_name) ``` -```bash -$ great_expectations checkpoint run -``` Finally, check your Data Docs sites to confirm that a new Validation Result has been added. diff --git a/docs/docusaurus/docs/integrations/integration_datahub.md b/docs/docusaurus/docs/integrations/integration_datahub.md index e9638e1ef429..c2eeae378525 100644 --- a/docs/docusaurus/docs/integrations/integration_datahub.md +++ b/docs/docusaurus/docs/integrations/integration_datahub.md @@ -5,10 +5,6 @@ authors: url: https://datahubproject.io --- -import CLIRemoval from '/docs/components/warnings/_cli_removal.md' - - - :::info * Maintained By: DataHub * Status: Beta @@ -85,8 +81,9 @@ action_list: #### 3. Run the GX checkpoint -```bash -great_expectations checkpoint run my_checkpoint #replace my_checkpoint with your checkpoint name +Run the following command to retrieve and run a Checkpoint: + +```python name="tests/integration/docusaurus/reference/glossary/checkpoints.py retrieve_and_run" ``` #### 4. Hurray! diff --git a/tests/integration/common_workflows/simple_build_data_docs.py b/tests/integration/common_workflows/simple_build_data_docs.py deleted file mode 100644 index adf7695a2690..000000000000 --- a/tests/integration/common_workflows/simple_build_data_docs.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import tempfile - -import great_expectations as gx -from great_expectations.data_context.types.base import ( - DataContextConfig, - DatasourceConfig, - FilesystemStoreBackendDefaults, -) -from great_expectations.util import get_context - -""" -A simple test to verify that `context.build_data_docs()` works as expected. - -As indicated in issue #3772, calling `context.build_data_docs()` raised an unexpected exception -when Great Expectations was installed in a non-filesystem location (i.e. it failed when -GX was installed inside a zip file -which is a location allowed by PEP 273-). - -Therefore, this test is intended to be run after installing GX inside a zip file and -then setting the appropriate PYTHONPATH env variable. If desired, this test can also be -run after installing GX in a normal filesystem location (i.e. a directory). - -This test is OK if it finishes without raising an exception. - -To make it easier to debug this test, it prints: -* The location of the GX library: to verify that we are testing the library that we want -* The version of the GX library: idem -* data_docs url: If everything works, this will be a url (e.g. starting with file://...) - - -Additional info: https://github.com/great-expectations/great_expectations/issues/3772 and -https://www.python.org/dev/peps/pep-0273/ -""" - -print(f"Great Expectations location: {gx.__file__}") -print(f"Great Expectations version: {gx.__version__}") - -data_context_config = DataContextConfig( - datasources={"example_datasource": DatasourceConfig(class_name="PandasDatasource")}, - store_backend_defaults=FilesystemStoreBackendDefaults( - root_directory=tempfile.mkdtemp() + os.sep + "my_greatexp_workdir" - ), -) -context = get_context(project_config=data_context_config) -print(f"Great Expectations data_docs url: {context.build_data_docs()}") diff --git a/tests/integration/docusaurus/reference/glossary/checkpoints.py b/tests/integration/docusaurus/reference/glossary/checkpoints.py new file mode 100644 index 000000000000..278427174f6d --- /dev/null +++ b/tests/integration/docusaurus/reference/glossary/checkpoints.py @@ -0,0 +1,44 @@ +from great_expectations.datasource.fluent import Datasource +from great_expectations.datasource.fluent import DataAsset +from great_expectations.checkpoint import SimpleCheckpoint + +# +import great_expectations as gx + +context = gx.get_context() +# + +# to open Data Docs, we need validation results which we get by creating a suite and running a checkpoint +datasource: Datasource = context.get_datasource("taxi_datasource") +asset: DataAsset = datasource.get_asset("yellow_tripdata") +batch_request = asset.build_batch_request() +validator = context.get_validator(batch_request=batch_request) + +validator.expect_column_values_to_not_be_null("pickup_datetime") +validator.expect_column_values_to_be_between("passenger_count", auto=True) + +taxi_suite = validator.get_expectation_suite() +taxi_suite.expectation_suite_name = "taxi_suite" + +context.add_expectation_suite(expectation_suite=taxi_suite) + +# +checkpoint = SimpleCheckpoint( + name="taxi_checkpoint", + data_context=context, + batch_request=batch_request, + expectation_suite_name="taxi_suite", +) +checkpoint.run() +# + +# +context.add_checkpoint(checkpoint=checkpoint) +# + +# +checkpoint = context.get_checkpoint("taxi_checkpoint") +checkpoint.run() +# + +assert True diff --git a/tests/integration/docusaurus/reference/glossary/data_docs.py b/tests/integration/docusaurus/reference/glossary/data_docs.py new file mode 100644 index 000000000000..8a054f687086 --- /dev/null +++ b/tests/integration/docusaurus/reference/glossary/data_docs.py @@ -0,0 +1,41 @@ +import great_expectations as gx +from great_expectations.datasource.fluent import Datasource +from great_expectations.datasource.fluent import DataAsset +from great_expectations.checkpoint import SimpleCheckpoint + +context = gx.get_context() + +# to open data docs, we need validation results which we get by creating a suite and running a checkpoint +datasource: Datasource = context.get_datasource("taxi_datasource") +asset: DataAsset = datasource.get_asset("yellow_tripdata") +batch_request = asset.build_batch_request() +validator = context.get_validator(batch_request=batch_request) + +validator.expect_column_values_to_not_be_null("pickup_datetime") +validator.expect_column_values_to_be_between("passenger_count", auto=True) + +taxi_suite = validator.get_expectation_suite() +taxi_suite.expectation_suite_name = "taxi_suite" + +context.add_expectation_suite(expectation_suite=taxi_suite) + +checkpoint = SimpleCheckpoint( + name="taxi_checkpoint", + data_context=context, + batch_request=batch_request, + expectation_suite_name="taxi_suite", +) +checkpoint.run() + +# +context.build_data_docs() +context.open_data_docs() +# + +# +site_name = "new_site_name" +context.build_data_docs(site_names=site_name) +context.open_data_docs(site_name=site_name) +# + +assert True diff --git a/tests/integration/docusaurus/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.py b/tests/integration/docusaurus/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.py index 06202fff43b2..d28ebfee574d 100644 --- a/tests/integration/docusaurus/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.py +++ b/tests/integration/docusaurus/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_gcs.py @@ -191,7 +191,7 @@ def server_error(e): base_directory: uncommitted/data_docs/local_site/ site_index_builder: class_name: DefaultSiteIndexBuilder - gs_site: # this is a user-selected name - you may select your own + new_site_name: # this is a user-selected name - you may select your own class_name: SiteBuilder store_backend: class_name: TupleGCSStoreBackend @@ -220,12 +220,12 @@ def server_error(e): build_data_docs_command = """ # -great_expectations docs build --site-name gs_site +great_expectations docs build --site-name new_site_name # """ # Overwrite the version with the snippet tags build_data_docs_command = """ -great_expectations docs build --site-name gs_site +great_expectations docs build --site-name new_site_name """ result = subprocess.Popen( @@ -239,7 +239,7 @@ def server_error(e): # The following Data Docs sites will be built: - - gs_site: https://storage.googleapis.com//index.html + - new_site_name: https://storage.googleapis.com//index.html Would you like to proceed? [Y/n]: Y Building Data Docs... diff --git a/tests/integration/fixtures/yellow_trip_data_fluent_pandas/great_expectations.yml b/tests/integration/fixtures/yellow_trip_data_fluent_pandas/great_expectations.yml deleted file mode 100644 index 22fa04ce5460..000000000000 --- a/tests/integration/fixtures/yellow_trip_data_fluent_pandas/great_expectations.yml +++ /dev/null @@ -1,70 +0,0 @@ -config_version: 3.0 -datasources: {} -config_variables_file_path: uncommitted/config_variables.yml -plugins_directory: plugins/ -stores: - expectations_store: - class_name: ExpectationsStore - store_backend: - class_name: TupleFilesystemStoreBackend - base_directory: expectations/ - validations_store: - class_name: ValidationsStore - store_backend: - class_name: TupleFilesystemStoreBackend - base_directory: uncommitted/validations/ - evaluation_parameter_store: - class_name: EvaluationParameterStore - checkpoint_store: - class_name: CheckpointStore - store_backend: - class_name: TupleFilesystemStoreBackend - suppress_store_backend_id: true - base_directory: checkpoints/ - profiler_store: - class_name: ProfilerStore - store_backend: - class_name: TupleFilesystemStoreBackend - suppress_store_backend_id: true - base_directory: profilers/ -expectations_store_name: expectations_store -validations_store_name: validations_store -evaluation_parameter_store_name: evaluation_parameter_store -checkpoint_store_name: checkpoint_store -data_docs_sites: - local_site: - class_name: SiteBuilder - show_how_to_buttons: true - store_backend: - class_name: TupleFilesystemStoreBackend - base_directory: uncommitted/data_docs/local_site - site_index_builder: - class_name: DefaultSiteIndexBuilder -anonymous_usage_statistics: - data_context_id: fd1a2c05-09d3-438c-bbe6-ff24ebb04f3d - usage_statistics_url: https://dev.stats.greatexpectations.io/great_expectations/v1/usage_statistics - enabled: true -notebooks: -include_rendered_content: - expectation_validation_result: false - globally: false - expectation_suite: false -fluent_datasources: - taxi_source: - type: pandas_filesystem - assets: - yellow_tripdata: - type: csv - order_by: - - key: year - reverse: true - - key: month - reverse: false - batching_regex: yellow_tripdata_sample_(?P\d{4})-(?P\d{2}).csv - base_directory: data - taxi_frame_source: - type: pandas - assets: - taxi_file_asset: - type: csv - filepath_or_buffer: data/yellow_tripdata_sample_2019-01.csv diff --git a/tests/integration/fixtures/yellow_trip_data_fluent_pandas/great_expectations/great_expectations.yml b/tests/integration/fixtures/yellow_trip_data_fluent_pandas/great_expectations/great_expectations.yml index 95a42005f165..3cace54de7ca 100644 --- a/tests/integration/fixtures/yellow_trip_data_fluent_pandas/great_expectations/great_expectations.yml +++ b/tests/integration/fixtures/yellow_trip_data_fluent_pandas/great_expectations/great_expectations.yml @@ -40,15 +40,35 @@ data_docs_sites: base_directory: uncommitted/data_docs/local_site site_index_builder: class_name: DefaultSiteIndexBuilder + new_site_name: + class_name: SiteBuilder + show_how_to_buttons: true + store_backend: + class_name: TupleFilesystemStoreBackend + base_directory: uncommitted/data_docs/local_site + site_index_builder: + class_name: DefaultSiteIndexBuilder notebooks: include_rendered_content: expectation_validation_result: false globally: false expectation_suite: false fluent_datasources: - taxi_source: + taxi_datasource: type: pandas_filesystem assets: - taxi_asset: + yellow_tripdata: type: csv + order_by: + - key: year + reverse: true + - key: month + reverse: false + batching_regex: yellow_tripdata_sample_(?P\d{4})-(?P\d{2}).csv base_directory: data + taxi_frame_source: + type: pandas + assets: + taxi_file_asset: + type: csv + filepath_or_buffer: data/yellow_tripdata_sample_2019-01.csv diff --git a/tests/integration/test_script_runner.py b/tests/integration/test_script_runner.py index 97fe41d359fe..53f73a360428 100644 --- a/tests/integration/test_script_runner.py +++ b/tests/integration/test_script_runner.py @@ -228,6 +228,12 @@ data_context_dir="tests/integration/fixtures/no_datasources/great_expectations", data_dir="tests/test_sets/taxi_yellow_tripdata_samples", ), + IntegrationTestFixture( + name="data_docs", + user_flow_script="tests/integration/docusaurus/reference/glossary/data_docs.py", + data_context_dir="tests/integration/fixtures/yellow_trip_data_fluent_pandas/great_expectations", + data_dir="tests/test_sets/taxi_yellow_tripdata_samples", + ), IntegrationTestFixture( name="how_to_edit_an_existing_expectation_suite", user_flow_script="tests/integration/docusaurus/expectations/how_to_edit_an_expectation_suite.py", @@ -319,6 +325,12 @@ data_dir="tests/test_sets/taxi_yellow_tripdata_samples/first_3_files", backend_dependencies=[BackendDependencies.PANDAS], ), + IntegrationTestFixture( + name="checkpoints_glossary", + user_flow_script="tests/integration/docusaurus/reference/glossary/checkpoints.py", + data_context_dir="tests/integration/fixtures/yellow_trip_data_fluent_pandas/great_expectations", + data_dir="tests/test_sets/taxi_yellow_tripdata_samples", + ), IntegrationTestFixture( name="how_to_organize_batches_in_a_file_based_data_asset", user_flow_script="tests/integration/docusaurus/connecting_to_your_data/fluent_datasources/organize_batches_in_pandas_filesystem_datasource.py", @@ -398,10 +410,6 @@ ), ), ), - IntegrationTestFixture( - name="build_data_docs", - user_flow_script="tests/integration/common_workflows/simple_build_data_docs.py", - ), ] # populate integration_test_matrix with sub-lists