train_job.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "# DONUT TRAINING"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "## 1. Prepare workspace, environment, and cluster"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from azure.ai.ml.entities import AmlCompute\n",
        "from azure.ai.ml import MLClient\n",
        "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n",
        "from azure.ai.ml.entities import Environment\n",
        "from azure.ai.ml import command\n",
        "from azure.ai.ml import Input\n",
        "from azure.ai.ml.constants import AssetTypes"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "Prepare the workspace"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "gather": {
          "logged": 1688625485238
        }
      },
      "outputs": [],
      "source": [
        "try:\n",
        "    credential = DefaultAzureCredential()\n",
        "    # Check if given credential can get token successfully.\n",
        "    credential.get_token(\"https://management.azure.com/.default\")\n",
        "except Exception as ex:\n",
        "    # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n",
        "    credential = InteractiveBrowserCredential()\n",
        "\n",
        "# Get a handle to the workspace\n",
        "ml_client = MLClient(\n",
        "    credential=credential,\n",
        "    subscription_id=\"\", # enter credentials in Step 5\n",
        "    resource_group_name=\"\", # enter credentials in Step 5\n",
        "    workspace_name=\"\", # enter credentials in Step 5\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "Create the environment to be used later by the compute cluster"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1681384016962
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "env_docker_conda = Environment(\n",
        "    image=\"mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04\",\n",
        "    conda_file=\"environment.yaml\",\n",
        "    name=\"donut\",\n",
        "    description=\"Donut\",\n",
        ")\n",
        "ml_client.environments.create_or_update(env_docker_conda)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "Create compute resource, change the value of min_instances, max_instances if you want to have a cluster of a different size."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1688027298790
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "gpu_compute_target = \"gpu-cluster\"\n",
        "\n",
        "try:\n",
        "    # let's see if the compute target already exists\n",
        "    gpu_cluster = ml_client.compute.get(gpu_compute_target)\n",
        "    print(\n",
        "        f\"You already have a cluster named {gpu_compute_target}, we'll reuse it as is.\"\n",
        "    )\n",
        "\n",
        "except Exception:\n",
        "    print(\"Creating a new gpu compute target...\")\n",
        "\n",
        "    gpu_cluster = AmlCompute(\n",
        "        name=\"gpu-cluster\",\n",
        "        type=\"amlcompute\",\n",
        "        size=\"STANDARD_NC6\",  # 4 x NVIDIA Tesla K80\n",
        "        min_instances=0,\n",
        "        max_instances=4,\n",
        "        idle_time_before_scale_down=120,\n",
        "        tier=\"Dedicated\",\n",
        "    )\n",
        "\n",
        "    gpu_cluster = ml_client.begin_create_or_update(gpu_cluster)\n",
        "\n",
        "print(\n",
        "    f\"AMLCompute with name {gpu_cluster.name} is created, the compute size is {gpu_cluster.size}\"\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 2. Create and submit job"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "gather": {
          "logged": 1688054804119
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "job = command(\n",
        "    inputs={\n",
        "       \"data\": Input(\n",
        "            type=\"uri_folder\",\n",
        "            path= \"\", # replace by the data uri in step 4\n",
        "            mode = \"ro_mount\"\n",
        "        ),\n",
        "        \"config\":  \"config/train_passport_terminal.yaml\" , # this file contain the hyperparameters and configuration for the training\n",
        "        \"exp_name\": \"mrz\", # experiment name, change it as you like\n",
        "      \n",
        "    },\n",
        "    code=\"./\",  # location of source code\n",
        "    command=\"python train.py --dataset_name_or_paths ${{inputs.data}} --config ${{inputs.config}} --exp_name ${{inputs.exp_name}} \",\n",
        "    environment=\"donut@latest\", # this refers to the environment created above\n",
        "    compute=\"gpu-cluster\", # this refers to the compute cluster created above\n",
        "    experiment_name = 'mrz',\n",
        "    display_name=\"main_job\", \n",
        "    instance_count=4,  \n",
        "    distribution={\n",
        "        \"type\": \"PyTorch\",\n",
        "        \"process_count_per_instance\": 1,  # number of GPus per node\n",
        "    },\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "After the job is uploaded, you can click on the link to get to the \"Jobs\" page"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "gather": {
          "logged": 1688054840236
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Your file exceeds 100 MB. If you experience low speeds, latency, or broken connections, we recommend using the AzCopyv10 tool for this file transfer.\n",
            "\n",
            "Example: azcopy copy '/mnt/batch/tasks/shared/LS_root/mounts/clusters/script-launch/code/Users/thithuyduyen.pham/donut_training/donut' 'https://safetravel.blob.core.windows.net/azureml-blobstore-61ed3287-b3b7-4ed7-8306-4e23b936201e/LocalUpload/49b805722c10f089f08ac1c4b979aba3/donut' \n",
            "\n",
            "See https://docs.microsoft.com/azure/storage/common/storage-use-azcopy-v10 for more information.\n",
            "\u001b[32mUploading donut (143.05 MBs): 100%|██████████| 143052127/143052127 [00:09<00:00, 14928318.03it/s]\n",
            "\u001b[39m\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<table style=\"width:100%\"><tr><th>Experiment</th><th>Name</th><th>Type</th><th>Status</th><th>Details Page</th></tr><tr><td>mrz</td><td>honest_stamp_5th95nc8zj</td><td>command</td><td>Starting</td><td><a href=\"https://ml.azure.com/runs/honest_stamp_5th95nc8zj?wsid=/subscriptions/cd8cfbf0-b616-4927-8541-6083db80f5d0/resourcegroups/dev-computer-vision/workspaces/safetravel-dev-ml&amp;tid=d3bc2180-cb1e-40f7-b59a-154105743342\" target=\"_blank\" rel=\"noopener\">Link to Azure Machine Learning studio</a></td></tr></table>"
            ],
            "text/plain": [
              "Command({'parameters': {}, 'init': False, 'type': 'command', 'status': 'Starting', 'log_files': None, 'name': 'honest_stamp_5th95nc8zj', 'description': None, 'tags': {}, 'properties': {'_azureml.ComputeTargetType': 'amlctrain', 'ContentSnapshotId': 'deac72d8-9dd6-4631-bffc-6c9e3c5677d2'}, 'print_as_yaml': True, 'id': '/subscriptions/cd8cfbf0-b616-4927-8541-6083db80f5d0/resourceGroups/dev-computer-vision/providers/Microsoft.MachineLearningServices/workspaces/safetravel-dev-ml/jobs/honest_stamp_5th95nc8zj', 'Resource__source_path': None, 'base_path': '/mnt/batch/tasks/shared/LS_root/mounts/clusters/script-launch/code/Users/thithuyduyen.pham/donut_training/donut', 'creation_context': <azure.ai.ml.entities._system_data.SystemData object at 0x7f4485a62f40>, 'serialize': <msrest.serialization.Serializer object at 0x7f44859e6070>, 'allowed_keys': {}, 'key_restriction': False, 'logger': <Logger attr_dict (WARNING)>, 'display_name': 'main_job', 'experiment_name': 'mrz', 'compute': 'gpu-cluster', 'services': {'Tracking': <azure.ai.ml.entities._job.job_service.JobService object at 0x7f4485a48cd0>, 'Studio': <azure.ai.ml.entities._job.job_service.JobService object at 0x7f4485a62e50>}, 'comment': None, 'job_inputs': {'data': {'type': 'uri_folder', 'path': 'azureml://subscriptions/cd8cfbf0-b616-4927-8541-6083db80f5d0/resourcegroups/dev-computer-vision/workspaces/safetravel-dev-ml/datastores/workspaceblobstore/paths/UI/2023-06-29_155149_UTC/passport/', 'mode': 'ro_mount'}, 'config': 'config/train_passport_terminal.yaml', 'exp_name': 'mrz'}, 'job_outputs': {'default': {'type': 'uri_folder', 'path': 'azureml://datastores/workspaceartifactstore/ExperimentRun/dcid.honest_stamp_5th95nc8zj', 'mode': 'rw_mount'}}, 'inputs': {'data': <azure.ai.ml.entities._job.pipeline._io.base.NodeInput object at 0x7f44859e6370>, 'config': <azure.ai.ml.entities._job.pipeline._io.base.NodeInput object at 0x7f44859e64f0>, 'exp_name': <azure.ai.ml.entities._job.pipeline._io.base.NodeInput object at 0x7f44859e61c0>}, 'outputs': {'default': <azure.ai.ml.entities._job.pipeline._io.base.NodeOutput object at 0x7f44859e6820>}, 'component': CommandComponent({'auto_increment_version': True, 'source': 'REMOTE.WORKSPACE.JOB', 'is_anonymous': False, 'name': 'honest_stamp_5th95nc8zj', 'description': None, 'tags': {}, 'properties': {}, 'print_as_yaml': True, 'id': None, 'Resource__source_path': None, 'base_path': PosixPath('.'), 'creation_context': <azure.ai.ml.entities._system_data.SystemData object at 0x7f4485a62f40>, 'serialize': <msrest.serialization.Serializer object at 0x7f4485a55160>, 'command': 'python train.py --dataset_name_or_paths ${{inputs.data}} --config ${{inputs.config}} --exp_name ${{inputs.exp_name}} ', 'code': '/subscriptions/cd8cfbf0-b616-4927-8541-6083db80f5d0/resourceGroups/dev-computer-vision/providers/Microsoft.MachineLearningServices/workspaces/safetravel-dev-ml/codes/72f00fbc-0bac-4e83-a862-41aa2f8fdecd/versions/1', 'environment_variables': {}, 'environment': '/subscriptions/cd8cfbf0-b616-4927-8541-6083db80f5d0/resourceGroups/dev-computer-vision/providers/Microsoft.MachineLearningServices/workspaces/safetravel-dev-ml/environments/donut/versions/15', 'distribution': <azure.ai.ml.entities._job.distribution.PyTorchDistribution object at 0x7f4485a62640>, 'resources': None, 'version': None, 'latest_version': None, 'schema': None, 'type': 'command', 'display_name': 'main_job', 'is_deterministic': True, 'inputs': {'data': {'type': 'uri_folder', 'path': 'azureml://subscriptions/cd8cfbf0-b616-4927-8541-6083db80f5d0/resourcegroups/dev-computer-vision/workspaces/safetravel-dev-ml/datastores/workspaceblobstore/paths/UI/2023-06-29_155149_UTC/passport/', 'mode': 'ro_mount'}, 'config': {'type': 'string', 'default': 'config/train_passport_terminal.yaml'}, 'exp_name': {'type': 'string', 'default': 'mrz'}}, 'outputs': {'default': {'type': 'uri_folder', 'path': 'azureml://datastores/workspaceartifactstore/ExperimentRun/dcid.honest_stamp_5th95nc8zj', 'mode': 'rw_mount'}}, 'yaml_str': None, 'other_parameter': {'status': 'Starting', 'parameters': {}}}), 'referenced_control_flow_node_instance_id': None, 'kwargs': {'services': {'Tracking': <azure.ai.ml.entities._job.job_service.JobService object at 0x7f4485a48cd0>, 'Studio': <azure.ai.ml.entities._job.job_service.JobService object at 0x7f4485a62e50>}, 'status': 'Starting', 'creation_context': <azure.ai.ml.entities._system_data.SystemData object at 0x7f4485a62f40>}, 'instance_id': 'b94f0f0d-0843-40a4-8c5f-df903346295b', 'source': 'BUILDER', 'validate_required_input_not_provided': True, 'limits': None, 'identity': None, 'distribution': <azure.ai.ml.entities._job.distribution.PyTorchDistribution object at 0x7f4485a62640>, 'environment_variables': {}, 'environment': 'donut:15', 'resources': {'instance_count': 4, 'shm_size': '2g'}, 'swept': False})"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ml_client.create_or_update(job)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "python38-azureml"
    },
    "kernelspec": {
      "display_name": "Python 3.8 - AzureML",
      "language": "python",
      "name": "python38-azureml"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      },
      "ms_spell_check": {
        "ms_spell_check_language": "en"
      }
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}