From d631746c5528c9e95bb6abe1150a733e4dc51853 Mon Sep 17 00:00:00 2001 From: kicha0 Date: Tue, 25 Oct 2022 23:27:24 -0700 Subject: [PATCH] update requirments, fix cliv2 and sdkv2 breaking change (#133) --- ...aidashboard-diabetes-decision-making.ipynb | 2 +- ...-diabetes-regression-model-debugging.ipynb | 2 +- ...using-classification-model-debugging.ipynb | 8 +- ...eaidashboard-housing-decision-making.ipynb | 2 +- ...rogrammer-regression-model-debugging.ipynb | 407 ++---------------- requirements.txt | 4 +- scripts/Obtain-Workspace.ps1 | 8 +- scripts/register_azureml.py | 2 +- test/rai/test_rai_pipeline.py | 6 +- 9 files changed, 52 insertions(+), 389 deletions(-) diff --git a/examples/notebooks/responsibleaidashboard-diabetes-decision-making.ipynb b/examples/notebooks/responsibleaidashboard-diabetes-decision-making.ipynb index 594a5bea..727f4efa 100644 --- a/examples/notebooks/responsibleaidashboard-diabetes-decision-making.ipynb +++ b/examples/notebooks/responsibleaidashboard-diabetes-decision-making.ipynb @@ -378,7 +378,7 @@ " f.write(yaml_contents.format(yaml_contents))\n", " \n", "train_component_definition = load_component(\n", - " path=yaml_filename\n", + " source=yaml_filename\n", ")\n", "\n", "ml_client.components.create_or_update(train_component_definition)" diff --git a/examples/notebooks/responsibleaidashboard-diabetes-regression-model-debugging.ipynb b/examples/notebooks/responsibleaidashboard-diabetes-regression-model-debugging.ipynb index 8443f459..b685bb92 100644 --- a/examples/notebooks/responsibleaidashboard-diabetes-regression-model-debugging.ipynb +++ b/examples/notebooks/responsibleaidashboard-diabetes-regression-model-debugging.ipynb @@ -378,7 +378,7 @@ " f.write(yaml_contents.format(yaml_contents))\n", " \n", "train_component_definition = load_component(\n", - " path=yaml_filename\n", + " source=yaml_filename\n", ")\n", "\n", "ml_client.components.create_or_update(train_component_definition)" diff --git a/examples/notebooks/responsibleaidashboard-housing-classification-model-debugging.ipynb b/examples/notebooks/responsibleaidashboard-housing-classification-model-debugging.ipynb index 2f832295..7c656aed 100644 --- a/examples/notebooks/responsibleaidashboard-housing-classification-model-debugging.ipynb +++ b/examples/notebooks/responsibleaidashboard-housing-classification-model-debugging.ipynb @@ -413,7 +413,7 @@ " f.write(yaml_contents.format(yaml_contents))\n", " \n", "train_component_definition = load_component(\n", - " path=yaml_filename\n", + " source=yaml_filename\n", ")\n", "\n", "ml_client.components.create_or_update(train_component_definition)" @@ -909,7 +909,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10.4 64-bit", + "display_name": "Python 3.8.13 64-bit ('responsibleai-0.18')", "language": "python", "name": "python3" }, @@ -923,11 +923,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.8.13" }, "vscode": { "interpreter": { - "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" + "hash": "8fd340b5477ca1a0b454d48a3973beff39fee032ada47a04f6f3725b469a8988" } } }, diff --git a/examples/notebooks/responsibleaidashboard-housing-decision-making.ipynb b/examples/notebooks/responsibleaidashboard-housing-decision-making.ipynb index 3835b38e..ddc6cc0e 100644 --- a/examples/notebooks/responsibleaidashboard-housing-decision-making.ipynb +++ b/examples/notebooks/responsibleaidashboard-housing-decision-making.ipynb @@ -412,7 +412,7 @@ " f.write(yaml_contents.format(yaml_contents))\n", " \n", "train_component_definition = load_component(\n", - " path=yaml_filename\n", + " source=yaml_filename\n", ")\n", "\n", "ml_client.components.create_or_update(train_component_definition)" diff --git a/examples/notebooks/responsibleaidashboard-programmer-regression-model-debugging.ipynb b/examples/notebooks/responsibleaidashboard-programmer-regression-model-debugging.ipynb index 2db270db..531b7651 100644 --- a/examples/notebooks/responsibleaidashboard-programmer-regression-model-debugging.ipynb +++ b/examples/notebooks/responsibleaidashboard-programmer-regression-model-debugging.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "53b4eeac", "metadata": {}, "outputs": [], @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "f1ad79f9", "metadata": {}, "outputs": [], @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "78053935", "metadata": {}, "outputs": [], @@ -70,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "5f875f18", "metadata": {}, "outputs": [], @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "4c7bbe58", "metadata": {}, "outputs": [], @@ -99,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "8c4eb082", "metadata": {}, "outputs": [], @@ -117,251 +117,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "1027fa92", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
scorestyleYOEIDEProgramming languagelocationNumber of github repos contributed toEmployerOSjob titleage
04.0tabs6.0XCodeJavaEurope0.0KWindowsSWE 239.1
10.0tabs5.0Visual StudioJavaEurope0.0FWindowsSWE 236.1
29.0spaces18.0Visual StudioJavaEurope0.0AWindowsSWE 237.1
38.0tabs9.0IntellijPythonEurope0.0FWindowsSWE 134.1
43.0tabs8.0pyCharmPythonEurope3.0GWindowsSWE 137.9
....................................
19959.0spaces15.0XCodeJavaEurope0.0JWindowsSWE 230.3
19962.0spaces18.0Visual StudioPHPEurope0.0FLinuxDistinguished Engineer30.1
19978.0spaces14.0Visual StudioC#Europe0.0FWindowsSWE 233.1
19982.0tabs6.0VSCodeSwiftNorth America0.0JLinuxDistinguished Engineer28.8
19990.0tabs7.0XCodeJavaAsia0.0CWindowsSWE 236.1
\n", - "

2000 rows × 11 columns

\n", - "
" - ], - "text/plain": [ - " score style YOE IDE Programming language location \\\n", - "0 4.0 tabs 6.0 XCode Java Europe \n", - "1 0.0 tabs 5.0 Visual Studio Java Europe \n", - "2 9.0 spaces 18.0 Visual Studio Java Europe \n", - "3 8.0 tabs 9.0 Intellij Python Europe \n", - "4 3.0 tabs 8.0 pyCharm Python Europe \n", - "... ... ... ... ... ... ... \n", - "1995 9.0 spaces 15.0 XCode Java Europe \n", - "1996 2.0 spaces 18.0 Visual Studio PHP Europe \n", - "1997 8.0 spaces 14.0 Visual Studio C# Europe \n", - "1998 2.0 tabs 6.0 VSCode Swift North America \n", - "1999 0.0 tabs 7.0 XCode Java Asia \n", - "\n", - " Number of github repos contributed to Employer OS \\\n", - "0 0.0 K Windows \n", - "1 0.0 F Windows \n", - "2 0.0 A Windows \n", - "3 0.0 F Windows \n", - "4 3.0 G Windows \n", - "... ... ... ... \n", - "1995 0.0 J Windows \n", - "1996 0.0 F Linux \n", - "1997 0.0 F Windows \n", - "1998 0.0 J Linux \n", - "1999 0.0 C Windows \n", - "\n", - " job title age \n", - "0 SWE 2 39.1 \n", - "1 SWE 2 36.1 \n", - "2 SWE 2 37.1 \n", - "3 SWE 1 34.1 \n", - "4 SWE 1 37.9 \n", - "... ... ... \n", - "1995 SWE 2 30.3 \n", - "1996 Distinguished Engineer 30.1 \n", - "1997 SWE 2 33.1 \n", - "1998 Distinguished Engineer 28.8 \n", - "1999 SWE 2 36.1 \n", - "\n", - "[2000 rows x 11 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import mltable\n", "\n", @@ -381,7 +140,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "5b42df3d", "metadata": {}, "outputs": [], @@ -399,18 +158,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "395435fc", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found the config file in: /home/workspace/work/RAI-vNext-Preview/config.json\n" - ] - } - ], + "outputs": [], "source": [ "from azure.ai.ml import MLClient\n", "from azure.identity import DefaultAzureCredential\n", @@ -428,33 +179,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "62eb02a2", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32mUploading train (0.02 MBs): 100%|██████████| 19510/19510 [00:00<00:00, 39405.02it/s]\n", - "\u001b[39m\n", - "\n", - "\u001b[32mUploading test (0.01 MBs): 100%|██████████| 13302/13302 [00:00<00:00, 37347.49it/s]\n", - "\u001b[39m\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "Data({'skip_validation': False, 'mltable_schema_url': None, 'referenced_uris': ['./programmers-test.parquet'], 'type': 'mltable', 'is_anonymous': False, 'auto_increment_version': False, 'name': 'Programmers_Test_MLTable', 'description': 'RAI programmers test data', 'tags': {}, 'properties': {}, 'id': '/subscriptions/fac34303-435d-4486-8c3f-7094d82a0b60/resourceGroups/RAIPM/providers/Microsoft.MachineLearningServices/workspaces/RAIPM2/data/Programmers_Test_MLTable/versions/6', 'Resource__source_path': None, 'base_path': '/home/workspace/work/RAI-vNext-Preview/examples/notebooks', 'creation_context': , 'serialize': , 'version': '6', 'latest_version': None, 'path': 'azureml://subscriptions/fac34303-435d-4486-8c3f-7094d82a0b60/resourcegroups/RAIPM/workspaces/RAIPM2/datastores/workspaceblobstore/paths/LocalUpload/d41b357e3e7193f0832161f679eba692/test/'})" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from azure.ai.ml.entities import Data\n", "from azure.ai.ml.constants import AssetTypes\n", @@ -495,7 +223,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "e78d869b", "metadata": {}, "outputs": [], @@ -515,18 +243,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "a523f144", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Writing programmer_component_src/training_script_reg.py\n" - ] - } - ], + "outputs": [], "source": [ "%%writefile programmer_component_src/training_script_reg.py\n", "\n", @@ -653,30 +373,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "3d54e43f", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32mUploading programmer_component_src (0.0 MBs): 100%|██████████| 3601/3601 [00:00<00:00, 36437.01it/s]\n", - "\u001b[39m\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "CommandComponent({'auto_increment_version': False, 'source': 'REMOTE.WORKSPACE.COMPONENT', 'is_anonymous': False, 'name': 'rai_programmers_training_component', 'description': None, 'tags': {}, 'properties': {}, 'id': '/subscriptions/fac34303-435d-4486-8c3f-7094d82a0b60/resourceGroups/RAIPM/providers/Microsoft.MachineLearningServices/workspaces/RAIPM2/components/rai_programmers_training_component/versions/6', 'Resource__source_path': None, 'base_path': './', 'creation_context': , 'serialize': , 'command': 'python training_script_reg.py --training_data ${{inputs.training_data}} --target_column_name ${{inputs.target_column_name}} --model_output ${{outputs.model_output}}', 'code': '/subscriptions/fac34303-435d-4486-8c3f-7094d82a0b60/resourceGroups/RAIPM/providers/Microsoft.MachineLearningServices/workspaces/RAIPM2/codes/cba5141d-76eb-48e9-81f4-1d64225bf742/versions/1', 'environment_variables': None, 'environment': '/subscriptions/fac34303-435d-4486-8c3f-7094d82a0b60/resourceGroups/RAIPM/providers/Microsoft.MachineLearningServices/workspaces/RAIPM2/environments/AML-RAI-Environment/versions/22', 'distribution': None, 'resources': {'instance_count': 1, 'properties': {}}, 'version': '6', 'latest_version': None, 'schema': 'http://azureml/sdk-2-0/CommandComponent.json', 'type': 'command', 'display_name': 'Programmers training component for RAI example', 'is_deterministic': True, 'inputs': {'training_data': {'type': 'path'}, 'target_column_name': {'type': 'string'}}, 'outputs': {'model_output': {'type': 'path'}}, 'yaml_str': None, 'other_parameter': {}, 'func': })" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from azure.ai.ml import load_component\n", "\n", @@ -709,7 +409,7 @@ " f.write(yaml_contents)\n", " \n", "train_component_definition = load_component(\n", - " path=yaml_filename\n", + " source=yaml_filename\n", ")\n", "\n", "ml_client.components.create_or_update(train_component_definition)" @@ -725,18 +425,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "1e40fc38", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found existing compute: rai-cluster\n" - ] - } - ], + "outputs": [], "source": [ "from azure.ai.ml.entities import AmlCompute\n", "\n", @@ -768,7 +460,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "ad76242b", "metadata": {}, "outputs": [], @@ -789,7 +481,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "cb6c6cec", "metadata": {}, "outputs": [], @@ -842,25 +534,10 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "f854eef5", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Latest status : Running\n", - "Latest status : Running\n", - "Latest status : Running\n", - "Latest status : Running\n", - "Latest status : Running\n", - "Latest status : Running\n", - "Latest status : Running\n", - "Latest status : Completed\n" - ] - } - ], + "outputs": [], "source": [ "from azure.ai.ml.entities import PipelineJob\n", "\n", @@ -891,7 +568,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "7d3e6e6e", "metadata": {}, "outputs": [], @@ -910,7 +587,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "d67b942e", "metadata": {}, "outputs": [], @@ -957,7 +634,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "id": "872e1fac", "metadata": {}, "outputs": [], @@ -1008,7 +685,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "a62105a7", "metadata": {}, "outputs": [], @@ -1118,7 +795,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "id": "e4d86ec2", "metadata": {}, "outputs": [], @@ -1161,30 +838,10 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "id": "2ca757f7", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32mUploading rai_programmer_regression_score_card_config.json\u001b[32m (< 1 MB): 100%|██████████| 492/492 [00:00<00:00, 4.72kB/s]\n", - "\u001b[39m\n", - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Latest status : Running\n", - "Latest status : Running\n", - "Latest status : Running\n", - "Latest status : Completed\n" - ] - } - ], + "outputs": [], "source": [ "insights_job = submit_and_wait(ml_client, insights_pipeline_job)" ] diff --git a/requirements.txt b/requirements.txt index 35ed5eb3..2418fce1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ azure-common -azure-ai-ml==0.1.0b6 -mltable==0.1.0b3 +azure-ai-ml~=1.0.0 +mltable==0.1.0b4 azureml_dataprep azureml_dataprep_rslex responsibleai~=0.18.0 diff --git a/scripts/Obtain-Workspace.ps1 b/scripts/Obtain-Workspace.ps1 index fb86aaa8..77146982 100644 --- a/scripts/Obtain-Workspace.ps1 +++ b/scripts/Obtain-Workspace.ps1 @@ -116,7 +116,13 @@ function Create-EpochWorkspace( function Create-ConfigJson( $workspace ) { - $parts = $workspace.storage_account.split('/') + if (Get-Member -inputobject $workspace -name "storageAccount" -Membertype Properties) { + $parts = $workspace.storageAccount.split('/') + } + else { + $parts = $workspace.storage_account.split('/') + } + $sub_id = $parts[2] $rg_name = $parts[4] Write-Host "Extracted subscription: $sub_id" diff --git a/scripts/register_azureml.py b/scripts/register_azureml.py index fdd3a4a5..1047ce3b 100644 --- a/scripts/register_azureml.py +++ b/scripts/register_azureml.py @@ -91,7 +91,7 @@ def process_directory(directory: Path, ml_client: MLClient, version: int) -> Non _logger.info("Registering component: {0}".format(c)) processed_file = c + ".processed" process_file(c, processed_file, replacements) - curr_component = load_component(path=processed_file) + curr_component = load_component(source=processed_file) ml_client.components.create_or_update(curr_component) _logger.info("Registered {0}".format(curr_component.name)) else: diff --git a/test/rai/test_rai_pipeline.py b/test/rai/test_rai_pipeline.py index dca8474b..7876e889 100644 --- a/test/rai/test_rai_pipeline.py +++ b/test/rai/test_rai_pipeline.py @@ -27,7 +27,7 @@ def test_classification_pipeline_from_yaml(self, ml_client, component_config): replacements = {"VERSION_REPLACEMENT_STRING": str(component_config["version"])} process_file(pipeline_file, pipeline_processed_file, replacements) - pipeline_job = load_job(path=pipeline_processed_file) + pipeline_job = load_job(source=pipeline_processed_file) submit_and_wait(ml_client, pipeline_job) @@ -39,7 +39,7 @@ def test_boston_pipeline_from_yaml(self, ml_client, component_config): replacements = {"VERSION_REPLACEMENT_STRING": str(component_config["version"])} process_file(pipeline_file, pipeline_processed_file, replacements) - pipeline_job = load_job(path=pipeline_processed_file) + pipeline_job = load_job(source=pipeline_processed_file) submit_and_wait(ml_client, pipeline_job) @@ -56,7 +56,7 @@ def test_cli_example_sample_yaml(self, ml_client, component_config): } process_file(pipeline_file, pipeline_processed_file, replacements) - pipeline_job = load_job(path=pipeline_processed_file) + pipeline_job = load_job(source=pipeline_processed_file) submit_and_wait(ml_client, pipeline_job)