Skip to content

Commit

Permalink
Metadata commands & single file download (#115)
Browse files Browse the repository at this point in the history 
* Initial changes for cmf commands conversion to function

* cmf_commands

* updating artifact pull commands

* Updated

* Initial commint for single file download

* code for single artifact download

* removed unnecessary comments

* removing unneccesary comments

* Fixed a small error

* Resolved an error in minio artifact pull

* Resolved a small issue creating problem in artifact pull for ssh remote

* cmf init commands created outside cmf.py

* Update cmf_client.md

Updated the cmf_client

* adding cmf init as helper functions

* changes in test_push and pull

* adding jupyter notebook feature

* modifying jupyter notebook

* improving naming, commenting,folder structure

* Update Readme.md

---------

Co-authored-by: Abhinav Chobey <chobey@abhinav-cmf-hpe.labs.hpecorp.net>
Co-authored-by: abhinavchobey <111754147+abhinavchobey@users.noreply.github.com>
  • Loading branch information
@varkha-d-sharma @abhinavchobey
3 people authored Oct 5, 2023
1 parent 3cde65b commit abdeac6
Show file tree
Hide file tree
Showing 11 changed files with 1,513 additions and 106 deletions.
169 changes: 169 additions & 0 deletions cmflib/cmf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,19 @@
link_execution_to_input_artifact,
)
from cmflib.utils.cmf_config import CmfConfig
from cmflib.cmf_commands_wrapper import (
_metadata_push,
_metadata_pull,
_artifact_pull,
_artifact_push,
_artifact_pull_single,
_cmf_cmd_init,
_init_local,
_init_minioS3,
_init_amazonS3,
_init_sshremote

)

class Cmf:
"""This class provides methods to log metadata for distributed AI pipelines.
Expand Down Expand Up @@ -1545,3 +1557,160 @@ def commit_existing(self, uri: str, custom_properties: t.Optional[t.Dict] = None
# first, middle, last = str(index).split("/")
# print(last)
# os.symlink(str(index), slicedir + "/ " + last)

def metadata_push(pipeline_name,filename,execution_id: str = ""):
"""Pushes mlmd file to cmf-server """
# Required arguments: pipeline_name, filename (mlmd file path)
#Optional arguments: Execution_ID
output = _metadata_push(pipeline_name,filename, execution_id)
return output

def metadata_pull(pipeline_name,filename ="./mlmd", execution_id: str = ""):
"""Pulls mlmd file from cmf-server"""
# Required arguments: pipeline_name, filename(file path to store mlmd file)
#Optional arguments: Execution_ID
output = _metadata_pull(pipeline_name,filename, execution_id)
return output

def artifact_pull(pipeline_name,filename="./mlmd"):
"""Pulls artifacts from initialized repository """
# Required arguments: Pipeline_name
# Optional arguments: filename( path to store artifacts)
output = _artifact_pull(pipeline_name,filename)
return output

def artifact_pull_single(pipeline_name,filename,artifact_name):
"""Pulls artifacts from initialized repository """
# Required arguments: Pipeline_name
# Optional arguments: filename( path to store artifacts)
output = _artifact_pull_single(pipeline_name,filename,artifact_name)
return output

def artifact_push():
"""Push artifacts to initialized repository"""
output = _artifact_push()
return output

def cmf_init_show():
output=_cmf_cmd_init()
return output

def cmf_init(type: str="",
path: str="",
git_remote_url: str="",
cmf_server_url: str = "",
neo4j_user: str = "",
neo4j_password: str = "",
neo4j_uri: str = "",
url: str="",
endpoint_url: str="",
access_key_id: str="",
secret_key: str="",
user: str="",
password: str="",
port: int=0
):
if type=="":
return print("Error: Type is not provided")
if type not in ["local","minioS3","amazonS3","sshremote"]:
return print("Error: Type value is undefined"+ " "+type+".Expected: "+",".join(["local","minioS3","amazonS3","sshremote"]))

if neo4j_user!="" and neo4j_password != "" and neo4j_uri != "":
pass
elif neo4j_user == "" and neo4j_password == "" and neo4j_uri == "":
pass
else:
return print("Error: Enter all neo4j parameters.")

args={'path':path,
'git_remote_url':git_remote_url,
'url':url,
'endpoint_url':endpoint_url,
'access_key_id':access_key_id,
'secret_key':secret_key,
'user':user,
'password':password,
}

status_args=non_related_args(type,args)

if type=="local" and path!= "" and git_remote_url!= "" :
"""Initialize local repository"""
output = _init_local(
path, git_remote_url, cmf_server_url, neo4j_user, neo4j_password, neo4j_uri
)
if status_args != []:
print("There are non-related arguments: "+",".join(status_args)+".Please remove them.")
return output

elif type=="minioS3" and url!= "" and endpoint_url!= "" and access_key_id!= "" and secret_key!= "" and git_remote_url!= "":
"""Initialize minioS3 repository"""
output = _init_minioS3(
url,
endpoint_url,
access_key_id,
secret_key,
git_remote_url,
cmf_server_url,
neo4j_user,
neo4j_password,
neo4j_uri,
)
if status_args != []:
print("There are non-related arguments: "+",".join(status_args)+".Please remove them.")
return output

elif type=="amazonS3" and url!= "" and access_key_id!= "" and secret_key!= "" and git_remote_url!= "":
"""Initialize amazonS3 repository"""
output = _init_amazonS3(
url,
access_key_id,
secret_key,
git_remote_url,
cmf_server_url,
neo4j_user,
neo4j_password,
neo4j_uri,
)
if status_args != []:
print("There are non-related arguments: "+",".join(status_args)+".Please remove them.")

return output

elif type=="sshremote" and path !="" and user!="" and port!=0 and password!="" and git_remote_url!="":
"""Initialize sshremote repository"""
output = _init_sshremote(
path,
user,
port,
password,
git_remote_url,
cmf_server_url,
neo4j_user,
neo4j_password,
neo4j_uri,
)
if status_args != []:
print("There are non-related arguments: "+",".join(status_args)+".Please remove them.")

return output

else:
print("Error: Enter all arguments")


def non_related_args(type:str,args:dict):
available_args=[i for i,j in args.items() if j!=""]
local=["path","git_remote_url"]
minioS3=["url","endpoint_url","access_key_id","secret_key","git_remote_url"]
amazonS3=["url","access_key_id","secret_key","git_remote_url"]
sshremote=["path","user","port","password","git_remote_url"]

dict_repository_args={"local":local,"minioS3":minioS3,"amazonS3":amazonS3,"sshremote":sshremote}

for repo,arg in dict_repository_args.items():
if repo ==type:
non_related_args=list(set(available_args)-set(dict_repository_args[repo]))
return non_related_args


216 changes: 216 additions & 0 deletions cmflib/cmf_commands_wrapper.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,216 @@
from cmflib import cli


def _metadata_push(pipeline_name, file_name, execution_id):
cli_args = cli.parse_args(
[
"metadata",
"push",
"-p",
pipeline_name,
"-f",
file_name,
"-e",
execution_id
]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg

def _metadata_pull(pipeline_name, file_name, execution_id):
cli_args = cli.parse_args(
[
"metadata",
"pull",
"-p",
pipeline_name,
"-f",
file_name,
"-e",
execution_id
]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg

def _artifact_push():
cli_args = cli.parse_args(
[
"artifact",
"push",

]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg


def _artifact_pull(pipeline_name, file_name):

cli_args = cli.parse_args(
[
"artifact",
"pull",
"-p",
pipeline_name,
"-f",
file_name,

]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg

def _artifact_pull_single(pipeline_name, file_name,artifact_name):
print("inside_cmf_cmd_wrapper")
cli_args = cli.parse_args(
[
"artifact",
"pull",
"-p",
pipeline_name,
"-f",
file_name,
"-a",
artifact_name,

]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg


def _cmf_cmd_init():
cli_args = cli.parse_args(
[
"init",
"show"
]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg

def _init_local(path,git_remote_url,cmf_server_url,neo4j_user,neo4j_password,neo4j_uri):
cli_args = cli.parse_args(
[
"init",
"local",
"--path",
path,
"--git-remote-url",
git_remote_url,
"--cmf-server-url",
cmf_server_url,
"--neo4j-user",
neo4j_user,
"--neo4j-password",
neo4j_password,
"--neo4j-uri",
neo4j_uri
]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg


def _init_minioS3(url,endpoint_url,access_key_id,secret_key,git_remote_url,cmf_server_url,neo4j_user,neo4j_password,neo4j_uri):
cli_args = cli.parse_args(
[
"init",
"minioS3",
"--url",
url ,
"--endpoint-url",
endpoint_url,
"--access-key-id",
access_key_id,
"--secret-key",
secret_key,
"--git-remote-url",
git_remote_url,
"--cmf-server-url",
cmf_server_url,
"--neo4j-user",
neo4j_user,
"--neo4j-password",
neo4j_password,
"--neo4j-uri",
neo4j_uri
]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg

def _init_amazonS3(url,access_key_id,secret_key,git_remote_url,cmf_server_url,neo4j_user,neo4j_password,neo4j_uri):
cli_args = cli.parse_args(
[
"init",
"amazonS3",
"--url",
url ,
"--access-key-id",
access_key_id,
"--secret-key",
secret_key,
"--git-remote-url",
git_remote_url,
"--cmf-server-url",
cmf_server_url,
"--neo4j-user",
neo4j_user,
"--neo4j-password",
neo4j_password,
"--neo4j-uri",
neo4j_uri
]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg

def _init_sshremote(path,user,port,password,git_remote_url,cmf_server_url,neo4j_user,neo4j_password,neo4j_uri):
cli_args = cli.parse_args(
[
"init",
"sshremote",
"--path",
path,
"--user",
user ,
"--port",
port,
"--password",
password,
"--git-remote-url",
git_remote_url,
"--cmf-server-url",
cmf_server_url,
"--neo4j-user",
neo4j_user,
"--neo4j-password",
neo4j_password,
"--neo4j-uri",
neo4j_uri
]
)
cmd = cli_args.func(cli_args)
msg = cmd.do_run()
print(msg)
return msg
Loading

0 comments on commit abdeac6

Please sign in to comment.