Skip to content

Commit

Permalink
fixes for name of main python env
Browse files Browse the repository at this point in the history
  • Loading branch information
@ismukhin
ismukhin committed Jan 5, 2025
1 parent ed4bb4d commit a56d536
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 20 deletions.
17 changes: 9 additions & 8 deletions src/deployment/jenkins/tvm_build_envs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,26 @@


class EnvCreator:
def __init__(self, frameworks, py_version, conda):
def __init__(self, frameworks, py_version, conda, branch):
self.frameworks = frameworks.split(',')
self.py_version = py_version
self.conda_prefix = conda
self.branch = branch

def _run(self, cmd):
return subprocess.run(cmd, shell=True)

def create_envs(self):
if len(self.frameworks) != 0:
for framework in self.frameworks:
self._run(f'{self.conda_prefix}/bin/conda create -y --name tvm_{framework} --clone tvm_main')
self._run(f'{self.conda_prefix}/bin/conda create -y --name tvm_{framework} --clone tvm_main_{self.branch}')
if framework != 'mxnet':
self._run(f'{self.conda_prefix}/envs/tvm_{framework}/bin/pip3 install {framework}')
self._run(f'{self.conda_prefix}/envs/tvm_{framework}_{self.branch}/bin/pip3 install {framework}')
else:
self._run(f'{self.conda_prefix}/envs/tvm_{framework}/bin/pip3 install {framework}==1.9.1')
self._run(f'{self.conda_prefix}/envs/tvm_{framework}/bin/pip3 install gluoncv[full]')
self._run(f'{self.conda_prefix}/envs/tvm_{framework}/bin/pip3 uninstall -y numpy')
self._run(f'{self.conda_prefix}/envs/tvm_{framework}/bin/pip3 install numpy==1.23.1')
self._run(f'{self.conda_prefix}/envs/tvm_{framework}_{self.branch}/bin/pip3 install {framework}==1.9.1')
self._run(f'{self.conda_prefix}/envs/tvm_{framework}_{self.branch}/bin/pip3 install gluoncv[full]')
self._run(f'{self.conda_prefix}/envs/tvm_{framework}_{self.branch}/bin/pip3 uninstall -y numpy')
self._run(f'{self.conda_prefix}/envs/tvm_{framework}_{self.branch}/bin/pip3 install numpy==1.23.1')


def cli_arguments_parse():
Expand Down Expand Up @@ -55,7 +56,7 @@ def cli_arguments_parse():

def main():
args = cli_arguments_parse()
cr = EnvCreator(args.frameworks, args.py, args.conda)
cr = EnvCreator(args.frameworks, args.py, args.conda, args.branch)
cr.create_envs()


Expand Down
10 changes: 5 additions & 5 deletions src/deployment/jenkins/tvm_build_pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ def _run(self, cmd):
return subprocess.run(cmd, shell=True)

def build_tvm(self):
self._run(f'{self.conda}/bin/conda create -y -n tvm_main python=={self.py_version}')
self._run(f'{self.conda}/bin/conda install -n tvm_main -c conda-forge -y gcc=12.1.0')
self._run(f'{self.conda}/bin/conda install -n tvm_main -c conda-forge -y gxx_linux-64')
self._run(f'{self.conda}/envs/tvm_main/bin/pip3 install -r requirements.txt')
self._run(f'{self.conda}/bin/conda create -y -n tvm_main_{self.branch} python=={self.py_version}')
self._run(f'{self.conda}/bin/conda install -n tvm_main_{self.branch} -c conda-forge -y gcc=12.1.0')
self._run(f'{self.conda}/bin/conda install -n tvm_main_{self.branch} -c conda-forge -y gxx_linux-64')
self._run(f'{self.conda}/envs/tvm_main_{self.branch}/bin/pip3 install -r requirements.txt')
self._run(f'git clone --recursive https://github.com/apache/tvm -b {self.branch}')
self._run(f'cd tvm && mkdir -p build && cd build && cmake -DUSE_LLVM=ON ../ && make -j$(nproc --all) && cd ../python && {self.conda}/envs/tvm_main/bin/python setup.py install --user')
self._run(f'cd tvm && mkdir -p build && cd build && cmake -DUSE_LLVM=ON ../ && make -j$(nproc --all) && cd ../python && {self.conda}/envs/tvm_main_{self.branch}/bin/python setup.py install --user')


def cli_arguments_parse():
Expand Down
13 changes: 10 additions & 3 deletions src/deployment/jenkins/tvm_compile_pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,15 @@ def parse(self):


class TVMCompilerProcess:
def __init__(self, models_dir, conda, output_dir, vm):
def __init__(self, models_dir, conda, output_dir, vm, branch):
self.converter = Path(__file__).parents[2]
self.converter = self.converter.joinpath('model_converters')
self.converter = self.converter.joinpath('tvm_converter')
self.converter = str(self.converter.joinpath('tvm_compiler.py'))
self.conda = conda
self.vm = vm
self.models_dir = models_dir.absolute().as_posix()
self.branch = branch
if output_dir is not None:
self.output_dir = output_dir
else:
Expand All @@ -51,7 +52,7 @@ def _add_option(self, name_of_arg):
self._command_line += f' {name_of_arg}'

def create_command_line(self, model_name, target, batch, opt_level):
self._command_line = (f'{self.conda}/envs/tvm_main/bin/python3 ' + f'{self.converter}')
self._command_line = (f'{self.conda}/envs/tvm_main_{self.branch}/bin/python3 ' + f'{self.converter}')
self._add_argument('--mod', f'{self.models_dir}/{model_name}/batch_{batch}/{model_name}.json')
self._add_argument('--params', f'{self.models_dir}/{model_name}/batch_{batch}/{model_name}.params')
self._add_argument('-t', f'"{target}"')
Expand Down Expand Up @@ -106,6 +107,11 @@ def cli_arguments_parse():
help='Flag to use VirtualMachine API',
action='store_true',
dest='vm')
parser.add_argument('-b', '--branch',
help='Branch to build tvm.',
dest='branch',
required=True,
type=str)

return parser.parse_args()

Expand All @@ -114,7 +120,8 @@ def main():
args = cli_arguments_parse()
parser = TXTParser(args.models_info)
models = parser.parse()
proc = TVMCompilerProcess(args.models_dir, args.conda, args.output_dir, args.vm)
proc = TVMCompilerProcess(args.models_dir, args.conda,
args.output_dir, args.vm, args.branch)
for (model_name, _, _,
_, _, batches, _) in models:
for batch in batches:
Expand Down
13 changes: 9 additions & 4 deletions src/deployment/jenkins/tvm_convert_pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,14 @@ def parse(self):


class TVMConverterProcess:
def __init__(self, models_dir, conda):
def __init__(self, models_dir, conda, branch):
self.converter = Path(__file__).parents[2]
self.conda = conda
self.converter = self.converter.joinpath('model_converters')
self.converter = self.converter.joinpath('tvm_converter')
self.converter = str(self.converter.joinpath('tvm_converter.py'))
self.models_dir = models_dir.absolute().as_posix()
self.branch = branch
self._command_line = f''

def _add_argument(self, name_of_arg, value_of_arg):
Expand All @@ -47,7 +48,7 @@ def _add_option(self, name_of_arg):

def create_command_line(self, model_name, model, weights,
framework, input_shape, batch, input_name):
self._command_line = (f'{self.conda}/envs/tvm_{framework}/bin/python3 ' + f'{self.converter}')
self._command_line = (f'{self.conda}/envs/tvm_{framework}_{self.branch}/bin/python3 ' + f'{self.converter}')
self._add_argument('-mn', model_name)
if model != '':
self._add_argument('-m', f'{self.models_dir}/{model}')
Expand All @@ -65,7 +66,6 @@ def create_command_line(self, model_name, model, weights,
def execute(self):
log.info(f'Starting process: {self._command_line}\n')
proc = subprocess.run(self._command_line, shell=True)
#log.info(f'Subprocess logs: \n\n{proc.stdout.decode()}')
self.exit_code = proc.returncode
self._command_line = ''

Expand All @@ -89,6 +89,11 @@ def cli_arguments_parse():
dest='conda',
required=True,
type=str)
parser.add_argument('-b', '--branch',
help='Branch to build tvm.',
dest='branch',
required=True,
type=str)

return parser.parse_args()

Expand All @@ -97,7 +102,7 @@ def main():
args = cli_arguments_parse()
parser = TXTParser(args.models_info)
models = parser.parse()
proc = TVMConverterProcess(args.models_dir, args.conda)
proc = TVMConverterProcess(args.models_dir, args.conda, args.branch)
for (model_name, model, weights,
framework, input_shape, batches, input_name) in models:
for batch in batches:
Expand Down

0 comments on commit a56d536

Please sign in to comment.