Skip to content

Commit

Permalink
Modified the NIM model deployment page, so the related resources can …
Browse files Browse the repository at this point in the history 
…be created
  • Loading branch information
@yzhao583
yzhao583 committed Aug 7, 2024
1 parent 0a97a7e commit 2491dc1
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 10 deletions.
10 changes: 10 additions & 0 deletions frontend/src/api/k8s/inferenceServices.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export const assembleInferenceService = (
isModelMesh?: boolean,
inferenceService?: InferenceServiceKind,
acceleratorState?: AcceleratorProfileState,
isStorageNeeded?: boolean,
): InferenceServiceKind => {
const {
storage,
Expand Down Expand Up @@ -155,6 +156,11 @@ export const assembleInferenceService = (
};
}

// If storage is not needed, remove storage from the inference service
if (isStorageNeeded !== undefined && !isStorageNeeded) {
delete updateInferenceService.spec.predictor.model?.storage;
}

return updateInferenceService;
};

Expand Down Expand Up @@ -226,6 +232,7 @@ export const createInferenceService = (
isModelMesh?: boolean,
acceleratorState?: AcceleratorProfileState,
dryRun = false,
isStorageNeeded?: boolean,
): Promise<InferenceServiceKind> => {
const inferenceService = assembleInferenceService(
data,
Expand All @@ -234,6 +241,7 @@ export const createInferenceService = (
isModelMesh,
undefined,
acceleratorState,
isStorageNeeded,
);
return k8sCreateResource<InferenceServiceKind>(
applyK8sAPIOptions(
Expand All @@ -253,6 +261,7 @@ export const updateInferenceService = (
isModelMesh?: boolean,
acceleratorState?: AcceleratorProfileState,
dryRun = false,
isStorageNeeded?: boolean,
): Promise<InferenceServiceKind> => {
const inferenceService = assembleInferenceService(
data,
Expand All @@ -261,6 +270,7 @@ export const updateInferenceService = (
isModelMesh,
existingData,
acceleratorState,
isStorageNeeded,
);

return k8sUpdateResource<InferenceServiceKind>(
Expand Down
14 changes: 13 additions & 1 deletion frontend/src/pages/modelServing/screens/projects/NIMServiceModal/DeployNIMServiceModal.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import {
} from '@patternfly/react-core';
import { EitherOrNone } from '@openshift/dynamic-plugin-sdk';
import {
createNIMPVC,
createNIMSecret,
getSubmitInferenceServiceResourceFn,
getSubmitServingRuntimeResourcesFn,
useCreateInferenceServiceObject,
Expand Down Expand Up @@ -37,6 +39,11 @@ import { useAccessReview } from '~/api';
import { SupportedArea, useIsAreaAvailable } from '~/concepts/areas';
import KServeAutoscalerReplicaSection from '~/pages/modelServing/screens/projects/kServeModal/KServeAutoscalerReplicaSection';

const NIM_SECRET_NAME = 'nvidia-nim-secrets';
const NIM_NGC_SECRET_NAME = 'ngc-secret';
const NIM_PVC_NAME = 'nim-pvc';
const NIM_PVC_SIZE = '50Gi';

const accessReviewResource: AccessReviewResourceAttributes = {
group: 'rbac.authorization.k8s.io',
resource: 'rolebindings',
Expand Down Expand Up @@ -107,7 +114,8 @@ const DeployNIMServiceModal: React.FC<DeployNIMServiceModalProps> = ({
}, [currentProjectName, setCreateDataInferenceService, isOpen]);

// Serving Runtime Validation
const isDisabledServingRuntime = namespace === '' || actionInProgress;
const isDisabledServingRuntime =
namespace === '' || actionInProgress || createDataServingRuntime.imageName === undefined;

const baseInputValueValid =
createDataServingRuntime.numReplicas >= 0 &&
Expand Down Expand Up @@ -178,9 +186,13 @@ const DeployNIMServiceModal: React.FC<DeployNIMServiceModalProps> = ({
acceleratorProfileState,
allowCreate,
editInfo?.secrets,
false,
);

Promise.all([
createNIMSecret(namespace, NIM_SECRET_NAME, false, false),
createNIMSecret(namespace, NIM_NGC_SECRET_NAME, true, false),
createNIMPVC(namespace, NIM_PVC_NAME, NIM_PVC_SIZE, false),
submitServingRuntimeResources({ dryRun: true }),
submitInferenceServiceResource({ dryRun: true }),
])
Expand Down
14 changes: 8 additions & 6 deletions frontend/src/pages/modelServing/screens/projects/NIMServiceModal/NIMModelListSection.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import {
CreatingServingRuntimeObject,
} from '~/pages/modelServing/screens/types';
import SimpleDropdownSelect from '~/components/SimpleDropdownSelect';
import { fetchNIMModelNames } from '~/pages/modelServing/screens/projects/utils';
import { fetchNIMModelNames, ModelInfo } from '~/pages/modelServing/screens/projects/utils';

type NIMModelListSectionProps = {
inferenceServiceData: CreatingInferenceServiceObject;
Expand All @@ -23,6 +23,7 @@ const NIMModelListSection: React.FC<NIMModelListSectionProps> = ({
isEditing,
}) => {
const [options, setOptions] = useState<{ key: string; label: string }[]>([]);
const [modelList, setModelList] = useState<ModelInfo[]>([]);

useEffect(() => {
const getModelNames = async () => {
Expand All @@ -32,27 +33,28 @@ const NIMModelListSection: React.FC<NIMModelListSectionProps> = ({
key: modelInfo.name,
label: `${modelInfo.displayName} - ${modelInfo.latestTag}`,
}));
setModelList(modelInfos);
setOptions(fetchedOptions);
}
};
getModelNames();
}, []);

const getSupportedModelFormatsInfo = (name: string) => {
const modelInfo = options.find((option) => option.key === name);
const modelInfo = modelList.find((model) => model.name === name);
if (modelInfo) {
return {
name: modelInfo.key,
version: modelInfo.label.split(' - ')[1],
name: modelInfo.name,
version: modelInfo.latestTag,
};
}
return { name: '', version: '' };
};

const getNIMImageName = (name: string) => {
const imageInfo = options.find((option) => option.key === name);
const imageInfo = modelList.find((model) => model.name === name);
if (imageInfo) {
return `nvcr.io/nim/meta/${name}:${imageInfo.label.split(' - ')[1]}`;
return `nvcr.io/${imageInfo.namespace}/${name}:${imageInfo.latestTag}`;
}
return '';
};
Expand Down
76 changes: 73 additions & 3 deletions frontend/src/pages/modelServing/screens/projects/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
ProjectKind,
SecretKind,
ServingRuntimeKind,
PersistentVolumeClaimKind,
} from '~/k8sTypes';
import {
DataConnection,
Expand Down Expand Up @@ -44,12 +45,16 @@ import {
getConfigMap,
updateInferenceService,
updateServingRuntime,
getSecret,
createPvc,
} from '~/api';
import { isDataConnectionAWS } from '~/pages/projects/screens/detail/data-connections/utils';
import { removeLeadingSlash } from '~/utilities/string';

const NAMESPACE = 'redhat-ods-applications';
const CONFIGMAP = 'nvidia-nim-images-data';
const NIM_NAMESPACE = 'redhat-ods-applications';
const NIM_CONFIGMAP_NAME = 'nvidia-nim-images-data';
const NIM_SECRET_NAME = 'nvidia-nim-access';
const NIM_NGC_SECRET_NAME = 'nvidia-nim-image-pull';

export const getServingRuntimeSizes = (config: DashboardConfigKind): ModelServingSize[] => {
let sizes = config.spec.modelServerSizes || [];
Expand Down Expand Up @@ -322,6 +327,7 @@ const createInferenceServiceAndDataConnection = (
isModelMesh?: boolean,
acceleratorProfileState?: AcceleratorProfileState,
dryRun = false,
isStorageNeeded?: boolean,
) => {
if (!existingStorage) {
return createAWSSecret(inferenceServiceData, dryRun).then((secret) =>
Expand All @@ -333,13 +339,15 @@ const createInferenceServiceAndDataConnection = (
isModelMesh,
acceleratorProfileState,
dryRun,
isStorageNeeded,
)
: createInferenceService(
inferenceServiceData,
secret.metadata.name,
isModelMesh,
acceleratorProfileState,
dryRun,
isStorageNeeded,
),
);
}
Expand All @@ -351,13 +359,15 @@ const createInferenceServiceAndDataConnection = (
isModelMesh,
acceleratorProfileState,
dryRun,
isStorageNeeded,
)
: createInferenceService(
inferenceServiceData,
undefined,
isModelMesh,
acceleratorProfileState,
dryRun,
isStorageNeeded,
);
};

Expand All @@ -369,6 +379,7 @@ export const getSubmitInferenceServiceResourceFn = (
acceleratorProfileState?: AcceleratorProfileState,
allowCreate?: boolean,
secrets?: SecretKind[],
isStorageNeeded?: boolean,
): ((opts: { dryRun?: boolean }) => Promise<void>) => {
const inferenceServiceData = {
...createData,
Expand Down Expand Up @@ -397,6 +408,7 @@ export const getSubmitInferenceServiceResourceFn = (
isModelMesh,
acceleratorProfileState,
dryRun,
isStorageNeeded,
).then((inferenceService) =>
setUpTokenAuth(
createData,
Expand Down Expand Up @@ -557,7 +569,7 @@ export interface ModelInfo {
}

export const fetchNIMModelNames = async (): Promise<ModelInfo[] | undefined> => {
const configMap = await getConfigMap(NAMESPACE, CONFIGMAP);
const configMap = await getConfigMap(NIM_NAMESPACE, NIM_CONFIGMAP_NAME);
if (configMap.data) {
const modelInfos: ModelInfo[] = Object.entries(configMap.data).map(([key, value]) => {
const modelData = JSON.parse(value); // Parse the JSON string
Expand All @@ -575,3 +587,61 @@ export const fetchNIMModelNames = async (): Promise<ModelInfo[] | undefined> =>
}
return undefined;
};

export const createNIMSecret = async (
projectName: string,
secretName: string,
isNGC: boolean,
dryRun: boolean,
): Promise<SecretKind> => {
const labels: Record<string, string> = {
[KnownLabels.DASHBOARD_RESOURCE]: 'true',
};
const data: Record<string, string> = {};
const newSecret = {
apiVersion: 'v1',
kind: 'Secret',
metadata: {
name: secretName,
namespace: projectName,
labels,
},
data,
type: isNGC ? 'kubernetes.io/dockerconfigjson' : 'Opaque',
};
const nimSecretData: SecretKind = isNGC
? await getSecret(NIM_NAMESPACE, NIM_NGC_SECRET_NAME)
: await getSecret(NIM_NAMESPACE, NIM_SECRET_NAME);

if (nimSecretData.data) {
if (!isNGC) {
data.NGC_API_KEY = nimSecretData.data.api_key;
} else {
data['.dockerconfigjson'] = nimSecretData.data['.dockerconfigjson'];
}
return createSecret(newSecret, { dryRun });
}

return Promise.reject(new Error(`Error creating NIM ${isNGC ? 'NGC' : null} secret`));
};

export const createNIMPVC = (
projectName: string,
pvcName: string,
pvcSize: string,
dryRun: boolean,
): Promise<PersistentVolumeClaimKind> =>
createPvc(
{
nameDesc: {
name: pvcName,
description: '',
},
size: pvcSize,
},
projectName,
undefined,
{
dryRun,
},
);

0 comments on commit 2491dc1

Please sign in to comment.