Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add usage snippets for Google Health AI models #1084

Open
wants to merge 12 commits into
base: main
Choose a base branch
from
Open

Add usage snippets for Google Health AI models #1084

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
67a3e17
Add CXR Foundation and Derm Foundation model libraries (Google Health…
ndebuhr Nov 26, 2024
d02c152
Implement the PR review suggestion for Derm Foundation repo name casing
ndebuhr Nov 26, 2024
a9dc7dc
Implement the PR review suggestion for CXR Foundation repo name casing
ndebuhr Nov 26, 2024
1560b82
Add model library usage snippets to the CXR Foundation and Derm Found…
ndebuhr Dec 24, 2024
1e841b1
Merge branch 'main' of github.com:huggingface/huggingface.js into goo…
ndebuhr Dec 24, 2024
4f24453
Merge branch 'main' of github.com:huggingface/huggingface.js into goo…
ndebuhr Jan 2, 2025
f31df49
Remove the CXR Foundation model library snippet
ndebuhr Jan 2, 2025
719c7af
Simplify the Derm Foundation model library usage snippet - focus on i…
ndebuhr Jan 2, 2025
c4d3af3
Add a new usage snippet for the CXR Foundation model library, which u…
ndebuhr Jan 2, 2025
4809837
Merge branch 'main' of github.com:huggingface/huggingface.js into goo…
ndebuhr Jan 7, 2025
fa66830
Trim down the Derm Foundation usage snippet comments
ndebuhr Jan 7, 2025
60a89d5
Apply suggestions from code review
Wauplin Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions packages/tasks/src/model-libraries-snippets.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ export const bm25s = (model: ModelData): string[] => [
retriever = BM25HF.load_from_hub("${model.id}")`,
];

export const cxr_foundation = (model: ModelData): string[] => [
ndebuhr marked this conversation as resolved.
Show resolved Hide resolved
`# The quick start notebook shows Hugging Face usage: https://github.com/Google-Health/cxr-foundation/blob/master/notebooks/quick_start_with_hugging_face.ipynb
# Other notebooks are also available: https://github.com/Google-Health/cxr-foundation/tree/master/notebooks`,
]

export const depth_anything_v2 = (model: ModelData): string[] => {
let encoder: string;
let features: string;
Expand Down Expand Up @@ -168,6 +173,59 @@ focallength_px = prediction["focallength_px"]`;
return [installSnippet, inferenceSnippet];
};

export const derm_foundation = (model: ModelData): string[] => [
ndebuhr marked this conversation as resolved.
Show resolved Hide resolved
`from PIL import Image
from io import BytesIO
from IPython.display import Image as IPImage, display
from huggingface_hub import from_pretrained_keras
import tensorflow as tf
import matplotlib.pyplot as plt
import requests

# Load test image from SCIN Dataset
ndebuhr marked this conversation as resolved.
Show resolved Hide resolved
# https://github.com/google-research-datasets/scin
IMAGE_URL = "https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png"
response = requests.get(IMAGE_URL, stream=True)
Wauplin marked this conversation as resolved.
Show resolved Hide resolved
# Raise an exception if the request fails
response.raise_for_status()
Wauplin marked this conversation as resolved.
Show resolved Hide resolved
# Load the image into a PIL Image object
image = Image.open(response.raw)

buf = BytesIO()
image.convert("RGB").save(buf, "PNG")
image_bytes = buf.getvalue()
Comment on lines +202 to +204
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this step mandatory for the provided example? https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png seems to already be an png file with rgb colors

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point! Below is a more condensed snippet:

from huggingface_hub import from_pretrained_keras
import tensorflow as tf, requests

# Load and format input
IMAGE_URL = "https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png"
input_tensor = tf.train.Example(
    features=tf.train.Features(
        feature={
            "image/encoded": tf.train.Feature(
                bytes_list=tf.train.BytesList(value=[requests.get(IMAGE_URL, stream=True).content])
            )
        }
    )
).SerializeToString()

# Load model and run inference
loaded_model = from_pretrained_keras("google/derm-foundation")
infer = loaded_model.signatures["serving_default"]
print(infer(inputs=tf.constant([input_tensor])))

# Format input
input_tensor = tf.train.Example(
features=tf.train.Features(
feature={
"image/encoded": tf.train.Feature(
bytes_list=tf.train.BytesList(value=[image_bytes])
)
}
)
).SerializeToString()

# Load the model directly from Hugging Face Hub
ndebuhr marked this conversation as resolved.
Show resolved Hide resolved
loaded_model = from_pretrained_keras("google/derm-foundation")

# Call inference
ndebuhr marked this conversation as resolved.
Show resolved Hide resolved
infer = loaded_model.signatures["serving_default"]
output = infer(inputs=tf.constant([input_tensor]))

# Extract the embedding vector
embedding_vector = output["embedding"].numpy().flatten()
print("Size of embedding vector:", len(embedding_vector))

# Plot the embedding vector
plt.figure(figsize=(12, 4))
plt.plot(embedding_vector)
plt.title("Embedding Vector")
plt.xlabel("Index")
plt.ylabel("Value")
plt.grid(True)
plt.show()`,
]

const diffusersDefaultPrompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k";

const diffusers_default = (model: ModelData) => [
Expand Down
2 changes: 2 additions & 0 deletions packages/tasks/src/model-libraries.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
prettyLabel: "CXR Foundation",
repoName: "cxr-foundation",
repoUrl: "https://github.com/google-health/cxr-foundation",
snippets: snippets.cxr_foundation,
filter: false,
countDownloads: `path:"precomputed_embeddings/embeddings.npz" OR path:"pax-elixr-b-text/saved_model.pb"`,
},
Expand Down Expand Up @@ -200,6 +201,7 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
prettyLabel: "Derm Foundation",
repoName: "derm-foundation",
repoUrl: "https://github.com/google-health/derm-foundation",
snippets: snippets.derm_foundation,
filter: false,
countDownloads: `path:"scin_dataset_precomputed_embeddings.npz" OR path:"saved_model.pb"`,
},
Expand Down