Merge branch 'main' of https://github.com/anyscale/templates into sum…

…anthrh/update-e2e-llm-template

Signed-off-by: SumanthRH <sumanthrh@anyscale.com>

.github/workflows/premerge.yaml

@@ -43,7 +43,7 @@ jobs:
       run: bash -ec "(cd ci; go fmt ./... ; git diff --exit-code)"
 
     - name: Test
-      run: bash -ec "(cd ci; go test -v ./...)"
+      run: CI=1 bash -ec "(cd ci; go test -v ./...)"
 
     - name: Build templates
       run: bash build.sh

BUILD.yaml

@@ -0,0 +1,26 @@
+# UNDER CONSTRUCTION: we in the process of adding template definitions
+# into this file. This file does not contain all the templates yet.
+
+# job-intro template is to be removed and replaced by
+# https://github.com/anyscale/first-job
+- name: job-intro
+  emoji: 🔰
+  title: Intro to Jobs
+  description: Introduction on how to use Anyscale Jobs
+  dir: templates/intro-jobs
+  cluster_env:
+    build_id: anyscaleray2340-py311
+  compute_config:
+    GCP: configs/basic-single-node/gce.yaml
+    AWS: configs/basic-single-node/aws.yaml
+
+- name: workspace-intro
+  emoji: 🔰
+  title: Intro to Workspaces
+  description: Introduction on how to use Anyscale Workspaces
+  dir: templates/intro-workspaces
+  cluster_env:
+    build_id: anyscaleray2340-py311
+  compute_config:
+    GCP: configs/basic-single-node/gce.yaml
+    AWS: configs/basic-single-node/aws.yaml

ci/auto-generate-readme.sh

@@ -1,45 +1,63 @@
 #!/bin/bash
+
+set -euo pipefail
+
 echo "Auto-generating README files..."
 
-# Search for notebook files named README.ipynb in the ../templates directory
-notebook_files=$(find ../templates -name "README.ipynb")
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+if [[ "$(pwd)" != "${REPO_ROOT}" ]]; then
+	echo "Must run this script at repo's root directory".
+	exit 1
+fi
 
-# Loop through each notebook file
-for notebook_file in $notebook_files; do
-    # Exclude specific notebooks from conversion
-    if [ "$notebook_file" != "../templates/templates/getting-started/README.ipynb" ] && [ "$notebook_file" != "../templates/templates/e2e-llm-workflows/README.ipynb" ] && ! grep -q "Time to complete" $notebook_file; then
-        echo "**********"
-        echo "LINT ERROR: $notebook_file must include 'Time to complete' statement, failing."
-        echo "**********"
-        exit 1
-    fi
-    if [ "$notebook_file" != "../templates/templates/e2e-llm-workflows/README.ipynb" ]; then
-        # Convert notebook file to README.md using nbconvert
-        jupyter nbconvert --to markdown "$notebook_file" --output-dir "$(dirname "$notebook_file")"
-    else
-        echo "Skipping README generation for $notebook_file"
-    fi
-done
+# Search for notebook files named README.ipynb in the ../templates directory
+TEMPLATES_DIRS=($(find "templates" -mindepth 1 -maxdepth 1 -type d))
 
 # Define the repo prefix
 REPO_PREFIX="https://raw.githubusercontent.com/anyscale/templates/main"
 
-# Search for README.md in the ../templates directory
-readme_files=$(find ../templates -name "README.md")
-
-# Loop through each readme files
-for readme_file in $readme_files; do
-    # Extract the path of the directory containing the README file, relative to the repository root
-    readme_dir=$(dirname "$readme_file" | sed "s|\.\./templates/||")
-
-    # Check the operating system
-    if [[ "$OSTYPE" == "darwin"* ]]; then
-        # macOS system
-        sed -i '' "s|<img src=\"\([^\"http://][^\":/][^\"].*\)\"|<img src=\"${REPO_PREFIX}/${readme_dir}/\1\"|g" "$readme_file"
-        sed -i '' "s|!\[.*\](\(assets/.*\))|<img src=\"${REPO_PREFIX}/${readme_dir}/\1\"/>|g" "$readme_file"
-    else
-        # Assuming Linux
-        sed -i "s|<img src=\"\([^\"http://][^\":/][^\"].*\)\"|<img src=\"${REPO_PREFIX}/${readme_dir}/\1\"|g" "$readme_file"
-        sed -i "s|!\[.*\](\(assets/.*\))|<img src=\"${REPO_PREFIX}/${readme_dir}/\1\"/>|g" "$readme_file"
+# Loop through each notebook file
+for TMPL in "${TEMPLATES_DIRS[@]}"; do
+	echo "===== Processing ${TMPL}"
+
+	if [[ ! -f "${TMPL}/README.ipynb" ]]; then
+		echo "README.ipynb file not found; skipping notebook conversion and checking."
+	else
+        # Exclude specific notebooks from conversion
+        TMPL_NAME="$(basename "${TMPL}")"
+        NOTEBOOK_FILE="${TMPL}/README.ipynb"
+
+        if [[ "${TMPL_NAME}" == "getting-started" || "${TMPL_NAME}" == "e2e-llm-workflows" || "${TMPL_NAME}" == "ray-summit-multi-modal-search" ]]; then
+            echo "Skip 'Time to complete' checking for ${TMPL_NAME}"
+        elif ! grep -q "Time to complete" "${NOTEBOOK_FILE}" ; then
+            echo "**********"
+            echo "LINT ERROR: ${NOTEBOOK_FILE} must include 'Time to complete' statement, failing."
+            echo "**********"
+            exit 1
+        fi
+
+        if [[ "${TMPL_NAME}" != "e2e-llm-workflows" ]]; then
+            # Convert notebook file to README.md using nbconvert
+            jupyter nbconvert --to markdown "${NOTEBOOK_FILE}" --output-dir "${TMPL}"
+        else
+            echo "Skipping README generation for ${NOTEBOOK_FILE}"
+        fi
+    fi
+
+	# Post-processing on README markdown files
+	README_FILE="${TMPL}/README.md"
+    if [[ ! -f "${TMPL}/README.md" ]]; then
+		echo "README.md file not found; skipping markdown processing."
+	else
+        # Check the operating system
+        if [[ "$OSTYPE" == "darwin"* ]]; then
+            # macOS system
+            sed -i '' "s|<img src=\"\([^\"http://][^\":/][^\"].*\)\"|<img src=\"${REPO_PREFIX}/${TMPL}/\1\"|g" "$README_FILE"
+            sed -i '' "s|!\[.*\](\(assets/.*\))|<img src=\"${REPO_PREFIX}/${TMPL}/\1\"/>|g" "$README_FILE"
+        else
+            # Assuming Linux
+            sed -i "s|<img src=\"\([^\"http://][^\":/][^\"].*\)\"|<img src=\"${REPO_PREFIX}/${TMPL}/\1\"|g" "$README_FILE"
+            sed -i "s|!\[.*\](\(assets/.*\))|<img src=\"${REPO_PREFIX}/${TMPL}/\1\"/>|g" "$README_FILE"
+        fi
     fi
 done

ci/go.mod

@@ -2,4 +2,7 @@ module github.com/anyscale/templates/ci
 
 go 1.23.1
 
-require gopkg.in/yaml.v2 v2.4.0
+require (
+	golang.org/x/net v0.29.0
+	gopkg.in/yaml.v2 v2.4.0
+)

ci/go.sum

@@ -1,3 +1,5 @@
+golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

ci/maketmpl/README.md

@@ -0,0 +1 @@
+Go language library for building and releasing templates for Anyscale.

ci/maketmpl/build_all.go

@@ -0,0 +1,26 @@
+package maketmpl
+
+import (
+	"fmt"
+	"log"
+	"path/filepath"
+)
+
+// BuildAll builds all the templates defined in the YAML file.
+func BuildAll(yamlFile, baseDir, outputDir string) error {
+	tmpls, err := readTemplates(yamlFile)
+	if err != nil {
+		return fmt.Errorf("read templates: %w", err)
+	}
+
+	for _, t := range tmpls {
+		log.Println("Building template:", t.Name)
+		b := newBuilder(t, baseDir)
+		tmplOutputDir := filepath.Join(outputDir, t.Name)
+		if err := b.build(tmplOutputDir); err != nil {
+			return fmt.Errorf("build template %q: %w", t.Name, err)
+		}
+	}
+
+	return nil
+}

ci/maketmpl/build_all_test.go

@@ -0,0 +1,38 @@
+package maketmpl
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestBuildAll(t *testing.T) {
+	tmp := t.TempDir()
+
+	if err := BuildAll("testdata/BUILD.yaml", "testdata", tmp); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tmpl := range []string{"reefy-ray", "fishy-ray"} {
+		// Check that critical files are generated.
+		for _, f := range []string{
+			buildDotZip,
+			rayAppDotJSON,
+			readmeDocMD,
+			readmeGitHubMD,
+		} {
+			stat, err := os.Stat(filepath.Join(tmp, tmpl, f))
+			if err != nil {
+				t.Errorf("os.Stat(%q): %v", f, err)
+				continue
+			}
+
+			if stat.IsDir() {
+				t.Errorf("%q is a directory", f)
+			}
+			if stat.Size() == 0 {
+				t.Errorf("%q is empty", f)
+			}
+		}
+	}
+}