Merge branch 'main' into lonnie-240920-inlineimg

ci/maketmpl/util.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 )
 
 func checkIsDir(path string) error {
@@ -34,3 +35,25 @@ func addFileToZip(z *zip.Writer, file, pathInZip string) error {
 	}
 	return nil
 }
+
+func buildZip(dir string, files []string, out string) error {
+	outFile, err := os.Create(out)
+	if err != nil {
+		return fmt.Errorf("create release zip file: %w", err)
+	}
+	defer outFile.Close()
+
+	z := zip.NewWriter(outFile)
+	for _, f := range files {
+		if err := addFileToZip(z, filepath.Join(dir, f), f); err != nil {
+			return fmt.Errorf("add file to zip: %w", err)
+		}
+	}
+	if err := z.Close(); err != nil {
+		return fmt.Errorf("close zip writer: %w", err)
+	}
+	if err := outFile.Sync(); err != nil {
+		return fmt.Errorf("flush zip file to storage: %w", err)
+	}
+	return nil
+}

ci/maketmpl/util_test.go

@@ -83,3 +83,76 @@ func TestAddFileToZip(t *testing.T) {
 		t.Errorf("content in zip file: want %q, got %q", content, got)
 	}
 }
+
+func TestBuildZip(t *testing.T) {
+	tmp := t.TempDir()
+
+	tmplDir := filepath.Join(tmp, "tmpl")
+	if err := os.Mkdir(tmplDir, 0700); err != nil {
+		t.Fatalf("create directory %q: %v", tmplDir, err)
+	}
+
+	files := []string{"a.txt", "b.txt", "sub/c.txt"}
+	for _, f := range files {
+		content := []byte(f)
+		dir := filepath.Dir(f)
+		if dir != "." {
+			if err := os.MkdirAll(
+				filepath.Join(tmplDir, dir), 0700,
+			); err != nil {
+				t.Fatalf("create directory for %q: %v", f, err)
+			}
+		}
+
+		if err := os.WriteFile(
+			filepath.Join(tmplDir, f), content, 0600,
+		); err != nil {
+			t.Fatalf("create file %q: %v", f, err)
+		}
+	}
+
+	z := filepath.Join(tmp, "out.zip")
+	if err := buildZip(tmplDir, files, z); err != nil {
+		t.Fatalf("buildZip: %v", err)
+	}
+
+	got := make(map[string][]byte)
+
+	r, err := zip.OpenReader(z)
+	if err != nil {
+		t.Fatalf("open zip file: %v", err)
+	}
+	defer r.Close()
+
+	for _, f := range r.File {
+		rc, err := f.Open()
+		if err != nil {
+			t.Fatalf("open file in zip: %v", err)
+		}
+
+		content, err := io.ReadAll(rc)
+		if err != nil {
+			t.Fatalf("read file in zip: %v", err)
+		}
+
+		if err := rc.Close(); err != nil {
+			t.Fatalf("close file in zip: %v", err)
+		}
+
+		got[f.Name] = content
+	}
+
+	if len(got) != len(files) {
+		t.Fatalf("want %d files in zip, got %d", len(files), len(got))
+	}
+
+	for _, f := range files {
+		content, ok := got[f]
+		if !ok {
+			t.Errorf("want file %q in zip, not found", f)
+		}
+		if !bytes.Equal(content, []byte(f)) {
+			t.Errorf("content of %q: want %q, got %q", f, f, content)
+		}
+	}
+}

configs/ray-summit-ai-libraries/aws.yaml

@@ -0,0 +1,12 @@
+head_node_type:
+  name: head
+  instance_type: g4dn.2xlarge
+
+worker_node_types:
+  - instance_type: g4dn.2xlarge
+    name: '1xT4:8CPU-32GB'
+    min_workers: 1
+    max_workers: 1
+
+flags:
+  allow-cross-zone-autoscaling: true

configs/ray-summit-core-masterclass/aws.yaml

@@ -0,0 +1,15 @@
+head_node_type:
+  name: head
+  instance_type: m5.4xlarge
+  resources:
+    cpu: 0
+    gpu: 0
+
+worker_node_types:
+  - instance_type: m5.4xlarge
+    name: 16CPU-64GB
+    min_workers: 1
+    max_workers: 1
+
+flags:
+  allow-cross-zone-autoscaling: true

configs/ray-summit-end-to-end-llms/aws.yaml

@@ -0,0 +1,18 @@
+head_node_type:
+  instance_type: m5.4xlarge
+  resources:
+    cpu: 0
+    gpu: 0
+
+worker_nodes:
+  - instance_type: m5.4xlarge
+    name: 16CPU-64GB
+    min_workers: 1
+    max_workers: 1
+  - instance_type: g6.4xlarge
+    name: '1xL4:16CPU-64GB'
+    min_workers: 0
+    max_workers: 2
+
+flags:
+  allow-cross-zone-autoscaling: true

configs/ray-summit-multi-modal-search/aws.yaml

@@ -0,0 +1,12 @@
+head_node_type:
+  instance_type: m5.4xlarge
+  resources:
+    cpu: 0
+    gpu: 0
+worker_node_types:
+  - instance_type: g6.4xlarge
+    name: '1xL4:16CPU-64GB'
+    min_workers: 2
+    max_workers: 2
+flags:
+  allow-cross-zone-autoscaling: true

configs/ray-summit-rag/aws.yaml

@@ -0,0 +1,15 @@
+head_node_type:
+  instance_type: m5.4xlarge
+  resources:
+    cpu: 0
+    gpu: 0
+    is_head_node: 1
+
+worker_node_types:
+  - instance_type: g6.4xlarge
+    name: '1xL4:16CPU-64GB'
+    min_workers: 2
+    max_workers: 2
+
+flags:
+  allow-cross-zone-autoscaling: true

configs/ray-summit-stable-diffusion/aws.yaml

@@ -0,0 +1,14 @@
+head_node_type:
+  instance_type: g6.4xlarge
+  resources:
+    cpu: 12
+    gpu: 1
+
+worker_nodes:
+  - instance_type: g6.4xlarge
+    name: '1xL4:16CPU-64GB'
+    min_workers: 1
+    max_workers: 1
+
+flags:
+  allow-cross-zone-autoscaling: true