fix: ignore empty chunk from chunk pipeline (#114)

Because:

- There is a typo in 'enterprise'.
- Sometimes the chunk pipeline returns an empty chunk.

This commit:

- Corrects the typo.
- Filters out the empty chunks.

pkg/service/mgmt.go

@@ -67,7 +67,7 @@ func (s *Service) GetNamespaceTier(ctx context.Context, ns *resource.Namespace)
 			if ok && statusError.Code() == codes.Unimplemented {
 				// Handle the case where the method is not implemented on the server
 				log.Warn("GetUserSubscriptionAdmin is not implemented. Assuming enterprise tier")
-				return TierEntriprise, nil
+				return TierEnterprise, nil
 			} else {
 				// Handle other errors
 				return "", fmt.Errorf("failed to get user subscription: %w", err)
@@ -90,7 +90,7 @@ func (s *Service) GetNamespaceTier(ctx context.Context, ns *resource.Namespace)
 			if ok && statusError.Code() == codes.Unimplemented {
 				// handle the case where the method is not implemented on the server
 				log.Warn("GetUserSubscriptionAdmin is not implemented. Assuming enterprise tier")
-				return TierEntriprise, nil
+				return TierEnterprise, nil
 			} else {
 				// handle other errors
 				return "", fmt.Errorf("failed to get organization subscription: %w", err)
@@ -102,7 +102,7 @@ func (s *Service) GetNamespaceTier(ctx context.Context, ns *resource.Namespace)
 		} else if sub.GetSubscription().Plan == mgmtPB.OrganizationSubscription_PLAN_TEAM {
 			return TierTeam, nil
 		} else if sub.GetSubscription().Plan == mgmtPB.OrganizationSubscription_PLAN_ENTERPRISE {
-			return TierEntriprise, nil
+			return TierEnterprise, nil
 		}
 		return "", fmt.Errorf("unknown organization subscription plan: %v", sub.GetSubscription().Plan)
 	default:
@@ -116,7 +116,7 @@ const (
 	TierFree       Tier = "free"
 	TierPro        Tier = "pro"
 	TierTeam       Tier = "team"
-	TierEntriprise Tier = "enterprise"
+	TierEnterprise Tier = "enterprise"
 )
 
 func (t Tier) String() string {
@@ -132,7 +132,7 @@ func (t Tier) GetPrivateCatalogLimit() int {
 	case TierTeam:
 		// unlimited
 		return math.MaxInt
-	case TierEntriprise:
+	case TierEnterprise:
 		// unlimited
 		return math.MaxInt
 
@@ -154,7 +154,7 @@ func (t Tier) GetFileStorageTotalQuota() (int, string) {
 	case TierTeam:
 		// 2GB
 		return 2 * gb, "2GB"
-	case TierEntriprise:
+	case TierEnterprise:
 		// unlimited
 		return math.MaxInt, "unlimited"
 	}

pkg/service/pipeline.go

@@ -119,7 +119,8 @@ type Chunk = struct {
 	Tokens int
 }
 
-// SplitMarkdownPipe using splitting pipeline to split markdown and consume caller's credits
+// SplitMarkdownPipe triggers the markdown splitting pipeline, processes the markdown text, and deducts credits from the caller's account.
+// It sets up the necessary metadata, triggers the pipeline, and processes the response to return the non-empty chunks.
 func (s *Service) SplitMarkdownPipe(ctx context.Context, caller uuid.UUID, requester uuid.UUID, markdown string) ([]Chunk, error) {
 	var md metadata.MD
 	if requester != uuid.Nil {
@@ -157,7 +158,8 @@ func (s *Service) SplitMarkdownPipe(ctx context.Context, caller uuid.UUID, reque
 	if err != nil {
 		return nil, err
 	}
-	// remove the empty chunk
+	// remove the empty chunk.
+	// note: this is a workaround for the pipeline bug that sometimes returns empty chunks.
 	var filteredResult []Chunk
 	for _, chunk := range result {
 		if chunk.Text != "" {
@@ -198,7 +200,8 @@ func GetChunksFromResponse(resp *pipelinePb.TriggerNamespacePipelineReleaseRespo
 	return chunks, nil
 }
 
-// SplitTextPipe using splitting pipeline to split text and consume caller's credits
+// SplitTextPipe splits the input text into chunks using the splitting pipeline and consumes the caller's credits.
+// It sets up the necessary metadata, triggers the pipeline, and processes the response to return the non-empty chunks.
 func (s *Service) SplitTextPipe(ctx context.Context, caller uuid.UUID, requester uuid.UUID, text string) ([]Chunk, error) {
 	var md metadata.MD
 	if requester != uuid.Nil {
@@ -237,7 +240,15 @@ func (s *Service) SplitTextPipe(ctx context.Context, caller uuid.UUID, requester
 	if err != nil {
 		return nil, fmt.Errorf("failed to get chunks from response: %w", err)
 	}
-	return result, nil
+	// remove the empty chunk.
+	// note: this is a workaround for the pipeline bug that sometimes returns empty chunks.
+	var filteredResult []Chunk
+	for _, chunk := range result {
+		if chunk.Text != "" {
+			filteredResult = append(filteredResult, chunk)
+		}
+	}
+	return filteredResult, nil
 }
 
 // EmbeddingTextPipe uses the embedding pipeline to convert text into vectors and consume caller's credits.