Skip to content

Commit

Permalink
feat: add config recursive chunker tests
Browse files Browse the repository at this point in the history
  • Loading branch information
PabloSanchi committed Aug 16, 2024
1 parent 5f67eff commit a548f82
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 4 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package jchunk.chunker.recursive;

import org.springframework.util.Assert;

import java.util.List;

/**
Expand Down Expand Up @@ -56,11 +58,13 @@ public static class Builder {
private List<String> separators = List.of("\n\n", "\n", " ", "");

public Builder chunkSize(Integer chunkSize) {
Assert.isTrue(chunkSize > 0, "Chunk size must be greater than 0");
this.chunkSize = chunkSize;
return this;
}

public Builder chunkOverlap(Integer chunkOverlap) {
Assert.isTrue(chunkOverlap >= 0, "Chunk overlap must be greater than or equal to 0");
this.chunkOverlap = chunkOverlap;
return this;
}
Expand All @@ -71,6 +75,7 @@ public Builder separators(List<String> separators) {
}

public Config build() {
Assert.isTrue(chunkSize > chunkOverlap, "Chunk size must be greater than chunk overlap");
return new Config(chunkSize, chunkOverlap, separators);
}

Expand Down
32 changes: 32 additions & 0 deletions jchunk-recursive-character/src/test/java/ConfigTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import jchunk.chunker.recursive.Config;
import org.junit.jupiter.api.Test;

import java.util.List;

import static org.assertj.core.api.Assertions.assertThat;

class ConfigTest {

@Test
void testDefaultConfig() {
Config config = Config.defaultConfig();

assertThat(config.getChunkSize()).isEqualTo(100);
assertThat(config.getChunkOverlap()).isEqualTo(20);
assertThat(config.getSeparators()).containsExactly("\n\n", "\n", " ", "");
}

@Test
void testCustomConfig() {
Config config = Config.builder()
.chunkSize(50)
.chunkOverlap(10)
.separators(List.of("-", "!", "?"))
.build();

assertThat(config.getChunkSize()).isEqualTo(50);
assertThat(config.getChunkOverlap()).isEqualTo(10);
assertThat(config.getSeparators()).containsExactly("-", "!", "?");
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import static org.assertj.core.api.Assertions.assertThat;

public class RecursiveCharacterChunkerTest {
class RecursiveCharacterChunkerTest {

RecursiveCharacterChunker chunker = new RecursiveCharacterChunker();

Expand All @@ -19,7 +19,7 @@ public class RecursiveCharacterChunkerTest {
""";

@Test
public void testSplit() {
void testSplit() {
List<Chunk> expectedChunks = List.of(
new Chunk(0, "One of the most important things I didn't understand about the"),
new Chunk(1, "world when I was a child is the degree to which the returns for"),
Expand All @@ -39,8 +39,7 @@ public void testSplit() {

List<Chunk> chunks = chunker.split(content);

assertThat(chunks).isNotNull();
assertThat(chunks).hasSize(expectedChunks.size());
assertThat(chunks).isNotNull().hasSize(expectedChunks.size());

for (int i = 0; i < chunks.size(); i++) {
assertThat(chunks.get(i).id()).isEqualTo(expectedChunks.get(i).id());
Expand Down

0 comments on commit a548f82

Please sign in to comment.