Skip to content

Commit

Permalink
add log line
Browse files Browse the repository at this point in the history
  • Loading branch information
jparkerweb committed Jun 6, 2024
1 parent 00ec017 commit 07c603f
Showing 1 changed file with 5 additions and 4 deletions.
9 changes: 5 additions & 4 deletions chunkit.js
Original file line number Diff line number Diff line change
Expand Up @@ -133,15 +133,16 @@ export async function cramit(

// Load the tokenizer
tokenizer = await AutoTokenizer.from_pretrained(onnxEmbeddingModel);

// Split the text into sentences
const sentences = sentencize(text);

// Create chunks
const chunks = createChunks(sentences, null, maxTokenSize, 0, logging);

if (logging) {
console.log('\n=============\nChunks\n=============');
console.log('\nCRAMIT');
console.log('=============\nChunks\n=============');
chunks.forEach((chunk, index) => {
console.log("\n");
console.log(`--------------`);
Expand Down

0 comments on commit 07c603f

Please sign in to comment.