Skip to content

Commit

Permalink
Fix BWC test generation after mondernizing LineDocFile (apache#13021)
Browse files Browse the repository at this point in the history
The changes on apache#12929 broke the generation code for BWC indices
since they are expecting vertain fields created by LineDocFile.
Yet, this change adds some sanity checks that run with unittest to ensure
the BWC generation is at least readable with the current version.

Relates to apache#12929
  • Loading branch information
s1monw committed Jan 17, 2024
1 parent 28b9f2e commit 564219a
Showing 1 changed file with 60 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -205,11 +205,20 @@ private Path getIndexDir() {
}

public void testCreateMoreTermsIndex() throws Exception {

Path indexDir = getIndexDir().resolve("moreterms");
Files.deleteIfExists(indexDir);
Directory dir = newFSDirectory(indexDir);
try (Directory dir = newFSDirectory(indexDir)) {
createMoreTermsIndex(dir);
}
}

public void testCreateMoreTermsIndexInternal() throws Exception {
try (Directory dir = newDirectory()) {
createMoreTermsIndex(dir);
}
}

private void createMoreTermsIndex(Directory dir) throws Exception {
LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
mp.setNoCFSRatio(1.0);
mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
Expand All @@ -220,22 +229,48 @@ public void testCreateMoreTermsIndex() throws Exception {
new IndexWriterConfig(analyzer).setMergePolicy(mp).setUseCompoundFile(false);
IndexWriter writer = new IndexWriter(dir, conf);
LineFileDocs docs = new LineFileDocs(new Random(0));
Field docIdDV = null;
Field titleDVField = null;
for (int i = 0; i < 50; i++) {
writer.addDocument(docs.nextDoc());
Document doc = docs.nextDoc();
if (docIdDV == null) {
docIdDV = new NumericDocValuesField("docid_intDV", 0);
doc.add(docIdDV);
}
docIdDV.setLongValue(doc.getField("docid_int").numericValue().longValue());
if (titleDVField == null) {
titleDVField = new SortedDocValuesField("titleDV", new BytesRef());
doc.add(titleDVField);
}
titleDVField.setBytesValue(new BytesRef(doc.getField("title").stringValue()));
writer.addDocument(doc);
}
docs.close();
writer.close();
dir.close();
try (DirectoryReader reader = DirectoryReader.open(dir)) {
searchExampleIndex(reader); // make sure we can search it
}
}

// gradlew test -Ptestmethod=testCreateSortedIndex -Ptests.codec=default
// -Ptests.useSecurityManager=false -Ptests.bwcdir=/tmp/sorted --tests TestBackwardsCompatibility
public void testCreateSortedIndex() throws Exception {

Path indexDir = getIndexDir().resolve("sorted");
Files.deleteIfExists(indexDir);
Directory dir = newFSDirectory(indexDir);
try (Directory dir = newFSDirectory(indexDir)) {
createSortedIndex(dir);
}
}

public void testCreateSortedIndexInternal() throws Exception {
// this runs without the -Ptests.bwcdir=/tmp/sorted to make sure we can actually index and
// search the created index
try (Directory dir = newDirectory()) {
createSortedIndex(dir);
}
}

public void createSortedIndex(Directory dir) throws Exception {
LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
mp.setNoCFSRatio(1.0);
mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
Expand All @@ -248,11 +283,14 @@ public void testCreateSortedIndex() throws Exception {
conf.setUseCompoundFile(false);
conf.setIndexSort(new Sort(new SortField("dateDV", SortField.Type.LONG, true)));
IndexWriter writer = new IndexWriter(dir, conf);
LineFileDocs docs = new LineFileDocs(random());
LineFileDocs docs = new LineFileDocs(new Random(0));
SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
parser.setTimeZone(TimeZone.getTimeZone("UTC"));
ParsePosition position = new ParsePosition(0);
Field docIdDV = null;
Field dateDVField = null;
Field titleDVField = null;

for (int i = 0; i < 50; i++) {
Document doc = docs.nextDoc();
String dateString = doc.get("date");
Expand All @@ -270,14 +308,28 @@ public void testCreateSortedIndex() throws Exception {
doc.add(dateDVField);
}
dateDVField.setLongValue(date.getTime());
if (docIdDV == null) {
docIdDV = new NumericDocValuesField("docid_intDV", 0);
doc.add(docIdDV);
}

docIdDV.setLongValue(doc.getField("docid_int").numericValue().longValue());
if (titleDVField == null) {
titleDVField = new SortedDocValuesField("titleDV", new BytesRef());
doc.add(titleDVField);
}
titleDVField.setBytesValue(new BytesRef(doc.getField("title").stringValue()));
if (i == 250) {
writer.commit();
}
writer.addDocument(doc);
}
writer.forceMerge(1);
writer.close();
dir.close();

try (DirectoryReader reader = DirectoryReader.open(dir)) {
searchExampleIndex(reader); // make sure we can search it
}
}

private void updateNumeric(IndexWriter writer, String id, String f, String cf, long value)
Expand Down

0 comments on commit 564219a

Please sign in to comment.