Fix addBackcompatIndexes.py to properly generate missing versions (ap…

…ache#13095)

In apache#13046 several changes broke the addBackcompatIndexes.py script
to properly add and test the unreleased version. This updates the
script to again properly add the new version.

Closes apache#13094

Co-authored-by: Dawid Weiss <dawid.weiss@carrotsearch.com>

dev-tools/scripts/addBackcompatIndexes.py

@@ -96,16 +96,15 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
   scriptutil.run('rm -rf %s' % bc_index_dir)
   print('done')
 
-def update_backcompat_tests(types, index_version, current_version):
-  print('  adding new indexes %s to backcompat tests...' % types, end='', flush=True)
+def update_backcompat_tests(index_version, current_version):
+  print('  adding new indexes to backcompat tests...', end='', flush=True)
   module = 'lucene/backward-codecs'
-  filename = '%s/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java' % module
+
+  filename = None
   if not current_version.is_back_compat_with(index_version):
-    matcher = re.compile(r'final String\[\] unsupportedNames = {|};')
-  elif 'sorted' in types:
-    matcher = re.compile(r'static final String\[\] oldSortedNames = {|};')
+    filename = '%s/src/test/org/apache/lucene/backward_index/unsupported_versions.txt' % module
   else:
-    matcher = re.compile(r'static final String\[\] oldNames = {|};')
+    filename = '%s/src/test/org/apache/lucene/backward_index/versions.txt' % module
 
   strip_dash_suffix_re = re.compile(r'-.*')
 
@@ -114,53 +113,25 @@ def find_version(x):
     x = re.sub(strip_dash_suffix_re, '', x) # remove the -suffix if any
     return scriptutil.Version.parse(x)
 
-  class Edit(object):
-    start = None
-    def __call__(self, buffer, match, line):
-      if self.start:
-        # find where this version should exist
-        i = len(buffer) - 1
-        previous_version_exists = not ('};' in line and buffer[-1].strip().endswith("{"))
-        if previous_version_exists: # Only look if there is a version here
-          v = find_version(buffer[i])
-          while i >= self.start and v.on_or_after(index_version):
-            i -= 1
-            v = find_version(buffer[i])
-        i += 1 # readjust since we skipped past by 1
-
-        # unfortunately python doesn't have a range remove from list...
-        # here we want to remove any previous references to the version we are adding
-        while i < len(buffer) and index_version.on_or_after(find_version(buffer[i])):
-          buffer.pop(i)
-
-        if i == len(buffer) and previous_version_exists and not buffer[-1].strip().endswith(","):
-          # add comma
-          buffer[-1] = buffer[-1].rstrip() + ",\n" 
-
-        if previous_version_exists:
-          last = buffer[-1]
-          spaces = ' ' * (len(last) - len(last.lstrip()))
-        else:
-          spaces = '    '
-        for (j, t) in enumerate(types):
-          if t == 'sorted':
-            newline = spaces + ('"sorted.%s"') % index_version
-          else:
-            newline = spaces + ('"%s-%s"' % (index_version, t))
-          if j < len(types) - 1 or i < len(buffer):
-            newline += ','
-          buffer.insert(i, newline + '\n')
-          i += 1
-
-        buffer.append(line)
-        return True
-
-      if 'Names = {' in line:
-        self.start = len(buffer) # location of first index name
-      buffer.append(line)
-      return False
+  def edit(buffer, match, line):
+    v = find_version(line)
+    changed = False
+    if v.on_or_after(index_version):
+       if not index_version.on_or_after(v):
+         buffer.append(('%s\n') % index_version)
+       changed = True
+    buffer.append(line)
+    return changed
+
+  def append(buffer, changed):
+    if changed:
+      return changed
+    if not buffer[len(buffer)-1].endswith('\n'):
+      buffer.append('\n')
+    buffer.append(('%s\n') % index_version)
+    return True
 
-  changed = scriptutil.update_file(filename, matcher, Edit())
+  changed = scriptutil.update_file(filename, re.compile(r'.*'), edit, append)
   print('done' if changed else 'uptodate')
 
 def check_backcompat_tests():
@@ -251,9 +222,8 @@ def main():
     print ('\nMANUAL UPDATE REQUIRED: edit TestGenerateBwcIndices to enable moreterms, dvupdates, and empty index testing')
 
   print('\nAdding backwards compatibility tests')
-  update_backcompat_tests(['cfs', 'nocfs'], c.version, current_version)
-  if should_make_sorted:
-    update_backcompat_tests(['sorted'], c.version, current_version)
+  update_backcompat_tests(c.version, current_version)
+
 
   print('\nTesting changes')
   check_backcompat_tests()

dev-tools/scripts/scriptutil.py

@@ -88,7 +88,7 @@ def run(cmd, cwd=None):
     raise e
   return output.decode('utf-8') 
 
-def update_file(filename, line_re, edit):
+def update_file(filename, line_re, edit, append=None):
   infile = open(filename, 'r')
   buffer = [] 
 
@@ -102,6 +102,8 @@ def update_file(filename, line_re, edit):
           return False
         continue
     buffer.append(line)
+  if append:
+    changed = append(buffer, changed) # in the case did not change in edit but have an append function
   if not changed:
     raise Exception('Could not find %s in %s' % (line_re, filename))
   with open(filename, 'w') as f:

lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java

@@ -19,20 +19,25 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.LineNumberReader;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
@@ -48,25 +53,31 @@
 
 public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase {
 
-  protected final Version version;
+  static final Set<String> OLD_VERSIONS;
+  protected static final Set<Version> BINARY_SUPPORTED_VERSIONS;
+
   private static final Version LATEST_PREVIOUS_MAJOR = getLatestPreviousMajorVersion();
+
+  protected final Version version;
   protected final String indexPattern;
-  protected static final Set<Version> BINARY_SUPPORTED_VERSIONS;
 
   static {
-    String[] oldVersions =
-        new String[] {
-          "8.0.0", "8.0.0", "8.1.0", "8.1.0", "8.1.1", "8.1.1", "8.2.0", "8.2.0", "8.3.0", "8.3.0",
-          "8.3.1", "8.3.1", "8.4.0", "8.4.0", "8.4.1", "8.4.1", "8.5.0", "8.5.0", "8.5.1", "8.5.1",
-          "8.5.2", "8.5.2", "8.6.0", "8.6.0", "8.6.1", "8.6.1", "8.6.2", "8.6.2", "8.6.3", "8.6.3",
-          "8.7.0", "8.7.0", "8.8.0", "8.8.0", "8.8.1", "8.8.1", "8.8.2", "8.8.2", "8.9.0", "8.9.0",
-          "8.10.0", "8.10.0", "8.10.1", "8.10.1", "8.11.0", "8.11.0", "8.11.1", "8.11.1", "8.11.2",
-          "8.11.2", "8.11.3", "8.11.3", "9.0.0", "9.1.0", "9.2.0", "9.3.0", "9.4.0", "9.4.1",
-          "9.4.2", "9.5.0", "9.6.0", "9.7.0", "9.8.0", "9.9.0", "9.9.1", "9.9.2"
-        };
-
+    String name = "versions.txt";
+    try (LineNumberReader in =
+        new LineNumberReader(
+            IOUtils.getDecodingReader(
+                IOUtils.requireResourceNonNull(
+                    BackwardsCompatibilityTestBase.class.getResourceAsStream(name), name),
+                StandardCharsets.UTF_8))) {
+      OLD_VERSIONS =
+          in.lines()
+              .filter(Predicate.not(String::isBlank))
+              .collect(Collectors.toCollection(LinkedHashSet::new));
+    } catch (IOException exception) {
+      throw new RuntimeException("failed to load resource", exception);
+    }
     Set<Version> binaryVersions = new HashSet<>();
-    for (String version : oldVersions) {
+    for (String version : OLD_VERSIONS) {
       try {
         Version v = Version.parse(version);
         assertTrue("Unsupported binary version: " + v, v.major >= Version.MIN_SUPPORTED_MAJOR - 1);