From 8503e85e34bd9f79c9833df04c96ebd40ddc7030 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 5 Jun 2024 12:57:33 -0500 Subject: [PATCH] Remove multi-search-db CI testing because it's unreliable --- .github/workflows/ci.yml | 32 +++++--------------------------- indexers/base_test.go | 18 ++++-------------- indexers/contacts_test.go | 18 +++++++++--------- 3 files changed, 18 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f71074..e325417 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,12 +6,6 @@ jobs: test: name: Test runs-on: ubuntu-latest - container: - image: node:20 - strategy: - matrix: - searchdb-host: [elastic7, elastic8, opensearch2] - fail-fast: false services: postgres: @@ -20,26 +14,15 @@ jobs: POSTGRES_DB: indexer_test POSTGRES_USER: indexer_test POSTGRES_PASSWORD: temba + ports: + - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - elastic7: + elastic: image: elasticsearch:7.17.9 env: discovery.type: single-node - options: --ulimit memlock=-1:-1 --ulimit nofile=65536:65536 - elastic8: - image: elasticsearch:8.13.4 - env: - discovery.type: single-node - xpack.security.enabled: false - options: --ulimit memlock=-1:-1 --ulimit nofile=65536:65536 - opensearch2: - image: opensearchproject/opensearch:2.13.0 - env: - discovery.type: single-node - plugins.security.disabled: true - OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m - DISABLE_INSTALL_DEMO_CONFIG: true - OPENSEARCH_INITIAL_ADMIN_PASSWORD: temba + ports: + - 9200:9200 options: --ulimit memlock=-1:-1 --ulimit nofile=65536:65536 steps: @@ -51,11 +34,6 @@ jobs: with: go-version: ${{ env.go-version }} - - name: Setup environment - run: | - echo "INDEXER_DB=postgres://indexer_test:temba@postgres:5432/indexer_test?sslmode=disable" >> "$GITHUB_ENV" - echo "INDEXER_ELASTIC_URL=http://${{ matrix.searchdb-host }}:9200" >> "$GITHUB_ENV" - - name: Run tests run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./... diff --git a/indexers/base_test.go b/indexers/base_test.go index 6374ccf..b63882f 100644 --- a/indexers/base_test.go +++ b/indexers/base_test.go @@ -21,20 +21,10 @@ import ( "github.com/stretchr/testify/require" ) -const aliasName = "indexer_test" - -func getenv(key, def string) string { - val := os.Getenv(key) - if val == "" { - return def - } - return val -} - func setup(t *testing.T) (*indexer.Config, *sql.DB) { cfg := indexer.NewDefaultConfig() - cfg.DB = getenv("INDEXER_DB", "postgres://indexer_test:temba@localhost:5432/indexer_test?sslmode=disable") - cfg.ElasticURL = getenv("INDEXER_ELASTIC_URL", "http://localhost:9200") + cfg.DB = "postgres://indexer_test:temba@localhost:5432/indexer_test?sslmode=disable" + cfg.ContactsIndex = "indexer_test" testDB, err := os.ReadFile("../testdb.sql") require.NoError(t, err) @@ -49,7 +39,7 @@ func setup(t *testing.T) (*indexer.Config, *sql.DB) { existing := elasticRequest(t, cfg, http.MethodGet, "/_aliases", nil) for name := range existing { - if strings.HasPrefix(name, aliasName) { + if strings.HasPrefix(name, cfg.ContactsIndex) { elasticRequest(t, cfg, http.MethodDelete, "/"+name, nil) } } @@ -60,7 +50,7 @@ func setup(t *testing.T) (*indexer.Config, *sql.DB) { } func assertQuery(t *testing.T, cfg *indexer.Config, query elastic.Query, expected []int64, msgAndArgs ...interface{}) { - results := elasticRequest(t, cfg, http.MethodPost, "/"+aliasName+"/_search", + results := elasticRequest(t, cfg, http.MethodPost, "/"+cfg.ContactsIndex+"/_search", map[string]any{"query": query, "sort": []map[string]any{{"id": "asc"}}}, ) hits := results["hits"].(map[string]any)["hits"].([]any) diff --git a/indexers/contacts_test.go b/indexers/contacts_test.go index e5082e5..bcc9943 100644 --- a/indexers/contacts_test.go +++ b/indexers/contacts_test.go @@ -166,7 +166,7 @@ var contactQueryTests = []struct { func TestContacts(t *testing.T) { cfg, db := setup(t) - ix1 := indexers.NewContactIndexer(cfg.ElasticURL, aliasName, 2, 1, 4) + ix1 := indexers.NewContactIndexer(cfg.ElasticURL, cfg.ContactsIndex, 2, 1, 4) assert.Equal(t, "indexer_test", ix1.Name()) dbModified, err := ix1.GetDBLastModified(context.Background(), db) @@ -174,7 +174,7 @@ func TestContacts(t *testing.T) { assert.WithinDuration(t, time.Date(2017, 11, 10, 21, 11, 59, 890662000, time.UTC), dbModified, 0) // error trying to get ES last modified on before index exists - _, err = ix1.GetESLastModified(aliasName) + _, err = ix1.GetESLastModified(cfg.ContactsIndex) assert.Error(t, err) expectedIndexName := fmt.Sprintf("indexer_test_%s", time.Now().Format("2006_01_02")) @@ -185,12 +185,12 @@ func TestContacts(t *testing.T) { time.Sleep(1 * time.Second) - esModified, err := ix1.GetESLastModified(aliasName) + esModified, err := ix1.GetESLastModified(cfg.ContactsIndex) assert.NoError(t, err) assert.WithinDuration(t, time.Date(2017, 11, 10, 21, 11, 59, 890662000, time.UTC), esModified, 0) assertIndexerStats(t, ix1, 9, 0) - assertIndexesWithPrefix(t, cfg, aliasName, []string{expectedIndexName}) + assertIndexesWithPrefix(t, cfg, cfg.ContactsIndex, []string{expectedIndexName}) for _, tc := range contactQueryTests { assertQuery(t, cfg, tc.query, tc.expected, "query mismatch for %s", tc.query) @@ -215,7 +215,7 @@ func TestContacts(t *testing.T) { time.Sleep(1 * time.Second) - assertIndexesWithPrefix(t, cfg, aliasName, []string{expectedIndexName}) + assertIndexesWithPrefix(t, cfg, cfg.ContactsIndex, []string{expectedIndexName}) // should only match new john, old john is gone assertQuery(t, cfg, elastic.Match("name", "john"), []int64{2}) @@ -229,7 +229,7 @@ func TestContacts(t *testing.T) { require.NoError(t, err) // and simulate another indexer doing a parallel rebuild - ix2 := indexers.NewContactIndexer(cfg.ElasticURL, aliasName, 2, 1, 4) + ix2 := indexers.NewContactIndexer(cfg.ElasticURL, cfg.ContactsIndex, 2, 1, 4) indexName2, err := ix2.Index(db, true, false) assert.NoError(t, err) @@ -239,20 +239,20 @@ func TestContacts(t *testing.T) { time.Sleep(1 * time.Second) // check we have a new index but the old index is still around - assertIndexesWithPrefix(t, cfg, aliasName, []string{expectedIndexName, expectedIndexName + "_1"}) + assertIndexesWithPrefix(t, cfg, cfg.ContactsIndex, []string{expectedIndexName, expectedIndexName + "_1"}) // and the alias points to the new index assertQuery(t, cfg, elastic.Match("name", "eric"), []int64{2}) // simulate another indexer doing a parallel rebuild with cleanup - ix3 := indexers.NewContactIndexer(cfg.ElasticURL, aliasName, 2, 1, 4) + ix3 := indexers.NewContactIndexer(cfg.ElasticURL, cfg.ContactsIndex, 2, 1, 4) indexName3, err := ix3.Index(db, true, true) assert.NoError(t, err) assert.Equal(t, expectedIndexName+"_2", indexName3) // new index used assertIndexerStats(t, ix3, 8, 0) // check we cleaned up indexes besides the new one - assertIndexesWithPrefix(t, cfg, aliasName, []string{expectedIndexName + "_2"}) + assertIndexesWithPrefix(t, cfg, cfg.ContactsIndex, []string{expectedIndexName + "_2"}) // check that the original indexer now indexes against the new index indexName, err = ix1.Index(db, false, false)