Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rewrite tests to make HTTP calls to elastic and not use library #68

Merged
merged 1 commit into from
May 10, 2024
Merged

Rewrite tests to make HTTP calls to elastic and not use library #68

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ require (
github.com/lib/pq v1.10.9
github.com/nyaruka/ezconf v0.3.0
github.com/nyaruka/gocommon v1.53.2
github.com/olivere/elastic/v7 v7.0.32
github.com/pkg/errors v0.9.1
github.com/samber/slog-multi v1.0.2
github.com/samber/slog-sentry v1.2.2
Expand All @@ -20,9 +19,7 @@ require (
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/go-chi/chi/v5 v5.0.12 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/naoina/go-stringutil v0.1.0 // indirect
github.com/naoina/toml v0.1.1 // indirect
github.com/nyaruka/librato v1.1.1 // indirect
Expand Down
8 changes: 0 additions & 8 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
Expand All @@ -18,14 +16,10 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
Expand All @@ -40,8 +34,6 @@ github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw
github.com/nyaruka/null/v2 v2.0.3/go.mod h1:OCVeCkCXwrg5/qE6RU0c1oUVZBy+ZDrT+xYg1XSaIWA=
github.com/nyaruka/phonenumbers v1.3.4 h1:bF1Wdh++fxw09s3surhVeBhXEcUKG07pHeP8HQXqjn8=
github.com/nyaruka/phonenumbers v1.3.4/go.mod h1:Ut+eFwikULbmCenH6InMKL9csUNLyxHuBLyfkpum11s=
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
Expand Down
63 changes: 40 additions & 23 deletions indexers/base_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,29 @@
package indexers_test

import (
"context"
"bytes"
"database/sql"
"log"
"encoding/json"
"io"
"log/slog"
"net/http"
"os"
"sort"
"strconv"
"strings"
"testing"

"github.com/nyaruka/gocommon/httpx"
"github.com/nyaruka/gocommon/jsonx"
"github.com/nyaruka/rp-indexer/v9/indexers"
"github.com/olivere/elastic/v7"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

const elasticURL = "http://localhost:9200"
const aliasName = "indexer_test"

func setup(t *testing.T) (*sql.DB, *elastic.Client) {
func setup(t *testing.T) *sql.DB {
testDB, err := os.ReadFile("../testdb.sql")
require.NoError(t, err)

Expand All @@ -30,44 +33,41 @@ func setup(t *testing.T) (*sql.DB, *elastic.Client) {
_, err = db.Exec(string(testDB))
require.NoError(t, err)

es, err := elastic.NewClient(elastic.SetURL(elasticURL), elastic.SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)), elastic.SetSniff(false))
require.NoError(t, err)

// delete all indexes with our alias prefix
existing, err := es.IndexNames()
require.NoError(t, err)
existing := elasticRequest(t, http.MethodGet, "/_aliases", nil)

for _, name := range existing {
for name := range existing {
if strings.HasPrefix(name, aliasName) {
_, err = es.DeleteIndex(name).Do(context.Background())
require.NoError(t, err)
elasticRequest(t, http.MethodDelete, "/"+name, nil)
}
}

slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})))

return db, es
return db
}

func assertQuery(t *testing.T, client *elastic.Client, query elastic.Query, expected []int64, msgAndArgs ...interface{}) {
results, err := client.Search().Index(aliasName).Query(query).Sort("id", true).Pretty(true).Do(context.Background())
assert.NoError(t, err)
func assertQuery(t *testing.T, query []byte, expected []int64, msgAndArgs ...interface{}) {
results := elasticRequest(t, http.MethodPost, "/"+aliasName+"/_search",
map[string]any{"query": json.RawMessage(query), "sort": []map[string]any{{"id": "asc"}}},
)
hits := results["hits"].(map[string]any)["hits"].([]any)

actual := make([]int64, len(results.Hits.Hits))
for h, hit := range results.Hits.Hits {
asInt, _ := strconv.Atoi(hit.Id)
actual := make([]int64, len(hits))
for h, hit := range hits {
idStr := hit.(map[string]any)["_id"].(string)
asInt, _ := strconv.Atoi(idStr)
actual[h] = int64(asInt)
}

assert.Equal(t, expected, actual, msgAndArgs...)
}

func assertIndexesWithPrefix(t *testing.T, es *elastic.Client, prefix string, expected []string) {
all, err := es.IndexNames()
require.NoError(t, err)
func assertIndexesWithPrefix(t *testing.T, prefix string, expected []string) {
all := elasticRequest(t, http.MethodGet, "/_aliases", nil)

actual := []string{}
for _, name := range all {
for name := range all {
if strings.HasPrefix(name, prefix) {
actual = append(actual, name)
}
Expand All @@ -81,3 +81,20 @@ func assertIndexerStats(t *testing.T, ix indexers.Indexer, expectedIndexed, expe
assert.Equal(t, expectedIndexed, actual.Indexed, "indexed mismatch")
assert.Equal(t, expectedDeleted, actual.Deleted, "deleted mismatch")
}

func elasticRequest(t *testing.T, method, path string, data map[string]any) map[string]any {
var body io.Reader
if data != nil {
body = bytes.NewReader(jsonx.MustMarshal(data))
}
req, err := httpx.NewRequest(method, elasticURL+path, body, map[string]string{"Content-Type": "application/json"})
require.NoError(t, err)

trace, err := httpx.DoTrace(http.DefaultClient, req, nil, nil, -1)
require.NoError(t, err)

resp, err := jsonx.DecodeGeneric(trace.ResponseBody)
require.NoError(t, err)

return resp.(map[string]any)
}
Loading
Loading