From 7f3f219c889e318c8159369e33f0b900c0a9e8f7 Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Tue, 14 Nov 2023 09:15:59 +0300 Subject: [PATCH 1/6] github badges github badges --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 199239f6e..52caced5f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,15 @@ # Pandora +[![Release](https://github.com/yandex/pandora/actions/workflows/release.yml/badge.svg)](https://github.com/yandex/pandora/actions/workflows/release.yml) +[![Release](https://img.shields.io/github/v/release/yandex/pandora.svg?style=flat-square)](https://github.com/yandex/pandora/releases) +[![Test](https://github.com/yandex/pandora/actions/workflows/test.yml/badge.svg)](https://github.com/yandex/pandora/actions/workflows/test.yml) +[![Coverage Status](https://coveralls.io/repos/yandex/pandora/badge.svg?branch=develop&service=github)](https://coveralls.io/github/yandex/pandora?branch=master) +[![codecov](https://codecov.io/gh/yandex/pandora/badge.svg?precision=2)](https://app.codecov.io/gh/yandex/pandora) +![Code lines](https://sloc.xyz/github/yandex/pandora/?category=code) + +[![PkgGoDev](https://pkg.go.dev/badge/github.com/yandex/pandora)](https://pkg.go.dev/github.com/yandex/pandora) +[![Go Report Card](https://goreportcard.com/badge/github.com/yandex/pandora)](https://goreportcard.com/report/github.com/yandex/pandora) [![Join the chat at https://gitter.im/yandex/pandora](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/yandex/pandora?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Build Status](https://travis-ci.org/yandex/pandora.svg)](https://travis-ci.org/yandex/pandora) -[![Coverage Status](https://coveralls.io/repos/yandex/pandora/badge.svg?branch=develop&service=github)](https://coveralls.io/github/yandex/pandora?branch=develop) -[![Documentation Status](https://readthedocs.org/projects/yandexpandora/badge/?version=develop)](https://yandexpandora.readthedocs.io/en/develop/?badge=develop) Pandora is a high-performance load generator in Go language. It has built-in HTTP(S) and HTTP/2 support and you can write your own load scenarios in Go, compiling them just before your test. From ab2a780ff55aa8df9495ebb14b1509a3af6ee919 Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Tue, 14 Nov 2023 09:40:07 +0300 Subject: [PATCH 2/6] github actions release github actions release --- .github/workflows/release.yml | 50 +++++++++++++++++++++++++++++++++++ .mapping.json | 1 + 2 files changed, 51 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..6958e948f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,50 @@ +name: Release + +on: + release: + types: [created] + +permissions: + contents: write + packages: write + +jobs: + releases-matrix: + name: Release Go Binary + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest] + arch: [amd64] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.20.x + cache: true + + - name: Test + run: go test -race -covermode atomic ./... + + - name: Set GITHUB_ENV + run: | + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + echo "GOOS=linux" >> $GITHUB_ENV + else + echo "GOOS=darwin" >> $GITHUB_ENV + fi + + - name: Build + run: | + export GOARCH=${{ matrix.arch }} + export CGO_ENABLED=0 + go build -o pandora_${{ github.event.release.tag_name }}_${GOOS}_${{ matrix.arch }} + + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: pandora_${{ github.event.release.tag_name }}_${{ env.GOOS }}_${{ matrix.arch }} diff --git a/.mapping.json b/.mapping.json index d1f42f783..0febfaea3 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1,4 +1,5 @@ { + ".github/workflows/release.yml":"load/projects/pandora/.github/workflows/release.yml", ".github/workflows/test.yml":"load/projects/pandora/.github/workflows/test.yml", ".gitignore":"load/projects/pandora/.gitignore", ".goxc.json":"load/projects/pandora/.goxc.json", From 4708406a6e3a043e7ef376606682de62dbc98836 Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Thu, 23 Nov 2023 18:35:15 +0300 Subject: [PATCH 3/6] engine tests without ginko --- .mapping.json | 1 - core/engine/engine_suite_test.go | 21 -- core/engine/engine_test.go | 492 ++++++++++++++++++++----------- core/engine/instance_test.go | 191 ++++++------ 4 files changed, 425 insertions(+), 280 deletions(-) delete mode 100644 core/engine/engine_suite_test.go diff --git a/.mapping.json b/.mapping.json index 0febfaea3..0f902ec11 100644 --- a/.mapping.json +++ b/.mapping.json @@ -162,7 +162,6 @@ "core/datasource/file_test.go":"load/projects/pandora/core/datasource/file_test.go", "core/datasource/std.go":"load/projects/pandora/core/datasource/std.go", "core/engine/engine.go":"load/projects/pandora/core/engine/engine.go", - "core/engine/engine_suite_test.go":"load/projects/pandora/core/engine/engine_suite_test.go", "core/engine/engine_test.go":"load/projects/pandora/core/engine/engine_test.go", "core/engine/instance.go":"load/projects/pandora/core/engine/instance.go", "core/engine/instance_test.go":"load/projects/pandora/core/engine/instance_test.go", diff --git a/core/engine/engine_suite_test.go b/core/engine/engine_suite_test.go deleted file mode 100644 index 752baf610..000000000 --- a/core/engine/engine_suite_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package engine - -import ( - "testing" - - "github.com/yandex/pandora/lib/ginkgoutil" - "github.com/yandex/pandora/lib/monitoring" -) - -func TestEngine(t *testing.T) { - ginkgoutil.RunSuite(t, "Engine Suite") -} - -func newTestMetrics() Metrics { - return Metrics{ - &monitoring.Counter{}, - &monitoring.Counter{}, - &monitoring.Counter{}, - &monitoring.Counter{}, - } -} diff --git a/core/engine/engine_test.go b/core/engine/engine_test.go index 5729b7456..76deea9b9 100644 --- a/core/engine/engine_test.go +++ b/core/engine/engine_test.go @@ -2,41 +2,43 @@ package engine import ( "context" + "reflect" "sync" + "testing" "time" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/aggregator" "github.com/yandex/pandora/core/config" coremock "github.com/yandex/pandora/core/mocks" "github.com/yandex/pandora/core/provider" "github.com/yandex/pandora/core/schedule" - "github.com/yandex/pandora/lib/ginkgoutil" + "github.com/yandex/pandora/lib/monitoring" "go.uber.org/atomic" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) -var _ = Describe("config validation", func() { - It("dive validation", func() { +func Test_ConfigValidation(t *testing.T) { + t.Run("dive validation", func(t *testing.T) { conf := Config{ Pools: []InstancePoolConfig{ {}, }, } err := config.Validate(conf) - Expect(err).To(HaveOccurred()) + require.Error(t, err) }) - - It("pools required", func() { + t.Run("pools required", func(t *testing.T) { conf := Config{} err := config.Validate(conf) - Expect(err).To(HaveOccurred()) + require.Error(t, err) }) - -}) +} func newTestPoolConf() (InstancePoolConfig, *coremock.Gun) { gun := &coremock.Gun{} @@ -56,7 +58,7 @@ func newTestPoolConf() (InstancePoolConfig, *coremock.Gun) { return conf, gun } -var _ = Describe("instance pool", func() { +func Test_InstancePool(t *testing.T) { var ( gun *coremock.Gun conf InstancePoolConfig @@ -70,7 +72,7 @@ var _ = Describe("instance pool", func() { ) // Conf for starting only instance. - BeforeEach(func() { + var beforeEach = func() { conf, gun = newTestPoolConf() onWaitDone = func() { old := waitDoneCalled.Swap(true) @@ -80,27 +82,28 @@ var _ = Describe("instance pool", func() { } waitDoneCalled.Store(false) ctx, cancel = context.WithCancel(context.Background()) - }) - - JustBeforeEach(func() { + } + var justBeforeEach = func() { metrics := newTestMetrics() - p = newPool(ginkgoutil.NewLogger(), metrics, onWaitDone, conf) - }) + p = newPool(newNopLogger(), metrics, onWaitDone, conf) + } + _ = cancel + + t.Run("shoot ok", func(t *testing.T) { + beforeEach() + justBeforeEach() - Context("shoot ok", func() { - It("", func() { - err := p.Run(ctx) - Expect(err).To(BeNil()) - ginkgoutil.AssertExpectations(gun) - Expect(waitDoneCalled.Load()).To(BeTrue()) - }, 1) + err := p.Run(ctx) + require.NoError(t, err) + gun.AssertExpectations(t) + require.True(t, waitDoneCalled.Load()) }) - Context("context canceled", func() { + t.Run("context canceled", func(t *testing.T) { var ( blockShoot sync.WaitGroup ) - BeforeEach(func() { + var beforeEachContext = func() { blockShoot.Add(1) prov := &coremock.Provider{} prov.On("Run", mock.Anything, mock.Anything). @@ -114,87 +117,139 @@ var _ = Describe("instance pool", func() { return struct{}{}, true }) conf.Provider = prov - }) - It("", func() { - err := p.Run(ctx) - Expect(err).To(Equal(context.Canceled)) - ginkgoutil.AssertNotCalled(gun, "Shoot") - Expect(waitDoneCalled.Load()).To(BeFalse()) - blockShoot.Done() - Eventually(waitDoneCalled.Load).Should(BeTrue()) - }, 1) + } + + beforeEach() + beforeEachContext() + justBeforeEach() + + err := p.Run(ctx) + require.Equal(t, context.Canceled, err) + gun.AssertNotCalled(t, "Shoot") + assert.False(t, waitDoneCalled.Load()) + blockShoot.Done() + + tick := time.NewTicker(100 * time.Millisecond) + i := 0 + for range tick.C { + if waitDoneCalled.Load() { + break + } + if i > 6 { + break + } + i++ + } + tick.Stop() + assert.True(t, waitDoneCalled.Load()) //TODO: eventually }) - Context("provider failed", func() { + t.Run("provider failed", func(t *testing.T) { + beforeEach() + var ( failErr = errors.New("test err") blockShootAndAggr sync.WaitGroup ) - BeforeEach(func() { - blockShootAndAggr.Add(1) - prov := &coremock.Provider{} - prov.On("Run", mock.Anything, mock.Anything). - Return(func(context.Context, core.ProviderDeps) error { - return failErr - }) - prov.On("Acquire").Return(func() (core.Ammo, bool) { - blockShootAndAggr.Wait() - return nil, false + blockShootAndAggr.Add(1) + prov := &coremock.Provider{} + prov.On("Run", mock.Anything, mock.Anything). + Return(func(context.Context, core.ProviderDeps) error { + return failErr }) - conf.Provider = prov - aggr := &coremock.Aggregator{} - aggr.On("Run", mock.Anything, mock.Anything). - Return(func(context.Context, core.AggregatorDeps) error { - blockShootAndAggr.Wait() - return nil - }) - conf.Aggregator = aggr - }) - It("", func() { - err := p.Run(ctx) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring(failErr.Error())) - ginkgoutil.AssertNotCalled(gun, "Shoot") - Consistently(waitDoneCalled.Load, 0.1).Should(BeFalse()) - blockShootAndAggr.Done() - Eventually(waitDoneCalled.Load).Should(BeTrue()) + prov.On("Acquire").Return(func() (core.Ammo, bool) { + blockShootAndAggr.Wait() + return nil, false }) + conf.Provider = prov + aggr := &coremock.Aggregator{} + aggr.On("Run", mock.Anything, mock.Anything). + Return(func(context.Context, core.AggregatorDeps) error { + blockShootAndAggr.Wait() + return nil + }) + conf.Aggregator = aggr + + justBeforeEach() + + err := p.Run(ctx) + require.Error(t, err) + require.ErrorContains(t, err, failErr.Error()) + gun.AssertNotCalled(t, "Shoot") + + assert.False(t, waitDoneCalled.Load()) + blockShootAndAggr.Done() + + tick := time.NewTicker(100 * time.Millisecond) + i := 0 + for range tick.C { + if waitDoneCalled.Load() { + break + } + if i > 6 { + break + } + i++ + } + tick.Stop() + assert.True(t, waitDoneCalled.Load()) //TODO: eventually }) - Context("aggregator failed", func() { + t.Run("aggregator failed", func(t *testing.T) { + beforeEach() failErr := errors.New("test err") - BeforeEach(func() { - aggr := &coremock.Aggregator{} - aggr.On("Run", mock.Anything, mock.Anything).Return(failErr) - conf.Aggregator = aggr - }) - It("", func() { - err := p.Run(ctx) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring(failErr.Error())) - Eventually(waitDoneCalled.Load).Should(BeTrue()) - }, 1) + aggr := &coremock.Aggregator{} + aggr.On("Run", mock.Anything, mock.Anything).Return(failErr) + conf.Aggregator = aggr + justBeforeEach() + + err := p.Run(ctx) + require.Error(t, err) + require.ErrorContains(t, err, failErr.Error()) + tick := time.NewTicker(100 * time.Millisecond) + i := 0 + for range tick.C { + if waitDoneCalled.Load() { + break + } + if i > 6 { + break + } + i++ + } + tick.Stop() + assert.True(t, waitDoneCalled.Load()) //TODO: eventually }) - Context("start instances failed", func() { + t.Run("start instances failed", func(t *testing.T) { failErr := errors.New("test err") - BeforeEach(func() { - conf.NewGun = func() (core.Gun, error) { - return nil, failErr + beforeEach() + conf.NewGun = func() (core.Gun, error) { + return nil, failErr + } + justBeforeEach() + + err := p.Run(ctx) + require.Error(t, err) + require.ErrorContains(t, err, failErr.Error()) + tick := time.NewTicker(100 * time.Millisecond) + i := 0 + for range tick.C { + if waitDoneCalled.Load() { + break } - }) - It("", func() { - err := p.Run(ctx) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring(failErr.Error())) - Eventually(waitDoneCalled.Load).Should(BeTrue()) - }, 1) + if i > 6 { + break + } + i++ + } + tick.Stop() + assert.True(t, waitDoneCalled.Load()) //TODO: eventually }) +} -}) - -var _ = Describe("multiple instance", func() { - It("out of ammo - instance start is canceled", func() { +func Test_MultipleInstance(t *testing.T) { + t.Run("out of ammo - instance start is canceled", func(t *testing.T) { conf, _ := newTestPoolConf() conf.Provider = provider.NewNum(3) conf.NewRPSSchedule = func() (core.Schedule, error) { @@ -204,30 +259,30 @@ var _ = Describe("multiple instance", func() { schedule.NewOnce(2), schedule.NewConst(1, 5*time.Second), ) - pool := newPool(ginkgoutil.NewLogger(), newTestMetrics(), nil, conf) + pool := newPool(newNopLogger(), newTestMetrics(), nil, conf) ctx := context.Background() err := pool.Run(ctx) - Expect(err).NotTo(HaveOccurred()) - Expect(pool.metrics.InstanceStart.Get()).To(BeNumerically("<=", 3)) - }, 1) + require.NoError(t, err) + require.True(t, pool.metrics.InstanceStart.Get() == 3) + }) - It("when provider run done it does not mean out of ammo; instance start is not canceled", func() { + t.Run("when provider run done it does not mean out of ammo; instance start is not canceled", func(t *testing.T) { conf, _ := newTestPoolConf() conf.Provider = provider.NewNumBuffered(3) conf.NewRPSSchedule = func() (core.Schedule, error) { return schedule.NewOnce(1), nil } conf.StartupSchedule = schedule.NewOnce(3) - pool := newPool(ginkgoutil.NewLogger(), newTestMetrics(), nil, conf) + pool := newPool(newNopLogger(), newTestMetrics(), nil, conf) ctx := context.Background() err := pool.Run(ctx) - Expect(err).NotTo(HaveOccurred()) - Expect(pool.metrics.InstanceStart.Get()).To(BeNumerically("==", 3)) - }, 1) + require.NoError(t, err) + require.True(t, pool.metrics.InstanceStart.Get() <= 3) + }) - It("out of RPS - instance start is canceled", func() { + t.Run("out of RPS - instance start is canceled", func(t *testing.T) { conf, _ := newTestPoolConf() conf.NewRPSSchedule = func() (core.Schedule, error) { return schedule.NewOnce(5), nil @@ -236,20 +291,19 @@ var _ = Describe("multiple instance", func() { schedule.NewOnce(2), schedule.NewConst(1, 2*time.Second), ) - pool := newPool(ginkgoutil.NewLogger(), newTestMetrics(), nil, conf) + pool := newPool(newNopLogger(), newTestMetrics(), nil, conf) ctx := context.Background() err := pool.Run(ctx) - Expect(err).NotTo(HaveOccurred()) - Expect(pool.metrics.InstanceStart.Get()).To(BeNumerically("<=", 3)) + require.NoError(t, err) + require.True(t, pool.metrics.InstanceStart.Get() <= 3) }) - -}) +} // TODO instance start canceled after out of ammo // TODO instance start cancdled after RPS finish -var _ = Describe("engine", func() { +func Test_Engine(t *testing.T) { var ( gun1, gun2 *coremock.Gun confs []InstancePoolConfig @@ -257,33 +311,37 @@ var _ = Describe("engine", func() { cancel context.CancelFunc engine *Engine ) - BeforeEach(func() { + _ = cancel + var beforeEach = func() { confs = make([]InstancePoolConfig, 2) confs[0], gun1 = newTestPoolConf() confs[1], gun2 = newTestPoolConf() ctx, cancel = context.WithCancel(context.Background()) - }) + } - JustBeforeEach(func() { + var justBeforeEach = func() { metrics := newTestMetrics() - engine = New(ginkgoutil.NewLogger(), metrics, Config{confs}) - }) + engine = New(newNopLogger(), metrics, Config{confs}) + } - Context("shoot ok", func() { - It("", func() { - err := engine.Run(ctx) - Expect(err).To(BeNil()) - ginkgoutil.AssertExpectations(gun1, gun2) - }) + t.Run("shoot ok", func(t *testing.T) { + beforeEach() + justBeforeEach() + + err := engine.Run(ctx) + require.NoError(t, err) + gun1.AssertExpectations(t) + gun2.AssertExpectations(t) }) - Context("context canceled", func() { + t.Run("context canceled", func(t *testing.T) { + // Cancel context on ammo acquire, an check that engine returns before // instance finish. var ( blockPools sync.WaitGroup ) - BeforeEach(func() { + var beforeEachCtx = func() { blockPools.Add(1) for i := range confs { prov := &coremock.Provider{} @@ -300,92 +358,186 @@ var _ = Describe("engine", func() { }) confs[i].Provider = prov } - }) + } + beforeEach() + beforeEachCtx() + justBeforeEach() + + err := engine.Run(ctx) + require.Equal(t, err, context.Canceled) + awaited := make(chan struct{}) + go func() { + defer close(awaited) + engine.Wait() + }() - It("", func() { - err := engine.Run(ctx) - Expect(err).To(Equal(context.Canceled)) - awaited := make(chan struct{}) - go func() { - defer close(awaited) - engine.Wait() - }() - Consistently(awaited, 0.1).ShouldNot(BeClosed()) - blockPools.Done() - Eventually(awaited).Should(BeClosed()) - }) + assert.False(t, IsClosed(awaited)) + blockPools.Done() + + tick := time.NewTicker(100 * time.Millisecond) + i := 0 + for range tick.C { + if IsClosed(awaited) { + break + } + if i > 6 { + break + } + i++ + } + tick.Stop() + assert.True(t, IsClosed(awaited)) //TODO: eventually }) - Context("one pool failed", func() { + t.Run("one pool failed", func(t *testing.T) { + beforeEach() var ( failErr = errors.New("test err") ) - BeforeEach(func() { - aggr := &coremock.Aggregator{} - aggr.On("Run", mock.Anything, mock.Anything).Return(failErr) - confs[0].Aggregator = aggr - }) + aggr := &coremock.Aggregator{} + aggr.On("Run", mock.Anything, mock.Anything).Return(failErr) + confs[0].Aggregator = aggr - It("", func() { - err := engine.Run(ctx) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring(failErr.Error())) - engine.Wait() - }, 1) + justBeforeEach() + + err := engine.Run(ctx) + require.Error(t, err) + require.ErrorContains(t, err, failErr.Error()) + engine.Wait() }) -}) +} -var _ = Describe("build instance schedule", func() { - It("per instance schedule ", func() { +func Test_BuildInstanceSchedule(t *testing.T) { + t.Run("per instance schedule", func(t *testing.T) { conf, _ := newTestPoolConf() conf.RPSPerInstance = true - pool := newPool(ginkgoutil.NewLogger(), newTestMetrics(), nil, conf) + pool := newPool(newNopLogger(), newTestMetrics(), nil, conf) newInstanceSchedule, err := pool.buildNewInstanceSchedule(context.Background(), func() { - Fail("should not be called") + panic("should not be called") }) - Expect(err).NotTo(HaveOccurred()) - ginkgoutil.ExpectFuncsEqual(newInstanceSchedule, conf.NewRPSSchedule) + require.NoError(t, err) + + val1 := reflect.ValueOf(newInstanceSchedule) + val2 := reflect.ValueOf(conf.NewRPSSchedule) + require.Equal(t, val1.Pointer(), val2.Pointer()) }) - It("shared schedule create failed", func() { + t.Run("shared schedule create failed", func(t *testing.T) { conf, _ := newTestPoolConf() scheduleCreateErr := errors.New("test err") conf.NewRPSSchedule = func() (core.Schedule, error) { return nil, scheduleCreateErr } - pool := newPool(ginkgoutil.NewLogger(), newTestMetrics(), nil, conf) + pool := newPool(newNopLogger(), newTestMetrics(), nil, conf) newInstanceSchedule, err := pool.buildNewInstanceSchedule(context.Background(), func() { - Fail("should not be called") + panic("should not be called") }) - Expect(err).To(Equal(scheduleCreateErr)) - Expect(newInstanceSchedule).To(BeNil()) + + require.Error(t, err) + require.Equal(t, err, scheduleCreateErr) + require.Nil(t, newInstanceSchedule) }) - It("shared schedule work", func() { + t.Run("shared schedule work", func(t *testing.T) { conf, _ := newTestPoolConf() var newScheduleCalled bool conf.NewRPSSchedule = func() (core.Schedule, error) { - Expect(newScheduleCalled).To(BeFalse()) + require.False(t, newScheduleCalled) newScheduleCalled = true return schedule.NewOnce(1), nil } - pool := newPool(ginkgoutil.NewLogger(), newTestMetrics(), nil, conf) + pool := newPool(newNopLogger(), newTestMetrics(), nil, conf) ctx, cancel := context.WithCancel(context.Background()) newInstanceSchedule, err := pool.buildNewInstanceSchedule(context.Background(), cancel) - Expect(err).NotTo(HaveOccurred()) + require.NoError(t, err) schedule, err := newInstanceSchedule() - Expect(err).NotTo(HaveOccurred()) + require.NoError(t, err) - Expect(newInstanceSchedule()).To(Equal(schedule)) - - Expect(ctx.Done()).NotTo(BeClosed()) + assert.False(t, IsClosed(ctx.Done())) _, ok := schedule.Next() - Expect(ok).To(BeTrue()) - Expect(ctx.Done()).NotTo(BeClosed()) + assert.True(t, ok) + assert.False(t, IsClosed(ctx.Done())) _, ok = schedule.Next() - Expect(ok).To(BeFalse()) - Expect(ctx.Done()).To(BeClosed()) + assert.False(t, ok) + assert.True(t, IsClosed(ctx.Done())) }) +} + +func IsClosed(actual any) (success bool) { + if !isChan(actual) { + return false + } + channelValue := reflect.ValueOf(actual) + channelType := reflect.TypeOf(actual) + if channelType.ChanDir() == reflect.SendDir { + return false + } -}) + winnerIndex, _, open := reflect.Select([]reflect.SelectCase{ + {Dir: reflect.SelectRecv, Chan: channelValue}, + {Dir: reflect.SelectDefault}, + }) + + var closed bool + if winnerIndex == 0 { + closed = !open + } else if winnerIndex == 1 { + closed = false + } + + return closed +} + +func isChan(a interface{}) bool { + if isNil(a) { + return false + } + return reflect.TypeOf(a).Kind() == reflect.Chan +} + +func isNil(a interface{}) bool { + if a == nil { + return true + } + + switch reflect.TypeOf(a).Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return reflect.ValueOf(a).IsNil() + } + + return false +} + +type TestLogWriter struct { + t *testing.T +} + +func (w *TestLogWriter) Write(p []byte) (n int, err error) { + w.t.Helper() + w.t.Log(string(p)) + return len(p), nil +} + +func newTestLogger(t *testing.T) *zap.Logger { + conf := zap.NewDevelopmentConfig() + enc := zapcore.NewConsoleEncoder(conf.EncoderConfig) + core := zapcore.NewCore(enc, zapcore.AddSync(&TestLogWriter{t: t}), zap.DebugLevel) + log := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zap.DPanicLevel)) + return log +} + +func newNopLogger() *zap.Logger { + core := zapcore.NewNopCore() + log := zap.New(core) + return log +} + +func newTestMetrics() Metrics { + return Metrics{ + &monitoring.Counter{}, + &monitoring.Counter{}, + &monitoring.Counter{}, + &monitoring.Counter{}, + } +} diff --git a/core/engine/instance_test.go b/core/engine/instance_test.go index 0945337a5..9f6adf4c3 100644 --- a/core/engine/instance_test.go +++ b/core/engine/instance_test.go @@ -3,18 +3,18 @@ package engine import ( "context" "errors" + "testing" "time" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/yandex/pandora/core" coremock "github.com/yandex/pandora/core/mocks" "github.com/yandex/pandora/core/schedule" - "github.com/yandex/pandora/lib/ginkgoutil" ) -var _ = Describe("Instance", func() { +func Test_Instance(t *testing.T) { var ( provider *coremock.Provider aggregator *coremock.Aggregator @@ -32,7 +32,7 @@ var _ = Describe("Instance", func() { newGun func() (core.Gun, error) ) - BeforeEach(func() { + var beforeEach = func() { provider = &coremock.Provider{} aggregator = &coremock.Aggregator{} gun = &coremock.Gun{} @@ -43,9 +43,9 @@ var _ = Describe("Instance", func() { metrics = newTestMetrics() newSchedule = func() (core.Schedule, error) { return sched, newScheduleErr } newGun = func() (core.Gun, error) { return gun, newGunErr } - }) + } - JustBeforeEach(func() { + var justBeforeEach = func() { deps := instanceDeps{ newSchedule, @@ -58,18 +58,18 @@ var _ = Describe("Instance", func() { false, }, } - ins, insCreateErr = newInstance(ctx, ginkgoutil.NewLogger(), "pool_0", 0, deps) - }) + ins, insCreateErr = newInstance(ctx, newNopLogger(), "pool_0", 0, deps) + } - AfterEach(func() { + var afterEach = func() { if newGunErr == nil && newScheduleErr == nil { - Expect(metrics.InstanceStart.Get()).To(BeEquivalentTo(1)) - Expect(metrics.InstanceFinish.Get()).To(BeEquivalentTo(1)) + assert.Equal(t, int64(1), metrics.InstanceStart.Get()) + assert.Equal(t, int64(1), metrics.InstanceFinish.Get()) } - }) + } - Context("all ok", func() { - BeforeEach(func() { + t.Run("all ok", func(t *testing.T) { + var beforeEachCtx = func() { const times = 5 sched = schedule.NewOnce(times) gun.On("Bind", aggregator, mock.Anything).Return(nil).Once() @@ -82,93 +82,108 @@ var _ = Describe("Instance", func() { gun.On("Shoot", i).Once() provider.On("Release", i).Once() } - }) - JustBeforeEach(func() { - Expect(insCreateErr).NotTo(HaveOccurred()) - }) - It("start ok", func() { + } + var justBeforeEachCtx = func() { + require.NoError(t, insCreateErr) + } + t.Run("start ok", func(t *testing.T) { + beforeEach() + beforeEachCtx() + justBeforeEachCtx() + justBeforeEach() + err := ins.Run(ctx) - Expect(err).NotTo(HaveOccurred()) - ginkgoutil.AssertExpectations(gun, provider) - }, 2) - - Context("gun implements io.Closer", func() { - var closeGun mockGunCloser - BeforeEach(func() { - closeGun = mockGunCloser{gun} - closeGun.On("Close").Return(nil) - newGun = func() (core.Gun, error) { - return closeGun, nil - } - }) - It("close called on instance close", func() { - err := ins.Run(ctx) - Expect(err).NotTo(HaveOccurred()) - ginkgoutil.AssertNotCalled(closeGun, "Close") - err = ins.Close() - Expect(err).NotTo(HaveOccurred()) - ginkgoutil.AssertExpectations(closeGun, provider) - }) + require.NoError(t, err) + gun.AssertExpectations(t) + provider.AssertExpectations(t) + afterEach() }) - }) - Context("context canceled after run", func() { - BeforeEach(func() { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(context.Background(), 10*time.Millisecond) - _ = cancel - sched := sched.(*coremock.Schedule) - sched.On("Next").Return(time.Now().Add(5*time.Second), true) - sched.On("Left").Return(1) - gun.On("Bind", aggregator, mock.Anything).Return(nil) - provider.On("Acquire").Return(struct{}{}, true) - provider.On("Release", mock.Anything).Return() - }) - It("start fail", func() { - err := ins.Run(ctx) - Expect(err).To(Equal(context.DeadlineExceeded)) - ginkgoutil.AssertExpectations(gun, provider) - }, 2) + t.Run("gun implements io.Closer / close called on instance close", func(t *testing.T) { + beforeEach() + beforeEachCtx() + closeGun := mockGunCloser{gun} + closeGun.On("Close").Return(nil) + newGun = func() (core.Gun, error) { + return closeGun, nil + } + justBeforeEachCtx() + justBeforeEach() + err := ins.Run(ctx) + require.NoError(t, err) + closeGun.AssertNotCalled(t, "Close") + err = ins.Close() + require.NoError(t, err) + closeGun.AssertExpectations(t) + provider.AssertExpectations(t) + + afterEach() + }) }) - Context("context canceled before run", func() { - BeforeEach(func() { - var cancel context.CancelFunc - ctx, cancel = context.WithCancel(ctx) - cancel() - gun.On("Bind", aggregator, mock.Anything).Return(nil) - }) - It("nothing acquired and schedule not started", func() { - err := ins.Run(ctx) - Expect(err).To(Equal(context.Canceled)) - ginkgoutil.AssertExpectations(gun, provider) - }, 2) + t.Run("context canceled after run / start fail", func(t *testing.T) { + beforeEach() + + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Millisecond) + _ = cancel + sched := sched.(*coremock.Schedule) + sched.On("Next").Return(time.Now().Add(5*time.Second), true) + sched.On("Left").Return(1) + gun.On("Bind", aggregator, mock.Anything).Return(nil) + provider.On("Acquire").Return(struct{}{}, true) + provider.On("Release", mock.Anything).Return() + + justBeforeEach() + err := ins.Run(ctx) + assert.Error(t, err) + assert.Equal(t, context.DeadlineExceeded, err) + gun.AssertExpectations(t) + provider.AssertExpectations(t) + + afterEach() }) - Context("schedule create failed", func() { - BeforeEach(func() { - sched = nil - newScheduleErr = errors.New("test err") - }) - It("instance create failed", func() { - Expect(insCreateErr).To(Equal(newScheduleErr)) - }) + t.Run("context canceled before run / nothing acquired and schedule not started", func(t *testing.T) { + beforeEach() + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + gun.On("Bind", aggregator, mock.Anything).Return(nil) + justBeforeEach() + + err := ins.Run(ctx) + require.Equal(t, context.Canceled, err) + gun.AssertExpectations(t) + provider.AssertExpectations(t) + + afterEach() }) - Context("gun create failed", func() { - BeforeEach(func() { - gun = nil - newGunErr = errors.New("test err") - }) - It("instance create failed", func() { - Expect(insCreateErr).To(Equal(newGunErr)) - }) + t.Run("schedule create failed / instance create failed", func(t *testing.T) { + beforeEach() + sched = nil + newScheduleErr = errors.New("test err") + justBeforeEach() + + require.Equal(t, newScheduleErr, insCreateErr) + + afterEach() }) -}) + t.Run("gun create failed / instance create failed", func(t *testing.T) { + beforeEach() + gun = nil + newGunErr = errors.New("test err") + justBeforeEach() + + require.Equal(t, newGunErr, insCreateErr) + afterEach() + }) +} type mockGunCloser struct { *coremock.Gun From 493319e72746b87734c2cb395100e5e51b8fc98c Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Thu, 23 Nov 2023 19:58:12 +0300 Subject: [PATCH 4/6] waiter context fix --- core/coreutil/waiter.go | 19 +++++++++---------- core/coreutil/waiter_test.go | 18 +++++++++--------- core/engine/engine.go | 6 +++--- core/engine/instance.go | 8 ++++---- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/core/coreutil/waiter.go b/core/coreutil/waiter.go index 88d3ce3dc..972e80c12 100644 --- a/core/coreutil/waiter.go +++ b/core/coreutil/waiter.go @@ -15,7 +15,6 @@ import ( // Waiter goroutine unsafe wrapper for efficient waiting schedule. type Waiter struct { sched core.Schedule - ctx context.Context slowDownItems int // Lazy initialized. @@ -23,17 +22,17 @@ type Waiter struct { lastNow time.Time } -func NewWaiter(sched core.Schedule, ctx context.Context) *Waiter { - return &Waiter{sched: sched, ctx: ctx} +func NewWaiter(sched core.Schedule) *Waiter { + return &Waiter{sched: sched} } // Wait waits for next waiter schedule event. // Returns true, if event successfully waited, or false // if waiter context is done, or schedule finished. -func (w *Waiter) Wait() (ok bool) { +func (w *Waiter) Wait(ctx context.Context) (ok bool) { // Check, that context is not done. Very quick: 5 ns for op, due to benchmark. select { - case <-w.ctx.Done(): + case <-ctx.Done(): w.slowDownItems = 0 return false default: @@ -65,15 +64,15 @@ func (w *Waiter) Wait() (ok bool) { select { case <-w.timer.C: return true - case <-w.ctx.Done(): + case <-ctx.Done(): return false } } // IsSlowDown returns true, if schedule contains 2 elements before current time. -func (w *Waiter) IsSlowDown() (ok bool) { +func (w *Waiter) IsSlowDown(ctx context.Context) (ok bool) { select { - case <-w.ctx.Done(): + case <-ctx.Done(): return false default: return w.slowDownItems >= 2 @@ -82,9 +81,9 @@ func (w *Waiter) IsSlowDown() (ok bool) { // IsFinished is quick check, that wait context is not canceled and there are some tokens left in // schedule. -func (w *Waiter) IsFinished() (ok bool) { +func (w *Waiter) IsFinished(ctx context.Context) (ok bool) { select { - case <-w.ctx.Done(): + case <-ctx.Done(): return true default: return w.sched.Left() == 0 diff --git a/core/coreutil/waiter_test.go b/core/coreutil/waiter_test.go index 75c5a1f56..5a94ba40c 100644 --- a/core/coreutil/waiter_test.go +++ b/core/coreutil/waiter_test.go @@ -16,9 +16,9 @@ import ( func TestWaiter_Unstarted(t *testing.T) { sched := schedule.NewOnce(1) ctx := context.Background() - w := NewWaiter(sched, ctx) + w := NewWaiter(sched) var i int - for ; w.Wait(); i++ { + for ; w.Wait(ctx); i++ { } require.Equal(t, 1, i) } @@ -31,11 +31,11 @@ func TestWaiter_WaitAsExpected(t *testing.T) { ) sched := schedule.NewConst(ops, duration) ctx := context.Background() - w := NewWaiter(sched, ctx) + w := NewWaiter(sched) start := time.Now() sched.Start(start) var i int - for ; w.Wait(); i++ { + for ; w.Wait(ctx); i++ { } finish := time.Now() @@ -48,8 +48,8 @@ func TestWaiter_ContextCanceledBeforeWait(t *testing.T) { sched := schedule.NewOnce(1) ctx, cancel := context.WithCancel(context.Background()) cancel() - w := NewWaiter(sched, ctx) - require.False(t, w.Wait()) + w := NewWaiter(sched) + require.False(t, w.Wait(ctx)) } func TestWaiter_ContextCanceledDuringWait(t *testing.T) { @@ -58,10 +58,10 @@ func TestWaiter_ContextCanceledDuringWait(t *testing.T) { start := time.Now() ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - w := NewWaiter(sched, ctx) + w := NewWaiter(sched) - require.True(t, w.Wait()) // 0 - require.False(t, w.Wait()) + require.True(t, w.Wait(ctx)) // 0 + require.False(t, w.Wait(ctx)) since := time.Since(start) require.True(t, since > timeout) diff --git a/core/engine/engine.go b/core/engine/engine.go index a7484385e..893528b9f 100644 --- a/core/engine/engine.go +++ b/core/engine/engine.go @@ -373,10 +373,10 @@ func (p *instancePool) startInstances( }, } - waiter := coreutil.NewWaiter(p.StartupSchedule, startCtx) + waiter := coreutil.NewWaiter(p.StartupSchedule) // If create all instances asynchronously, and creation will fail, too many errors appears in log. - ok := waiter.Wait() + ok := waiter.Wait(startCtx) if !ok { err = startCtx.Err() return @@ -393,7 +393,7 @@ func (p *instancePool) startInstances( }()} }() - for ; waiter.Wait(); started++ { + for ; waiter.Wait(startCtx); started++ { id := started go func() { runRes <- instanceRunResult{id, runNewInstance(runCtx, p.log, p.ID, id, deps)} diff --git a/core/engine/instance.go b/core/engine/instance.go index 8c36f3cdd..da8e4ad3e 100644 --- a/core/engine/instance.go +++ b/core/engine/instance.go @@ -80,10 +80,10 @@ func (i *instance) Run(ctx context.Context) (recoverErr error) { i.log.Debug("Instance started") i.metrics.InstanceStart.Add(1) - waiter := coreutil.NewWaiter(i.schedule, ctx) + waiter := coreutil.NewWaiter(i.schedule) // Checking, that schedule is not finished, required, to not consume extra ammo, // on finish in case of per instance schedule. - for !waiter.IsFinished() { + for !waiter.IsFinished(ctx) { err := func() error { ammo, ok := i.provider.Acquire() if !ok { @@ -94,10 +94,10 @@ func (i *instance) Run(ctx context.Context) (recoverErr error) { if tag.Debug { i.log.Debug("Ammo acquired", zap.Any("ammo", ammo)) } - if !waiter.Wait() { + if !waiter.Wait(ctx) { return nil } - if !i.discardOverflow || !waiter.IsSlowDown() { + if !i.discardOverflow || !waiter.IsSlowDown(ctx) { i.metrics.Request.Add(1) if tag.Debug { i.log.Debug("Shooting", zap.Any("ammo", ammo)) From 4a858e2d1dda16c5c99338c4eff836a8ddd5e738 Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Fri, 24 Nov 2023 13:38:14 +0300 Subject: [PATCH 5/6] remove badge remove badge --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 52caced5f..ccb4a9072 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ [![Release](https://github.com/yandex/pandora/actions/workflows/release.yml/badge.svg)](https://github.com/yandex/pandora/actions/workflows/release.yml) [![Release](https://img.shields.io/github/v/release/yandex/pandora.svg?style=flat-square)](https://github.com/yandex/pandora/releases) [![Test](https://github.com/yandex/pandora/actions/workflows/test.yml/badge.svg)](https://github.com/yandex/pandora/actions/workflows/test.yml) -[![Coverage Status](https://coveralls.io/repos/yandex/pandora/badge.svg?branch=develop&service=github)](https://coveralls.io/github/yandex/pandora?branch=master) [![codecov](https://codecov.io/gh/yandex/pandora/badge.svg?precision=2)](https://app.codecov.io/gh/yandex/pandora) ![Code lines](https://sloc.xyz/github/yandex/pandora/?category=code) From bbfbd6ec90925f6f8a462859163379be84271880 Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Fri, 24 Nov 2023 16:18:09 +0300 Subject: [PATCH 6/6] version up GetMapValue refactoring $ go test -bench=. -count=5 goos: darwin goarch: amd64 pkg: github.com/yandex/pandora/lib/mp cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz BenchmarkGetMapValue-12 45223 24348 ns/op BenchmarkGetMapValue-12 51969 23631 ns/op BenchmarkGetMapValue-12 52303 24771 ns/op BenchmarkGetMapValue-12 52562 23441 ns/op BenchmarkGetMapValue-12 49144 23131 ns/op BenchmarkGetMapValue2-12 49866 23739 ns/op BenchmarkGetMapValue2-12 47876 23441 ns/op BenchmarkGetMapValue2-12 49641 24082 ns/op BenchmarkGetMapValue2-12 50052 23706 ns/op BenchmarkGetMapValue2-12 51109 23714 ns/op PASS ok github.com/yandex/pandora/lib/mp 15.501s --- cli/cli.go | 2 +- lib/mp/map.go | 128 +++++++++-------- lib/mp/map_test.go | 345 +++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 394 insertions(+), 81 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index 32290bf24..12f9397ac 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -25,7 +25,7 @@ import ( "go.uber.org/zap/zapcore" ) -const Version = "0.5.16" +const Version = "0.5.17" const defaultConfigFile = "load" const stdinConfigSelector = "-" diff --git a/lib/mp/map.go b/lib/mp/map.go index 2c192af8f..25b7330cb 100644 --- a/lib/mp/map.go +++ b/lib/mp/map.go @@ -2,6 +2,7 @@ package mp import ( "fmt" + "reflect" "strconv" "strings" ) @@ -15,11 +16,10 @@ func (e *ErrSegmentNotFound) Error() string { return fmt.Sprintf("segment %s not found in path %s", e.segment, e.path) } -func GetMapValue(input map[string]any, path string, iter Iterator) (any, error) { +func GetMapValue(current map[string]any, path string, iter Iterator) (any, error) { var curSegment strings.Builder segments := strings.Split(path, ".") - currentData := input for i, segment := range segments { segment = strings.TrimSpace(segment) curSegment.WriteByte('.') @@ -29,80 +29,98 @@ func GetMapValue(input map[string]any, path string, iter Iterator) (any, error) indexStr := strings.ToLower(strings.TrimSpace(segment[openBraceIdx+1 : len(segment)-1])) segment = segment[:openBraceIdx] - value, exists := currentData[segment] - if !exists { + pathVal, ok := current[segment] + if !ok { return nil, &ErrSegmentNotFound{path: path, segment: segment} } - - mval, isMval := value.([]map[string]string) - if isMval { - index, err := calcIndex(indexStr, curSegment.String(), len(mval), iter) - if err != nil { - return nil, fmt.Errorf("failed to calc index: %w", err) - } - vval := mval[index] - currentData = make(map[string]any, len(vval)) - for k, v := range vval { - currentData[k] = v - } - continue + sliceElement, err := extractFromSlice(pathVal, indexStr, curSegment.String(), iter) + if err != nil { + return nil, fmt.Errorf("cant extract value path=`%s`,segment=`%s`,err=%w", segment, path, err) } - - mapSlice, isMapSlice := value.([]map[string]any) - if !isMapSlice { - anySlice, isAnySlice := value.([]any) - if isAnySlice { - index, err := calcIndex(indexStr, curSegment.String(), len(anySlice), iter) - if err != nil { - return nil, fmt.Errorf("failed to calc index: %w", err) - } - if i != len(segments)-1 { - return nil, fmt.Errorf("not last segment %s in path %s", segment, path) - } - return anySlice[index], nil - } - stringSlice, isStringSlice := value.([]string) - if isStringSlice { - index, err := calcIndex(indexStr, curSegment.String(), len(stringSlice), iter) - if err != nil { - return nil, fmt.Errorf("failed to calc index: %w", err) - } - if i != len(segments)-1 { - return nil, fmt.Errorf("not last segment %s in path %s", segment, path) - } - return stringSlice[index], nil + current, ok = sliceElement.(map[string]any) + if !ok { + if i != len(segments)-1 { + return nil, fmt.Errorf("not last segment %s in path %s", segment, path) } - return nil, fmt.Errorf("invalid type of segment %s in path %s", segment, path) + return sliceElement, nil } - - index, err := calcIndex(indexStr, curSegment.String(), len(mapSlice), iter) - if err != nil { - return nil, fmt.Errorf("failed to calc index: %w", err) - } - currentData = mapSlice[index] } else { - value, exists := currentData[segment] - if !exists { + pathVal, ok := current[segment] + if !ok { return nil, &ErrSegmentNotFound{path: path, segment: segment} } - var ok bool - currentData, ok = value.(map[string]any) + current, ok = pathVal.(map[string]any) if !ok { if i != len(segments)-1 { return nil, fmt.Errorf("not last segment %s in path %s", segment, path) } - return value, nil + return pathVal, nil } } } - return currentData, nil + return current, nil +} + +func extractFromSlice(curValue any, indexStr string, curSegment string, iter Iterator) (result any, err error) { + validTypes := []reflect.Type{ + reflect.TypeOf([]map[string]string{}), + reflect.TypeOf([]map[string]any{}), + reflect.TypeOf([]any{}), + reflect.TypeOf([]string{}), + reflect.TypeOf([]int{}), + reflect.TypeOf([]int64{}), + reflect.TypeOf([]float64{}), + } + + var valueLen int + var valueFound bool + for _, valueType := range validTypes { + if reflect.TypeOf(curValue) == valueType { + valueLen = reflect.ValueOf(curValue).Len() + valueFound = true + break + } + } + + if !valueFound { + return nil, fmt.Errorf("invalid type of value `%+v`, %T", curValue, curValue) + } + + index, err := calcIndex(indexStr, curSegment, valueLen, iter) + if err != nil { + return nil, fmt.Errorf("failed to calc index for %T; err: %w", curValue, err) + } + + switch v := curValue.(type) { + case []map[string]string: + currentData := make(map[string]any, len(v[index])) + for k, val := range v[index] { + currentData[k] = val + } + return currentData, nil + case []map[string]any: + return v[index], nil + case []any: + return v[index], nil + case []string: + return v[index], nil + case []int: + return v[index], nil + case []int64: + return v[index], nil + case []float64: + return v[index], nil + } + + // This line should never be reached, as we've covered all valid types above + return nil, fmt.Errorf("invalid type of value `%+v`, %T", curValue, curValue) } func calcIndex(indexStr string, segment string, length int, iter Iterator) (int, error) { index, err := strconv.Atoi(indexStr) if err != nil && indexStr != "next" && indexStr != "rand" && indexStr != "last" { - return 0, fmt.Errorf("invalid index: %s", indexStr) + return 0, fmt.Errorf("index should be integer or one of [next, rand, last], but got `%s`", indexStr) } if indexStr != "next" && indexStr != "rand" && indexStr != "last" { if index >= 0 && index < length { diff --git a/lib/mp/map_test.go b/lib/mp/map_test.go index 09aaeef6f..3a6abcd96 100644 --- a/lib/mp/map_test.go +++ b/lib/mp/map_test.go @@ -1,6 +1,7 @@ package mp import ( + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -8,15 +9,15 @@ import ( ) func TestGetMapValue(t *testing.T) { - tests := []struct { + var cases = []struct { name string reqMap map[string]any v string want any - wantErr bool + wantErr string }{ { - name: "", + name: "valid index for nested map with array map", reqMap: map[string]any{ "source": map[string]any{ "items": []map[string]any{ @@ -25,12 +26,64 @@ func TestGetMapValue(t *testing.T) { }, }, }, - v: "source.items[0].id", + v: "source.items[0].id", + want: "1", + }, + { + name: "invalid index for nested map with array map", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []map[string]any{ + {"id": "1"}, + {"id": "2"}, + }, + }, + }, + v: "source.items[asd].id", + wantErr: "failed to calc index for []map[string]interface {}; err: index should be integer or one of [next, rand, last], but got `asd`", + }, + { + name: "should slice type error ", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []map[string]any{ + {"id": "1"}, + {"id": "2"}, + }, + }, + }, + v: "source.items[0].id[0]", + wantErr: "cant extract value path=`id`,segment=`source.items[0].id[0]`,err=invalid type of value `1`, string", + }, + { + name: "not last segment", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []map[string]any{ + {"id": "1"}, + {"id": "2"}, + }, + }, + }, + v: "source.items[0].id.field", + wantErr: "not last segment id in path source.items[0].id.field", + }, + { + name: "segment data not found", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []map[string]any{ + {"id": "1"}, + {"id": "2"}, + }, + }, + }, + v: "source.data[0].id", want: "1", - wantErr: false, + wantErr: "segment data not found in path source.data[0].id", }, { - name: "", + name: "extract map from array / element in array / valid index", reqMap: map[string]any{ "source": map[string]any{ "items": []map[string]any{ @@ -39,12 +92,11 @@ func TestGetMapValue(t *testing.T) { }, }, }, - v: "source.items[1]", - want: map[string]any{"id": "2"}, - wantErr: false, + v: "source.items[1]", + want: map[string]any{"id": "2"}, }, { - name: "", + name: "retrieve non existent key / error when searching for missing key", reqMap: map[string]any{ "source": map[string]any{ "items": []map[string]any{ @@ -55,10 +107,10 @@ func TestGetMapValue(t *testing.T) { }, v: "source.items[1].title", want: nil, - wantErr: true, + wantErr: "segment title not found in path source.items[1].title", }, { - name: "", + name: "access items in nested map / valid path / return items list", reqMap: map[string]any{ "source": map[string]any{ "items": []map[string]any{ @@ -72,10 +124,9 @@ func TestGetMapValue(t *testing.T) { {"id": "1"}, {"id": "2"}, }, - wantErr: false, }, { - name: "", + name: "valid value for map[string]string in map[string]any", reqMap: map[string]any{ "source": map[string]any{ "items": []map[string]string{ @@ -84,23 +135,155 @@ func TestGetMapValue(t *testing.T) { }, }, }, - v: "source.items[0].id", + v: "source.items[0].id", + want: "1", + }, + { + name: "invalid index for map[string]string in map[string]any", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []map[string]string{ + {"id": "1"}, + {"id": "2"}, + }, + }, + }, + v: "source.items[asd].id", want: "1", - wantErr: false, + wantErr: "failed to calc index for []map[string]string; err: index should be integer or one of [next, rand, last], but got `asd`", + }, + { + name: "duplicate test case / same as / test case5", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []map[string]any{ + {"id": "1"}, + {"id": "2"}, + }, + }, + }, + v: "source.items[0].id", + want: "1", + }, + { + name: "valid key for []any in map[string]any", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []any{ + map[string]any{ + "id": "1", + "name": "name1", + }, + map[string]any{ + "id": "2", + "name": "name2", + }, + }, + }, + }, + v: "source.items[next].name", + want: "name1", }, { - name: "", + name: "invalid index for []any in map[string]any", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []any{ + map[string]any{ + "id": "1", + "name": "name1", + }, + map[string]any{ + "id": "2", + "name": "name2", + }, + }, + }, + }, + v: "source.items[asd].name", + wantErr: "failed to calc index for []interface {}; err: index should be integer or one of [next, rand, last], but got `asd`", + }, + { + name: "slice of strings", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []string{ + "1", + "2", + }, + }, + }, + v: "source.items[0]", + want: "1", + }, + { + name: "slice of strings / invalid index", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []string{ + "1", + "2", + }, + }, + }, + v: "source.items[asd]", + wantErr: "failed to calc index for []string; err: index should be integer or one of [next, rand, last], but got `asd`", + }, + { + name: "not last segment items in path for []string", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []string{ + "1", + "2", + }, + }, + }, + v: "source.items[0].key", + wantErr: "not last segment items in path source.items[0].key", + }, + { + name: "extract value from array / by index / success", reqMap: map[string]any{ "source": map[string]any{ "items": []any{11, 22, 33}, }, }, - v: "source.items[0]", - want: 11, - wantErr: false, + v: "source.items[0]", + want: 11, + }, + { + name: "slice of ints", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []int{11, 22, 33}, + }, + }, + v: "source.items[0]", + want: 11, + }, + { + name: "slice of ints", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []int64{11, 22, 33}, + }, + }, + v: "source.items[0]", + want: int64(11), + }, + { + name: "slice of ints", + reqMap: map[string]any{ + "source": map[string]any{ + "items": []float64{11, 22, 33}, + }, + }, + v: "source.items[0]", + want: float64(11), }, { - name: "", + name: "invalid key / in array / element / return error", reqMap: map[string]any{ "source": map[string]any{ "items": []any{11, 22, 33}, @@ -108,15 +291,15 @@ func TestGetMapValue(t *testing.T) { }, v: "source.items[0].id", want: nil, - wantErr: true, + wantErr: "not last segment items in path source.items[0].id", }, } - for _, tt := range tests { + for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { iter := NewNextIterator(0) got, err := GetMapValue(tt.reqMap, tt.v, iter) - if tt.wantErr { - require.Error(t, err) + if tt.wantErr != "" { + require.Contains(t, err.Error(), tt.wantErr) return } require.NoError(t, err) @@ -181,3 +364,115 @@ func Test_getValue_iterators(t *testing.T) { assert.Equal(t, 22, got) } + +var tmpJSON = `{ + "name": "John Doe", + "age": 30, + "isStudent": false, + "address": { + "street": "Main St", + "city": "New York", + "state": "NY", + "postalCode": "10001", + "geolocation": { + "lat": 40.7128, + "lng": 74.0060 + } + }, + "rates": [1, 2, 3], + "rates2": [1, "c", 3], + "emails": [ + "johndoe@gmail.com", + "johndoe@yahoo.com" + ], + "courses": [ + { + "courseName": "Math", + "grade": "A+" + }, + { + "courseName": "History", + "grade": "B" + } + ], + "skills": [ + "Programming", + "Design", + "Management" + ], + "projects": [ + { + "title": "Restaurant Web App", + "description": "A full stack web application for managing restaurant reservations.", + "technologies": [ + "HTML", + "CSS", + "JavaScript", + "React", + "Node.js" + ] + }, + { + "title": "E-commerce Website", + "description": "An e-commerce platform for a small business.", + "technologies": [ + "PHP", + "MySQL", + "CSS", + "Bootstrap" + ] + } + ] +}` + +var tmpJSONKeys = []string{ + "name", + "age", + "isStudent", + "address.street", + "address.city", + "address.state", + "address.postalCode", + "address.geolocation.lat", + "address.geolocation.lng", + "rates[0]", + "rates[1]", + "rates[2]", + "rates2[0]", + "rates2[1]", + "rates2[2]", + "emails[0]", + "emails[1]", + "courses[0].courseName", + "courses[0].grade", + "courses[1].courseName", + "courses[1].grade", + "skills[0]", + "skills[1]", + "skills[2]", + "projects[0].title", + "projects[0].description", + "projects[0].technologies[next]", + "projects[0].technologies[next]", + "projects[0].technologies[next]", + "projects[1].title", + "projects[1].description", + "projects[1].technologies[next]", + "projects[1].technologies[next]", + "projects[1].technologies[next]", +} + +func BenchmarkGetMapValue(b *testing.B) { + var data map[string]any + err := json.Unmarshal([]byte(tmpJSON), &data) + require.NoError(b, err) + iterator := NewNextIterator(0) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, k := range tmpJSONKeys { + _, err := GetMapValue(data, k, iterator) + require.NoError(b, err) + } + } +}