From 19b689507ad425f8ad7d4bd0d095286a1b121a46 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 5 Feb 2020 12:39:41 +0800 Subject: [PATCH 01/64] add two args --- drainer/config.go | 20 ++++++++++++++++++++ drainer/config_test.go | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/drainer/config.go b/drainer/config.go index 973d69b8c..047eccdd5 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -80,6 +80,8 @@ type SyncerConfig struct { EnableDispatch bool `toml:"enable-dispatch" json:"enable-dispatch"` SafeMode bool `toml:"safe-mode" json:"safe-mode"` EnableCausality bool `toml:"enable-detect" json:"enable-detect"` + PluginPath string `toml:"plugin-path" json:"plugin-path"` + PluginNames []string `toml:"plugin-names" json:"plugin-names"` } // Config holds the configuration of drainer @@ -106,6 +108,22 @@ type Config struct { tls *tls.Config } +type sliceNames []string + +func newSliceNames(vals []string, p *[]string) *sliceNames { + *p = vals + return (*sliceNames)(p) +} + +func (s *sliceNames) Set(val string) error { + *s = sliceNames(strings.Split(val, ",")) + return nil +} + +func (s *sliceNames) Get() interface{} { return []string(*s) } + +func (s *sliceNames) String() string { return strings.Join([]string(*s), ",") } + // NewConfig return an instance of configuration func NewConfig() *Config { cfg := &Config{ @@ -148,6 +166,8 @@ func NewConfig() *Config { fs.IntVar(&maxBinlogItemCount, "cache-binlog-count", defaultBinlogItemCount, "blurry count of binlogs in cache, limit cache size") fs.IntVar(&cfg.SyncedCheckTime, "synced-check-time", defaultSyncedCheckTime, "if we can't detect new binlog after many minute, we think the all binlog is all synced") fs.StringVar(new(string), "log-rotate", "", "DEPRECATED") + fs.StringVar(&cfg.SyncerCfg.PluginPath, "plugin-path", "", "The path of the plugins") + fs.Var(newSliceNames([]string{}, &cfg.SyncerCfg.PluginNames), "plugin-names", "The names of the plugins") return cfg } diff --git a/drainer/config_test.go b/drainer/config_test.go index 93a4dd378..22540044a 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -55,6 +55,8 @@ func (t *testDrainerSuite) TestConfig(c *C) { "-config", "../cmd/drainer/drainer.toml", "-addr", "192.168.15.10:8257", "-advertise-addr", "192.168.15.10:8257", + "-plugin-path", "/opt/drainer/plugin", + "-plugin-names", "p1, p2", } cfg := NewConfig() @@ -68,6 +70,8 @@ func (t *testDrainerSuite) TestConfig(c *C) { var strSQLMode *string c.Assert(cfg.SyncerCfg.StrSQLMode, Equals, strSQLMode) c.Assert(cfg.SyncerCfg.SQLMode, Equals, mysql.SQLMode(0)) + c.Assert(cfg.SyncerCfg.PluginPath, Equals, "/opt/drainer/plugin") + c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 2) } func (t *testDrainerSuite) TestValidateFilter(c *C) { From 4e0a4de0a0f2b36937459121ab77c786bc6da325 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 5 Feb 2020 21:16:35 +0800 Subject: [PATCH 02/64] support plugin --- drainer/loopbacksync/loopbacksync.go | 6 +- drainer/loopbacksync/loopbacksync_test.go | 2 +- drainer/syncer.go | 2 +- pkg/loader/handler.go | 5 ++ pkg/loader/handlerlist.go | 91 +++++++++++++++++++++++ pkg/loader/load.go | 29 +++++++- pkg/loader/plugin.go | 5 ++ pkg/loader/pluginlist.go | 36 +++++++++ 8 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 pkg/loader/handler.go create mode 100644 pkg/loader/handlerlist.go create mode 100644 pkg/loader/plugin.go create mode 100644 pkg/loader/pluginlist.go diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index 9960bb180..24bb78f24 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -29,14 +29,18 @@ type LoopBackSync struct { ChannelID int64 LoopbackControl bool SyncDDL bool + PluginPath string + PluginNames []string } //NewLoopBackSyncInfo return LoopBackSyncInfo objec -func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool) *LoopBackSync { +func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path string, names []string) *LoopBackSync { l := &LoopBackSync{ ChannelID: ChannelID, LoopbackControl: LoopbackControl, SyncDDL: SyncDDL, + PluginPath: path, + PluginNames: names, } return l } diff --git a/drainer/loopbacksync/loopbacksync_test.go b/drainer/loopbacksync/loopbacksync_test.go index 7306da0f3..55c565613 100644 --- a/drainer/loopbacksync/loopbacksync_test.go +++ b/drainer/loopbacksync/loopbacksync_test.go @@ -20,7 +20,7 @@ func TestNewLoopBackSyncInfo(t *testing.T) { var ChannelID int64 = 1 var LoopbackControl = true var SyncDDL = false - l := NewLoopBackSyncInfo(ChannelID, LoopbackControl, SyncDDL) + l := NewLoopBackSyncInfo(ChannelID, LoopbackControl, SyncDDL, "", nil) if l == nil { t.Error("alloc loopBackSyncInfo objec failed ") } diff --git a/drainer/syncer.go b/drainer/syncer.go index cecf1f47c..c8d87a983 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -77,7 +77,7 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( ignoreDBs = strings.Split(cfg.IgnoreSchemas, ",") } syncer.filter = filter.NewFilter(ignoreDBs, cfg.IgnoreTables, cfg.DoDBs, cfg.DoTables) - syncer.loopbackSync = loopbacksync.NewLoopBackSyncInfo(cfg.ChannelID, cfg.LoopbackControl, cfg.SyncDDL) + syncer.loopbackSync = loopbacksync.NewLoopBackSyncInfo(cfg.ChannelID, cfg.LoopbackControl, cfg.SyncDDL, cfg.PluginPath, cfg.PluginNames) var err error // create schema diff --git a/pkg/loader/handler.go b/pkg/loader/handler.go new file mode 100644 index 000000000..77f9d617f --- /dev/null +++ b/pkg/loader/handler.go @@ -0,0 +1,5 @@ +package loader + +type Handler interface { + NewPlugin() Plugin +} diff --git a/pkg/loader/handlerlist.go b/pkg/loader/handlerlist.go new file mode 100644 index 000000000..510b741d6 --- /dev/null +++ b/pkg/loader/handlerlist.go @@ -0,0 +1,91 @@ +package loader + +import ( + "errors" + "fmt" + "plugin" + "sync" +) + +const FactorFunc = "DoFilter" + +type HandlerList struct { + mp sync.Map +} + +func (hl *HandlerList) GetHandlerList() sync.Map { + return hl.mp +} + +func (hl *HandlerList) SetHandler(name string, h Handler) *HandlerList { + if len(name) == 0 || h == nil { + return hl + } + hl.mp.Store(name, h) + return hl +} + +func (hl *HandlerList) DeleteHandler(name string) { + hl.mp.Delete(name) +} + +func (hl *HandlerList) GetAllHandlersName() []string { + var ns []string = make([]string, 0) + hl.mp.Range(func(k, v interface{}) bool { + name, ok := k.(string) + if !ok { + return true + } + ns = append(ns, name) + return true + }) + return ns +} + +func (hl *HandlerList) GetAllPlugins() *PluginList { + pl := PluginList{} + hl.mp.Range(func(k, v interface{}) bool { + h, ok := v.(Handler) + if !ok { + return true + } + n, ok := k.(string) + if !ok { + return true + } + pl.SetPlugin(n, h.NewPlugin()) + return true + }) + return &pl +} + +func (hl *HandlerList) loadHandlerByName(path, name string) error { + fp := path + "/" + name + p, err := plugin.Open(fp) + if err != nil { + return errors.New(fmt.Sprintf("Open %s failed, err: %s", fp, err.Error())) + } + f, err := p.Lookup(FactorFunc) + if err != nil { + return errors.New(fmt.Sprint("Lookup %s faled", FactorFunc)) + } + newHandler, ok := f.(func() Handler) + if !ok { + return errors.New(fmt.Sprint("Type of %s is incorrect", FactorFunc)) + } + hr := newHandler() + hl.SetHandler(name, hr) + return nil +} + +func (hl *HandlerList) LoadHandlerByNames(path string, names []string) error { + if path == "" || names == nil || len(names) == 0 { + return nil + } + for _, name := range names { + if err := hl.loadHandlerByName(path, name); err != nil { + return err + } + } + return nil +} diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 465909174..bc5cd82f4 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -22,10 +22,9 @@ import ( "sync/atomic" "time" - "github.com/pingcap/tidb-binlog/drainer/loopbacksync" - "github.com/pingcap/errors" "github.com/pingcap/log" + "github.com/pingcap/tidb-binlog/drainer/loopbacksync" "github.com/pingcap/tidb-binlog/pkg/util" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -81,6 +80,8 @@ type loaderImpl struct { metrics *MetricsGroup + handleList HandlerList + // change update -> delete + replace // insert -> replace safeMode int32 @@ -197,6 +198,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { successTxn: make(chan *Txn), merge: true, saveAppliedTS: opts.saveAppliedTS, + handleList: HandlerList{}, ctx: ctx, cancel: cancel, @@ -205,6 +207,10 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { db.SetMaxOpenConns(opts.workerCount) db.SetMaxIdleConns(opts.workerCount) + err := s.handleList.LoadHandlerByNames(s.loopBackSyncInfo.PluginPath, s.loopBackSyncInfo.PluginNames) + if err != nil { + return nil, err + } return s, nil } @@ -523,6 +529,7 @@ func (s *loaderImpl) Run() error { input := txnManager.run() for { + pl := s.handleList.mp select { case txn, ok := <-input: if !ok { @@ -532,7 +539,14 @@ func (s *loaderImpl) Run() error { } return nil } - + (&pl).Range(func(k, v interface{}) bool { + p, ok := v.(Plugin) + if !ok { + return true + } + txn = p.DoFilter(txn) + return true + }) s.metricsInputTxn(txn) txnManager.pop(txn) if err := batch.put(txn); err != nil { @@ -555,6 +569,15 @@ func (s *loaderImpl) Run() error { return nil } + (&pl).Range(func(k, v interface{}) bool { + p, ok := v.(Plugin) + if !ok { + return true + } + txn = p.DoFilter(txn) + return true + }) + s.metricsInputTxn(txn) txnManager.pop(txn) if err := batch.put(txn); err != nil { diff --git a/pkg/loader/plugin.go b/pkg/loader/plugin.go new file mode 100644 index 000000000..6d8f12216 --- /dev/null +++ b/pkg/loader/plugin.go @@ -0,0 +1,5 @@ +package loader + +type Plugin interface { + DoFilter(*Txn) *Txn +} diff --git a/pkg/loader/pluginlist.go b/pkg/loader/pluginlist.go new file mode 100644 index 000000000..09e0c5fe3 --- /dev/null +++ b/pkg/loader/pluginlist.go @@ -0,0 +1,36 @@ +package loader + +import "sync" + +type PluginList struct { + mp sync.Map +} + +func (pl *PluginList) GetPluginList() sync.Map { + return pl.mp +} + +func (pl *PluginList) SetPlugin(name string, p Plugin) *PluginList { + if len(name) == 0 || p == nil { + return pl + } + pl.mp.Store(name, p) + return pl +} + +func (pl *PluginList) DeletePlugin(name string) { + pl.mp.Delete(name) +} + +func (lp *PluginList) GetAllPluginsName() []string { + var ns []string = make([]string, 0) + lp.mp.Range(func(k, v interface{}) bool { + name, ok := k.(string) + if !ok { + return true + } + ns = append(ns, name) + return true + }) + return ns +} From 2cc57b8ec15f2afa1517559d98fa2db0f02da95a Mon Sep 17 00:00:00 2001 From: tsthght Date: Fri, 7 Feb 2020 17:53:43 +0800 Subject: [PATCH 03/64] refine --- pkg/loader/load.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/loader/load.go b/pkg/loader/load.go index bc5cd82f4..29dcc6e4e 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -529,7 +529,7 @@ func (s *loaderImpl) Run() error { input := txnManager.run() for { - pl := s.handleList.mp + pl := s.handleList.GetAllPlugins().mp select { case txn, ok := <-input: if !ok { @@ -542,6 +542,7 @@ func (s *loaderImpl) Run() error { (&pl).Range(func(k, v interface{}) bool { p, ok := v.(Plugin) if !ok { + //todo log return true } txn = p.DoFilter(txn) From b5831088852064f561c7bf34d1dc27103c8b714d Mon Sep 17 00:00:00 2001 From: tsthght Date: Fri, 7 Feb 2020 18:34:07 +0800 Subject: [PATCH 04/64] optimize the plugin management structure --- pkg/loader/handler.go | 5 --- pkg/loader/handlerlist.go | 91 --------------------------------------- pkg/loader/load.go | 10 +++-- pkg/loader/pluginlist.go | 40 ++++++++++++++++- 4 files changed, 45 insertions(+), 101 deletions(-) delete mode 100644 pkg/loader/handler.go delete mode 100644 pkg/loader/handlerlist.go diff --git a/pkg/loader/handler.go b/pkg/loader/handler.go deleted file mode 100644 index 77f9d617f..000000000 --- a/pkg/loader/handler.go +++ /dev/null @@ -1,5 +0,0 @@ -package loader - -type Handler interface { - NewPlugin() Plugin -} diff --git a/pkg/loader/handlerlist.go b/pkg/loader/handlerlist.go deleted file mode 100644 index 510b741d6..000000000 --- a/pkg/loader/handlerlist.go +++ /dev/null @@ -1,91 +0,0 @@ -package loader - -import ( - "errors" - "fmt" - "plugin" - "sync" -) - -const FactorFunc = "DoFilter" - -type HandlerList struct { - mp sync.Map -} - -func (hl *HandlerList) GetHandlerList() sync.Map { - return hl.mp -} - -func (hl *HandlerList) SetHandler(name string, h Handler) *HandlerList { - if len(name) == 0 || h == nil { - return hl - } - hl.mp.Store(name, h) - return hl -} - -func (hl *HandlerList) DeleteHandler(name string) { - hl.mp.Delete(name) -} - -func (hl *HandlerList) GetAllHandlersName() []string { - var ns []string = make([]string, 0) - hl.mp.Range(func(k, v interface{}) bool { - name, ok := k.(string) - if !ok { - return true - } - ns = append(ns, name) - return true - }) - return ns -} - -func (hl *HandlerList) GetAllPlugins() *PluginList { - pl := PluginList{} - hl.mp.Range(func(k, v interface{}) bool { - h, ok := v.(Handler) - if !ok { - return true - } - n, ok := k.(string) - if !ok { - return true - } - pl.SetPlugin(n, h.NewPlugin()) - return true - }) - return &pl -} - -func (hl *HandlerList) loadHandlerByName(path, name string) error { - fp := path + "/" + name - p, err := plugin.Open(fp) - if err != nil { - return errors.New(fmt.Sprintf("Open %s failed, err: %s", fp, err.Error())) - } - f, err := p.Lookup(FactorFunc) - if err != nil { - return errors.New(fmt.Sprint("Lookup %s faled", FactorFunc)) - } - newHandler, ok := f.(func() Handler) - if !ok { - return errors.New(fmt.Sprint("Type of %s is incorrect", FactorFunc)) - } - hr := newHandler() - hl.SetHandler(name, hr) - return nil -} - -func (hl *HandlerList) LoadHandlerByNames(path string, names []string) error { - if path == "" || names == nil || len(names) == 0 { - return nil - } - for _, name := range names { - if err := hl.loadHandlerByName(path, name); err != nil { - return err - } - } - return nil -} diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 29dcc6e4e..f7db9a772 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -80,7 +80,7 @@ type loaderImpl struct { metrics *MetricsGroup - handleList HandlerList + pluginList PluginList // change update -> delete + replace // insert -> replace @@ -198,7 +198,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { successTxn: make(chan *Txn), merge: true, saveAppliedTS: opts.saveAppliedTS, - handleList: HandlerList{}, + pluginList: PluginList{}, ctx: ctx, cancel: cancel, @@ -207,7 +207,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { db.SetMaxOpenConns(opts.workerCount) db.SetMaxIdleConns(opts.workerCount) - err := s.handleList.LoadHandlerByNames(s.loopBackSyncInfo.PluginPath, s.loopBackSyncInfo.PluginNames) + err := s.pluginList.LoadPluginByNames(s.loopBackSyncInfo.PluginPath, s.loopBackSyncInfo.PluginNames) if err != nil { return nil, err } @@ -528,8 +528,10 @@ func (s *loaderImpl) Run() error { batch := fNewBatchManager(s) input := txnManager.run() + pl := s.pluginList.GetPluginList() + for { - pl := s.handleList.GetAllPlugins().mp + select { case txn, ok := <-input: if !ok { diff --git a/pkg/loader/pluginlist.go b/pkg/loader/pluginlist.go index 09e0c5fe3..9e3c49dae 100644 --- a/pkg/loader/pluginlist.go +++ b/pkg/loader/pluginlist.go @@ -1,6 +1,13 @@ package loader -import "sync" +import ( + "errors" + "fmt" + "plugin" + "sync" +) + +const FactorFunc = "NewPlugin" type PluginList struct { mp sync.Map @@ -34,3 +41,34 @@ func (lp *PluginList) GetAllPluginsName() []string { }) return ns } + +func (lp *PluginList) loadPluginByName(path, name string) error { + fp := path + "/" + name + p, err := plugin.Open(fp) + if err != nil { + return errors.New(fmt.Sprintf("Open %s failed, err: %s", fp, err.Error())) + } + f, err := p.Lookup(FactorFunc) + if err != nil { + return errors.New(fmt.Sprint("Lookup %s faled", FactorFunc)) + } + newPlugin, ok := f.(func() Plugin) + if !ok { + return errors.New(fmt.Sprint("Type of %s is incorrect", FactorFunc)) + } + pr := newPlugin() + lp.SetPlugin(name, pr) + return nil +} + +func (lp *PluginList) LoadPluginByNames(path string, names []string) error { + if path == "" || names == nil || len(names) == 0 { + return nil + } + for _, name := range names { + if err := lp.loadPluginByName(path, name); err != nil { + return err + } + } + return nil +} From 76c6be610946c236a67b192748c351e82c668c85 Mon Sep 17 00:00:00 2001 From: tsthght Date: Tue, 25 Feb 2020 09:36:34 +0800 Subject: [PATCH 05/64] modify plugin framework --- drainer/ilookback.go | 10 ++++ drainer/loopbacksync/loopbacksync.go | 3 ++ drainer/syncer.go | 44 +++++++++++++++-- go.mod | 1 + pkg/Hooks/test.go | 17 +++++++ pkg/loader/executor.go | 36 +++++++++----- pkg/loader/iloopback.go | 7 +++ pkg/loader/load.go | 40 ++++++--------- pkg/loader/plugin.go | 5 -- pkg/loader/pluginlist.go | 74 ---------------------------- pkg/plugin/plugins.go | 62 +++++++++++++++++++++++ pkg/plugin/plugins_test.go | 26 ++++++++++ 12 files changed, 206 insertions(+), 119 deletions(-) create mode 100644 drainer/ilookback.go create mode 100644 pkg/Hooks/test.go create mode 100644 pkg/loader/iloopback.go delete mode 100644 pkg/loader/plugin.go delete mode 100644 pkg/loader/pluginlist.go create mode 100644 pkg/plugin/plugins.go create mode 100644 pkg/plugin/plugins_test.go diff --git a/drainer/ilookback.go b/drainer/ilookback.go new file mode 100644 index 000000000..802c7d8bb --- /dev/null +++ b/drainer/ilookback.go @@ -0,0 +1,10 @@ +package drainer + +import ( + "github.com/pingcap/tidb-binlog/drainer/loopbacksync" + "github.com/pingcap/tidb-binlog/pkg/loader" +) + +type LoopBack interface { + FilterMarkTable(DMLs []*loader.DML, info *loopbacksync.LoopBackSync) (bool, error) +} diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index 24bb78f24..f1a910985 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -13,6 +13,8 @@ package loopbacksync +import "github.com/pingcap/tidb-binlog/pkg/plugin" + const ( //MarkTableName mark table name MarkTableName = "retl._drainer_repl_mark" @@ -31,6 +33,7 @@ type LoopBackSync struct { SyncDDL bool PluginPath string PluginNames []string + Hooks []*plugin.EventHooks } //NewLoopBackSyncInfo return LoopBackSyncInfo objec diff --git a/drainer/syncer.go b/drainer/syncer.go index c8d87a983..0f2cb756a 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -19,12 +19,12 @@ import ( "sync/atomic" "time" - "github.com/pingcap/tidb-binlog/drainer/loopbacksync" - "github.com/pingcap/tidb-binlog/pkg/loader" - "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/parser/model" + "github.com/pingcap/tidb-binlog/drainer/loopbacksync" + "github.com/pingcap/tidb-binlog/pkg/loader" + "github.com/pingcap/tidb-binlog/pkg/plugin" "go.uber.org/zap" "github.com/pingcap/tidb-binlog/drainer/checkpoint" @@ -78,6 +78,19 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( } syncer.filter = filter.NewFilter(ignoreDBs, cfg.IgnoreTables, cfg.DoDBs, cfg.DoTables) syncer.loopbackSync = loopbacksync.NewLoopBackSyncInfo(cfg.ChannelID, cfg.LoopbackControl, cfg.SyncDDL, cfg.PluginPath, cfg.PluginNames) + for _, name := range syncer.loopbackSync.PluginNames { + sym, err := plugin.LoadPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], + syncer.loopbackSync.PluginPath, name) + if err != nil { + return nil, err + } + newPlugin, ok := sym.(func() LoopBack) + if !ok { + continue + } + plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], + name, newPlugin()) + } var err error // create schema @@ -358,8 +371,33 @@ ForLoop: err = errors.Annotate(err, "handlePreviousDDLJobIfNeed failed") break ForLoop } + + var isErr = false var isFilterTransaction = false var err1 error + var txn *loader.Txn + txn, err1 = translator.TiBinlogToTxn(s.schema, "", "", binlog, preWrite) + if err1 != nil { + err = errors.Annotate(err1, "analyze transaction failed") + break ForLoop + } + hook := s.loopbackSync.Hooks[plugin.SyncerPlugin] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoopBack) + if !ok { + isErr = true + return false + } + isFilterTransaction, err1 = c.FilterMarkTable(txn.DMLs, s.loopbackSync) + if err1 != nil || isFilterTransaction { + return false + } + return true + }) + if err1 != nil { + err = errors.Annotate(err, "filterTable failed") + break ForLoop } + if s.loopbackSync != nil && s.loopbackSync.LoopbackControl { isFilterTransaction, err1 = loopBackStatus(binlog, preWrite, s.schema, s.loopbackSync) if err1 != nil { diff --git a/go.mod b/go.mod index 8d7039ec9..ece08e796 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/sirupsen/logrus v1.4.1 // indirect github.com/soheilhy/cmux v0.1.4 github.com/spf13/pflag v1.0.3 // indirect + github.com/stretchr/testify v1.4.0 github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d diff --git a/pkg/Hooks/test.go b/pkg/Hooks/test.go new file mode 100644 index 000000000..4dca6b56e --- /dev/null +++ b/pkg/Hooks/test.go @@ -0,0 +1,17 @@ +package main + +import ( + "github.com/pingcap/tidb-binlog/drainer/loopbacksync" + "github.com/pingcap/tidb-binlog/pkg/loader" +) + +type PluginDemo struct {} +func (pd PluginDemo)UpdateMarkTable(tx *loader.Tx, info *loopbacksync.LoopBackSync) *loader.Tx { + return nil +} + +func NewPlugin() loader.LoopBack { + return PluginDemo{} +} + + diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index 4587e34ed..a084ea4d2 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -20,10 +20,10 @@ import ( "strings" "time" - "github.com/pingcap/tidb-binlog/drainer/loopbacksync" - "github.com/pingcap/errors" "github.com/pingcap/log" + "github.com/pingcap/tidb-binlog/drainer/loopbacksync" + "github.com/pingcap/tidb-binlog/pkg/plugin" "github.com/pingcap/tidb-binlog/pkg/util" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -44,7 +44,6 @@ func newExecutor(db *gosql.DB) *executor { db: db, batchSize: defaultBatchSize, } - return exe } @@ -70,13 +69,13 @@ func (e *executor) execTableBatchRetry(ctx context.Context, dmls []*DML, retryNu } // a wrap of *sql.Tx with metrics -type tx struct { +type Tx struct { *gosql.Tx queryHistogramVec *prometheus.HistogramVec } // wrap of sql.Tx.Exec() -func (tx *tx) exec(query string, args ...interface{}) (gosql.Result, error) { +func (tx *Tx) exec(query string, args ...interface{}) (gosql.Result, error) { start := time.Now() res, err := tx.Tx.Exec(query, args...) if tx.queryHistogramVec != nil { @@ -86,7 +85,7 @@ func (tx *tx) exec(query string, args ...interface{}) (gosql.Result, error) { return res, err } -func (tx *tx) autoRollbackExec(query string, args ...interface{}) (res gosql.Result, err error) { +func (tx *Tx) autoRollbackExec(query string, args ...interface{}) (res gosql.Result, err error) { res, err = tx.exec(query, args...) if err != nil { log.Error("Exec fail, will rollback", zap.String("query", query), zap.Reflect("args", args), zap.Error(err)) @@ -99,7 +98,7 @@ func (tx *tx) autoRollbackExec(query string, args ...interface{}) (res gosql.Res } // wrap of sql.Tx.Commit() -func (tx *tx) commit() error { +func (tx *Tx) commit() error { start := time.Now() err := tx.Tx.Commit() if tx.queryHistogramVec != nil { @@ -109,7 +108,7 @@ func (tx *tx) commit() error { return errors.Trace(err) } -func (e *executor) updateMark(channel string, tx *tx) error { +func (e *executor) updateMark(channel string, tx *Tx) error { if e.info == nil { return nil } @@ -126,24 +125,39 @@ func (e *executor) updateMark(channel string, tx *tx) error { } // return a wrap of sql.Tx -func (e *executor) begin() (*tx, error) { +func (e *executor) begin() (*Tx, error) { sqlTx, err := e.db.Begin() if err != nil { return nil, errors.Trace(err) } - var tx = &tx{ + var tx = &Tx{ Tx: sqlTx, queryHistogramVec: e.queryHistogramVec, } + isErr := false + hook := e.info.Hooks[plugin.LoaderPlugin] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoopBack) + if !ok { + isErr = true + return false + } + tx = c.UpdateMarkTable(tx, e.info) + return true + }) + if isErr { + return nil, errors.New("type is incorrect") + } + /* if e.info != nil && e.info.LoopbackControl { err1 := e.updateMark("", tx) if err1 != nil { return nil, errors.Trace(err1) } } - + */ return tx, nil } diff --git a/pkg/loader/iloopback.go b/pkg/loader/iloopback.go new file mode 100644 index 000000000..64004fe37 --- /dev/null +++ b/pkg/loader/iloopback.go @@ -0,0 +1,7 @@ +package loader + +import "github.com/pingcap/tidb-binlog/drainer/loopbacksync" + +type LoopBack interface { + UpdateMarkTable(tx *Tx, info *loopbacksync.LoopBackSync) *Tx +} diff --git a/pkg/loader/load.go b/pkg/loader/load.go index f7db9a772..ffdf95192 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/tidb-binlog/drainer/loopbacksync" + "github.com/pingcap/tidb-binlog/pkg/plugin" "github.com/pingcap/tidb-binlog/pkg/util" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -80,8 +81,6 @@ type loaderImpl struct { metrics *MetricsGroup - pluginList PluginList - // change update -> delete + replace // insert -> replace safeMode int32 @@ -198,7 +197,6 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { successTxn: make(chan *Txn), merge: true, saveAppliedTS: opts.saveAppliedTS, - pluginList: PluginList{}, ctx: ctx, cancel: cancel, @@ -207,10 +205,20 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { db.SetMaxOpenConns(opts.workerCount) db.SetMaxIdleConns(opts.workerCount) - err := s.pluginList.LoadPluginByNames(s.loopBackSyncInfo.PluginPath, s.loopBackSyncInfo.PluginNames) - if err != nil { - return nil, err + for _, name := range s.loopBackSyncInfo.PluginNames { + sym, err := plugin.LoadPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], + s.loopBackSyncInfo.PluginPath, name) + if err != nil { + return nil, err + } + newPlugin, ok := sym.(func() LoopBack) + if !ok { + continue + } + plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], + name, newPlugin()) } + return s, nil } @@ -528,8 +536,6 @@ func (s *loaderImpl) Run() error { batch := fNewBatchManager(s) input := txnManager.run() - pl := s.pluginList.GetPluginList() - for { select { @@ -541,15 +547,6 @@ func (s *loaderImpl) Run() error { } return nil } - (&pl).Range(func(k, v interface{}) bool { - p, ok := v.(Plugin) - if !ok { - //todo log - return true - } - txn = p.DoFilter(txn) - return true - }) s.metricsInputTxn(txn) txnManager.pop(txn) if err := batch.put(txn); err != nil { @@ -572,15 +569,6 @@ func (s *loaderImpl) Run() error { return nil } - (&pl).Range(func(k, v interface{}) bool { - p, ok := v.(Plugin) - if !ok { - return true - } - txn = p.DoFilter(txn) - return true - }) - s.metricsInputTxn(txn) txnManager.pop(txn) if err := batch.put(txn); err != nil { diff --git a/pkg/loader/plugin.go b/pkg/loader/plugin.go deleted file mode 100644 index 6d8f12216..000000000 --- a/pkg/loader/plugin.go +++ /dev/null @@ -1,5 +0,0 @@ -package loader - -type Plugin interface { - DoFilter(*Txn) *Txn -} diff --git a/pkg/loader/pluginlist.go b/pkg/loader/pluginlist.go deleted file mode 100644 index 9e3c49dae..000000000 --- a/pkg/loader/pluginlist.go +++ /dev/null @@ -1,74 +0,0 @@ -package loader - -import ( - "errors" - "fmt" - "plugin" - "sync" -) - -const FactorFunc = "NewPlugin" - -type PluginList struct { - mp sync.Map -} - -func (pl *PluginList) GetPluginList() sync.Map { - return pl.mp -} - -func (pl *PluginList) SetPlugin(name string, p Plugin) *PluginList { - if len(name) == 0 || p == nil { - return pl - } - pl.mp.Store(name, p) - return pl -} - -func (pl *PluginList) DeletePlugin(name string) { - pl.mp.Delete(name) -} - -func (lp *PluginList) GetAllPluginsName() []string { - var ns []string = make([]string, 0) - lp.mp.Range(func(k, v interface{}) bool { - name, ok := k.(string) - if !ok { - return true - } - ns = append(ns, name) - return true - }) - return ns -} - -func (lp *PluginList) loadPluginByName(path, name string) error { - fp := path + "/" + name - p, err := plugin.Open(fp) - if err != nil { - return errors.New(fmt.Sprintf("Open %s failed, err: %s", fp, err.Error())) - } - f, err := p.Lookup(FactorFunc) - if err != nil { - return errors.New(fmt.Sprint("Lookup %s faled", FactorFunc)) - } - newPlugin, ok := f.(func() Plugin) - if !ok { - return errors.New(fmt.Sprint("Type of %s is incorrect", FactorFunc)) - } - pr := newPlugin() - lp.SetPlugin(name, pr) - return nil -} - -func (lp *PluginList) LoadPluginByNames(path string, names []string) error { - if path == "" || names == nil || len(names) == 0 { - return nil - } - for _, name := range names { - if err := lp.loadPluginByName(path, name); err != nil { - return err - } - } - return nil -} diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go new file mode 100644 index 000000000..8c85a2fc8 --- /dev/null +++ b/pkg/plugin/plugins.go @@ -0,0 +1,62 @@ +package plugin + +import ( + "errors" + "fmt" + "plugin" + "sync" +) + +//Plugin type we supported currently +type Kind uint8 +const ( + SyncerPlugin Kind = iota + LoaderPlugin + + FactorFunc = "NewPlugin" +) + +//EventHooks is a map of hook name to hook +type EventHooks struct { + sync.Map +} + +func (ehs *EventHooks) SetPlugin(name string, plg interface{}) *EventHooks { + if len(name) == 0 || ehs == nil { + return ehs + } + ehs.Store(name, plg) + return ehs +} + +func (ehs *EventHooks) GetAllPluginsName() []string { + if ehs == nil { + return nil + } + var ns []string = make([]string, 0) + ehs.Range(func(k, v interface{}) bool { + name, ok := k.(string) + if !ok { + return true + } + ns = append(ns, name) + return true + }) + return ns +} + +//LoadPlugin() can load plugin by plugin's name +func LoadPlugin(eh *EventHooks, path, name string) (plugin.Symbol, error) { + fp := path + "/" + name + p, err := plugin.Open(fp) + if err != nil { + return nil, errors.New(fmt.Sprintf("Open %s failed, err: %s", fp, err.Error())) + } + + return p.Lookup(FactorFunc) +} + +//RegisterPlugin() register plugin to EventHooks +func RegisterPlugin(ehs *EventHooks, name string, plg interface{}) { + ehs.SetPlugin(name, plg) +} \ No newline at end of file diff --git a/pkg/plugin/plugins_test.go b/pkg/plugin/plugins_test.go new file mode 100644 index 000000000..f8d80385d --- /dev/null +++ b/pkg/plugin/plugins_test.go @@ -0,0 +1,26 @@ +package plugin + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type ITest1 interface { + Do() int +} +type STest1 struct { + a int +} +func (s *STest1) Do () int { + return s.a +} + +func TestRegisterPlugin(t *testing.T) { + hook := &EventHooks{} + s := STest1{32} + + RegisterPlugin(hook, "test1", s) + p := hook.GetAllPluginsName() + assert.Equal(t, 1, len(p)) +} From 27989e26d157cbc51813cb57624c861684b47cfc Mon Sep 17 00:00:00 2001 From: tsthght Date: Tue, 25 Feb 2020 16:53:09 +0800 Subject: [PATCH 06/64] modify plugin framework --- drainer/config.go | 2 + drainer/{ilookback.go => filter_txn.go} | 2 +- drainer/loopbacksync/loopbacksync.go | 4 +- drainer/loopbacksync/loopbacksync_test.go | 2 +- drainer/syncer.go | 254 +++++++++++---------- pkg/Hooks/test.go | 8 +- pkg/loader/executor.go | 39 ++-- pkg/loader/{iloopback.go => extend_txn.go} | 2 +- pkg/loader/load.go | 24 +- 9 files changed, 184 insertions(+), 153 deletions(-) rename drainer/{ilookback.go => filter_txn.go} (65%) rename pkg/loader/{iloopback.go => extend_txn.go} (63%) diff --git a/drainer/config.go b/drainer/config.go index 047eccdd5..a222ec006 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -82,6 +82,7 @@ type SyncerConfig struct { EnableCausality bool `toml:"enable-detect" json:"enable-detect"` PluginPath string `toml:"plugin-path" json:"plugin-path"` PluginNames []string `toml:"plugin-names" json:"plugin-names"` + SupportPlugin bool `toml:"support-plugin" json:"support-plugin"` } // Config holds the configuration of drainer @@ -168,6 +169,7 @@ func NewConfig() *Config { fs.StringVar(new(string), "log-rotate", "", "DEPRECATED") fs.StringVar(&cfg.SyncerCfg.PluginPath, "plugin-path", "", "The path of the plugins") fs.Var(newSliceNames([]string{}, &cfg.SyncerCfg.PluginNames), "plugin-names", "The names of the plugins") + fs.BoolVar(&cfg.SyncerCfg.SupportPlugin, "support-plugin", false, "Whether plugin is supported,default: false") return cfg } diff --git a/drainer/ilookback.go b/drainer/filter_txn.go similarity index 65% rename from drainer/ilookback.go rename to drainer/filter_txn.go index 802c7d8bb..0721d646a 100644 --- a/drainer/ilookback.go +++ b/drainer/filter_txn.go @@ -6,5 +6,5 @@ import ( ) type LoopBack interface { - FilterMarkTable(DMLs []*loader.DML, info *loopbacksync.LoopBackSync) (bool, error) + FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) bool } diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index f1a910985..cd7d2eeee 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -34,16 +34,18 @@ type LoopBackSync struct { PluginPath string PluginNames []string Hooks []*plugin.EventHooks + SupportPlugin bool } //NewLoopBackSyncInfo return LoopBackSyncInfo objec -func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path string, names []string) *LoopBackSync { +func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path string, names []string, SupportPlug bool) *LoopBackSync { l := &LoopBackSync{ ChannelID: ChannelID, LoopbackControl: LoopbackControl, SyncDDL: SyncDDL, PluginPath: path, PluginNames: names, + SupportPlugin: SupportPlug, } return l } diff --git a/drainer/loopbacksync/loopbacksync_test.go b/drainer/loopbacksync/loopbacksync_test.go index 55c565613..ac490953a 100644 --- a/drainer/loopbacksync/loopbacksync_test.go +++ b/drainer/loopbacksync/loopbacksync_test.go @@ -20,7 +20,7 @@ func TestNewLoopBackSyncInfo(t *testing.T) { var ChannelID int64 = 1 var LoopbackControl = true var SyncDDL = false - l := NewLoopBackSyncInfo(ChannelID, LoopbackControl, SyncDDL, "", nil) + l := NewLoopBackSyncInfo(ChannelID, LoopbackControl, SyncDDL, "", nil, false) if l == nil { t.Error("alloc loopBackSyncInfo objec failed ") } diff --git a/drainer/syncer.go b/drainer/syncer.go index 0f2cb756a..b1fcb96c8 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -77,19 +77,21 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( ignoreDBs = strings.Split(cfg.IgnoreSchemas, ",") } syncer.filter = filter.NewFilter(ignoreDBs, cfg.IgnoreTables, cfg.DoDBs, cfg.DoTables) - syncer.loopbackSync = loopbacksync.NewLoopBackSyncInfo(cfg.ChannelID, cfg.LoopbackControl, cfg.SyncDDL, cfg.PluginPath, cfg.PluginNames) - for _, name := range syncer.loopbackSync.PluginNames { - sym, err := plugin.LoadPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], - syncer.loopbackSync.PluginPath, name) - if err != nil { - return nil, err - } - newPlugin, ok := sym.(func() LoopBack) - if !ok { - continue + syncer.loopbackSync = loopbacksync.NewLoopBackSyncInfo(cfg.ChannelID, cfg.LoopbackControl, cfg.SyncDDL, cfg.PluginPath, cfg.PluginNames, cfg.SupportPlugin) + if syncer.loopbackSync.SupportPlugin { + for _, name := range syncer.loopbackSync.PluginNames { + sym, err := plugin.LoadPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], + syncer.loopbackSync.PluginPath, name) + if err != nil { + return nil, err + } + newPlugin, ok := sym.(func() LoopBack) + if !ok { + continue + } + plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], + name, newPlugin()) } - plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], - name, newPlugin()) } var err error @@ -346,135 +348,151 @@ ForLoop: if startTS == commitTS { fakeBinlogs = append(fakeBinlogs, binlog) fakeBinlogPreAddTS = append(fakeBinlogPreAddTS, lastAddComitTS) - } else if jobID == 0 { - preWriteValue := binlog.GetPrewriteValue() - preWrite := &pb.PrewriteValue{} - err = preWrite.Unmarshal(preWriteValue) - if err != nil { - err = errors.Annotatef(err, "prewrite %s Unmarshal failed", preWriteValue) - break ForLoop - } - - err = s.rewriteForOldVersion(preWrite) - if err != nil { - err = errors.Annotate(err, "rewrite for old version fail") - break ForLoop - } - - log.Debug("get DML", zap.Int64("SchemaVersion", preWrite.SchemaVersion)) - if preWrite.SchemaVersion < lastDDLSchemaVersion { - log.Debug("encounter older schema dml") - } + } else { + if s.loopbackSync.SupportPlugin { + schema, table, err := s.schema.getSchemaTableAndDelete(b.job.BinlogInfo.SchemaVersion) + if err != nil { + err = errors.Trace(err) + break ForLoop + } - err = s.schema.handlePreviousDDLJobIfNeed(preWrite.SchemaVersion) - if err != nil { - err = errors.Annotate(err, "handlePreviousDDLJobIfNeed failed") - break ForLoop + preWrite := &pb.PrewriteValue{} + err = preWrite.Unmarshal(binlog.GetPrewriteValue()) + if err != nil { + err = errors.Annotatef(err, "prewrite %s Unmarshal failed", binlog.GetPrewriteValue()) + break ForLoop + } + var txn *loader.Txn + txn, err = translator.TiBinlogToTxn(s.schema, schema, table, binlog, preWrite) + var isFilterTransaction = false + txn, err = translator.TiBinlogToTxn(s.schema, "", "", binlog, preWrite) + if err != nil { + err = errors.Annotate(err, "analyze transaction failed") + break ForLoop + } + hook := s.loopbackSync.Hooks[plugin.SyncerPlugin] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoopBack) + if !ok { + return true + } + isFilterTransaction = c.FilterTxn(txn, s.loopbackSync) + if isFilterTransaction { + return false + } + return true + }) + if isFilterTransaction { + continue + } } - var isErr = false - var isFilterTransaction = false - var err1 error - var txn *loader.Txn - txn, err1 = translator.TiBinlogToTxn(s.schema, "", "", binlog, preWrite) - if err1 != nil { - err = errors.Annotate(err1, "analyze transaction failed") - break ForLoop - } - hook := s.loopbackSync.Hooks[plugin.SyncerPlugin] - hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoopBack) - if !ok { - isErr = true - return false - } - isFilterTransaction, err1 = c.FilterMarkTable(txn.DMLs, s.loopbackSync) - if err1 != nil || isFilterTransaction { - return false + if jobID == 0 { + preWriteValue := binlog.GetPrewriteValue() + preWrite := &pb.PrewriteValue{} + err = preWrite.Unmarshal(preWriteValue) + if err != nil { + err = errors.Annotatef(err, "prewrite %s Unmarshal failed", preWriteValue) + break ForLoop } - return true - }) - if err1 != nil { - err = errors.Annotate(err, "filterTable failed") - break ForLoop } - - if s.loopbackSync != nil && s.loopbackSync.LoopbackControl { - isFilterTransaction, err1 = loopBackStatus(binlog, preWrite, s.schema, s.loopbackSync) - if err1 != nil { - err = errors.Annotate(err1, "analyze transaction failed") + + err = s.rewriteForOldVersion(preWrite) + if err != nil { + err = errors.Annotate(err, "rewrite for old version fail") break ForLoop } - } - var ignore bool - ignore, err = filterTable(preWrite, s.filter, s.schema) - if err != nil { - err = errors.Annotate(err, "filterTable failed") - break ForLoop - } + log.Debug("get DML", zap.Int64("SchemaVersion", preWrite.SchemaVersion)) + if preWrite.SchemaVersion < lastDDLSchemaVersion { + log.Debug("encounter older schema dml") + } - if !ignore && !isFilterTransaction { - s.addDMLEventMetrics(preWrite.GetMutations()) - beginTime := time.Now() - lastAddComitTS = binlog.GetCommitTs() - err = s.dsyncer.Sync(&dsync.Item{Binlog: binlog, PrewriteValue: preWrite}) + err = s.schema.handlePreviousDDLJobIfNeed(preWrite.SchemaVersion) if err != nil { - err = errors.Annotatef(err, "add to dsyncer, commit ts %d", binlog.CommitTs) + err = errors.Annotate(err, "handlePreviousDDLJobIfNeed failed") break ForLoop } - executeHistogram.Observe(time.Since(beginTime).Seconds()) - } - } else if jobID > 0 { - log.Debug("get ddl binlog job", zap.Stringer("job", b.job)) - // Notice: the version of DDL Binlog we receive are Monotonically increasing - // DDL (with version 10, commit ts 100) -> DDL (with version 9, commit ts 101) would never happen - s.schema.addJob(b.job) + var isFilterTransaction = false + var err1 error + if s.loopbackSync != nil && s.loopbackSync.LoopbackControl && !s.loopbackSync.SupportPlugin { + isFilterTransaction, err1 = loopBackStatus(binlog, preWrite, s.schema, s.loopbackSync) + if err1 != nil { + err = errors.Annotate(err1, "analyze transaction failed") + break ForLoop + } + } - if !s.cfg.SyncDDL { - log.Info("Syncer skips DDL", zap.String("sql", b.job.Query), zap.Int64("ts", b.GetCommitTs()), zap.Bool("SyncDDL", s.cfg.SyncDDL)) - continue - } + var ignore bool + ignore, err = filterTable(preWrite, s.filter, s.schema) + if err != nil { + err = errors.Annotate(err, "filterTable failed") + break ForLoop + } - log.Debug("get DDL", zap.Int64("SchemaVersion", b.job.BinlogInfo.SchemaVersion)) - lastDDLSchemaVersion = b.job.BinlogInfo.SchemaVersion + if !ignore && !isFilterTransaction { + s.addDMLEventMetrics(preWrite.GetMutations()) + beginTime := time.Now() + lastAddComitTS = binlog.GetCommitTs() + err = s.dsyncer.Sync(&dsync.Item{Binlog: binlog, PrewriteValue: preWrite}) + if err != nil { + err = errors.Annotatef(err, "add to dsyncer, commit ts %d", binlog.CommitTs) + break ForLoop + } + executeHistogram.Observe(time.Since(beginTime).Seconds()) + } + } else if jobID > 0 { + log.Debug("get ddl binlog job", zap.Stringer("job", b.job)) - err = s.schema.handlePreviousDDLJobIfNeed(b.job.BinlogInfo.SchemaVersion) - if err != nil { - err = errors.Trace(err) - break ForLoop - } + // Notice: the version of DDL Binlog we receive are Monotonically increasing + // DDL (with version 10, commit ts 100) -> DDL (with version 9, commit ts 101) would never happen + s.schema.addJob(b.job) - if b.job.SchemaState == model.StateDeleteOnly && b.job.Type == model.ActionDropColumn { - log.Info("Syncer skips DeleteOnly DDL", zap.Stringer("job", b.job), zap.Int64("ts", b.GetCommitTs())) - continue - } + if !s.cfg.SyncDDL && !s.loopbackSync.SupportPlugin { + log.Info("Syncer skips DDL", zap.String("sql", b.job.Query), zap.Int64("ts", b.GetCommitTs()), zap.Bool("SyncDDL", s.cfg.SyncDDL)) + continue + } - sql := b.job.Query - var schema, table string - schema, table, err = s.schema.getSchemaTableAndDelete(b.job.BinlogInfo.SchemaVersion) - if err != nil { - err = errors.Trace(err) - break ForLoop - } + log.Debug("get DDL", zap.Int64("SchemaVersion", b.job.BinlogInfo.SchemaVersion)) + lastDDLSchemaVersion = b.job.BinlogInfo.SchemaVersion - if s.filter.SkipSchemaAndTable(schema, table) { - log.Info("skip ddl", zap.String("schema", schema), zap.String("table", table), - zap.String("sql", sql), zap.Int64("commit ts", commitTS)) - } else if sql != "" { - s.addDDLCount() - beginTime := time.Now() - lastAddComitTS = binlog.GetCommitTs() + err = s.schema.handlePreviousDDLJobIfNeed(b.job.BinlogInfo.SchemaVersion) + if err != nil { + err = errors.Trace(err) + break ForLoop + } - log.Info("add ddl item to syncer, you can add this commit ts to `ignore-txn-commit-ts` to skip this ddl if needed", - zap.String("sql", sql), zap.Int64("commit ts", binlog.CommitTs)) + if b.job.SchemaState == model.StateDeleteOnly && b.job.Type == model.ActionDropColumn { + log.Info("Syncer skips DeleteOnly DDL", zap.Stringer("job", b.job), zap.Int64("ts", b.GetCommitTs())) + continue + } - err = s.dsyncer.Sync(&dsync.Item{Binlog: binlog, PrewriteValue: nil, Schema: schema, Table: table}) + sql := b.job.Query + var schema, table string + schema, table, err = s.schema.getSchemaTableAndDelete(b.job.BinlogInfo.SchemaVersion) if err != nil { - err = errors.Annotatef(err, "add to dsyncer, commit ts %d", binlog.CommitTs) + err = errors.Trace(err) break ForLoop } - executeHistogram.Observe(time.Since(beginTime).Seconds()) + + if s.filter.SkipSchemaAndTable(schema, table) { + log.Info("skip ddl", zap.String("schema", schema), zap.String("table", table), + zap.String("sql", sql), zap.Int64("commit ts", commitTS)) + } else if sql != "" { + s.addDDLCount() + beginTime := time.Now() + lastAddComitTS = binlog.GetCommitTs() + + log.Info("add ddl item to syncer, you can add this commit ts to `ignore-txn-commit-ts` to skip this ddl if needed", + zap.String("sql", sql), zap.Int64("commit ts", binlog.CommitTs)) + + err = s.dsyncer.Sync(&dsync.Item{Binlog: binlog, PrewriteValue: nil, Schema: schema, Table: table}) + if err != nil { + err = errors.Annotatef(err, "add to dsyncer, commit ts %d", binlog.CommitTs) + break ForLoop + } + executeHistogram.Observe(time.Since(beginTime).Seconds()) + } } } } diff --git a/pkg/Hooks/test.go b/pkg/Hooks/test.go index 4dca6b56e..865262add 100644 --- a/pkg/Hooks/test.go +++ b/pkg/Hooks/test.go @@ -6,10 +6,16 @@ import ( ) type PluginDemo struct {} -func (pd PluginDemo)UpdateMarkTable(tx *loader.Tx, info *loopbacksync.LoopBackSync) *loader.Tx { +func (pd PluginDemo) ExtendTxn(tx *loader.Tx,info *loopbacksync.LoopBackSync) *loader.Tx { + //do sth return nil } +func (pd PluginDemo) FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) bool { + //do sth + return true +} + func NewPlugin() loader.LoopBack { return PluginDemo{} } diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index a084ea4d2..5faf54841 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -136,28 +136,29 @@ func (e *executor) begin() (*Tx, error) { queryHistogramVec: e.queryHistogramVec, } - isErr := false - hook := e.info.Hooks[plugin.LoaderPlugin] - hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoopBack) - if !ok { - isErr = true - return false + if e.info.SupportPlugin { + isErr := false + hook := e.info.Hooks[plugin.LoaderPlugin] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoopBack) + if !ok { + isErr = true + return false + } + tx = c.ExtendTxn(tx, e.info) + return true + }) + if isErr { + return nil, errors.New("type is incorrect") } - tx = c.UpdateMarkTable(tx, e.info) - return true - }) - if isErr { - return nil, errors.New("type is incorrect") - } - /* - if e.info != nil && e.info.LoopbackControl { - err1 := e.updateMark("", tx) - if err1 != nil { - return nil, errors.Trace(err1) + } else { + if e.info != nil && e.info.LoopbackControl { + err1 := e.updateMark("", tx) + if err1 != nil { + return nil, errors.Trace(err1) + } } } - */ return tx, nil } diff --git a/pkg/loader/iloopback.go b/pkg/loader/extend_txn.go similarity index 63% rename from pkg/loader/iloopback.go rename to pkg/loader/extend_txn.go index 64004fe37..ac7448888 100644 --- a/pkg/loader/iloopback.go +++ b/pkg/loader/extend_txn.go @@ -3,5 +3,5 @@ package loader import "github.com/pingcap/tidb-binlog/drainer/loopbacksync" type LoopBack interface { - UpdateMarkTable(tx *Tx, info *loopbacksync.LoopBackSync) *Tx + ExtendTxn(tx *Tx, info *loopbacksync.LoopBackSync) *Tx } diff --git a/pkg/loader/load.go b/pkg/loader/load.go index ffdf95192..0d7dce95e 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -205,18 +205,20 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { db.SetMaxOpenConns(opts.workerCount) db.SetMaxIdleConns(opts.workerCount) - for _, name := range s.loopBackSyncInfo.PluginNames { - sym, err := plugin.LoadPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], - s.loopBackSyncInfo.PluginPath, name) - if err != nil { - return nil, err - } - newPlugin, ok := sym.(func() LoopBack) - if !ok { - continue + if s.loopBackSyncInfo.SupportPlugin { + for _, name := range s.loopBackSyncInfo.PluginNames { + sym, err := plugin.LoadPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], + s.loopBackSyncInfo.PluginPath, name) + if err != nil { + return nil, err + } + newPlugin, ok := sym.(func() LoopBack) + if !ok { + continue + } + plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], + name, newPlugin()) } - plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], - name, newPlugin()) } return s, nil From 4811764fc6cd4fbfedc02f228f69a333c8b52adf Mon Sep 17 00:00:00 2001 From: tsthght Date: Tue, 25 Feb 2020 16:59:13 +0800 Subject: [PATCH 07/64] modify plugin demo --- pkg/plugin/plugindemo/Makefile | 3 +++ pkg/{Hooks/test.go => plugin/plugindemo/demo.go} | 7 +++---- 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 pkg/plugin/plugindemo/Makefile rename pkg/{Hooks/test.go => plugin/plugindemo/demo.go} (73%) diff --git a/pkg/plugin/plugindemo/Makefile b/pkg/plugin/plugindemo/Makefile new file mode 100644 index 000000000..cc0838a46 --- /dev/null +++ b/pkg/plugin/plugindemo/Makefile @@ -0,0 +1,3 @@ +plugin: + gofmt -w demo.go + go build -o demo.so -buildmode=plugin demo.go diff --git a/pkg/Hooks/test.go b/pkg/plugin/plugindemo/demo.go similarity index 73% rename from pkg/Hooks/test.go rename to pkg/plugin/plugindemo/demo.go index 865262add..c488ff3a2 100644 --- a/pkg/Hooks/test.go +++ b/pkg/plugin/plugindemo/demo.go @@ -5,8 +5,9 @@ import ( "github.com/pingcap/tidb-binlog/pkg/loader" ) -type PluginDemo struct {} -func (pd PluginDemo) ExtendTxn(tx *loader.Tx,info *loopbacksync.LoopBackSync) *loader.Tx { +type PluginDemo struct{} + +func (pd PluginDemo) ExtendTxn(tx *loader.Tx, info *loopbacksync.LoopBackSync) *loader.Tx { //do sth return nil } @@ -19,5 +20,3 @@ func (pd PluginDemo) FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) func NewPlugin() loader.LoopBack { return PluginDemo{} } - - From a5c557274fad1165cde6cdd459b5a468a32922da Mon Sep 17 00:00:00 2001 From: tsthght Date: Tue, 25 Feb 2020 17:06:30 +0800 Subject: [PATCH 08/64] gofmt --- pkg/plugin/plugins.go | 3 ++- pkg/plugin/plugins_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 8c85a2fc8..2411863e3 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -9,6 +9,7 @@ import ( //Plugin type we supported currently type Kind uint8 + const ( SyncerPlugin Kind = iota LoaderPlugin @@ -59,4 +60,4 @@ func LoadPlugin(eh *EventHooks, path, name string) (plugin.Symbol, error) { //RegisterPlugin() register plugin to EventHooks func RegisterPlugin(ehs *EventHooks, name string, plg interface{}) { ehs.SetPlugin(name, plg) -} \ No newline at end of file +} diff --git a/pkg/plugin/plugins_test.go b/pkg/plugin/plugins_test.go index f8d80385d..3608441ff 100644 --- a/pkg/plugin/plugins_test.go +++ b/pkg/plugin/plugins_test.go @@ -12,7 +12,8 @@ type ITest1 interface { type STest1 struct { a int } -func (s *STest1) Do () int { + +func (s *STest1) Do() int { return s.a } From 73c7f0b25ca92366f8429e0b87cf8c0800bd8b05 Mon Sep 17 00:00:00 2001 From: tsthght Date: Tue, 25 Feb 2020 17:12:57 +0800 Subject: [PATCH 09/64] add testcase --- drainer/config_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drainer/config_test.go b/drainer/config_test.go index 22540044a..156ebba24 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -57,6 +57,7 @@ func (t *testDrainerSuite) TestConfig(c *C) { "-advertise-addr", "192.168.15.10:8257", "-plugin-path", "/opt/drainer/plugin", "-plugin-names", "p1, p2", + "-support-plugin", } cfg := NewConfig() @@ -72,6 +73,7 @@ func (t *testDrainerSuite) TestConfig(c *C) { c.Assert(cfg.SyncerCfg.SQLMode, Equals, mysql.SQLMode(0)) c.Assert(cfg.SyncerCfg.PluginPath, Equals, "/opt/drainer/plugin") c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 2) + c.Assert(cfg.SyncerCfg.SupportPlugin, Equals, true) } func (t *testDrainerSuite) TestValidateFilter(c *C) { From c2ac0ecdf34c7d8c621544b6a2a782e1f0072b4a Mon Sep 17 00:00:00 2001 From: tsthght Date: Tue, 25 Feb 2020 17:22:23 +0800 Subject: [PATCH 10/64] modify testcase --- pkg/plugin/plugins_test.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/plugin/plugins_test.go b/pkg/plugin/plugins_test.go index 3608441ff..2e835a5cf 100644 --- a/pkg/plugin/plugins_test.go +++ b/pkg/plugin/plugins_test.go @@ -1,11 +1,20 @@ package plugin import ( - "testing" - - "github.com/stretchr/testify/assert" + "github.com/pingcap/check" ) +type PluginSuite struct { +} + +var _ = check.Suite(&PluginSuite{}) + +func (ps *PluginSuite) SetUpTest(c *check.C) { +} + +func (ps *PluginSuite) TearDownTest(c *check.C) { +} + type ITest1 interface { Do() int } @@ -17,11 +26,12 @@ func (s *STest1) Do() int { return s.a } -func TestRegisterPlugin(t *testing.T) { +func (ps *PluginSuite) TestRegisterPlugin(c *check.C) { hook := &EventHooks{} s := STest1{32} RegisterPlugin(hook, "test1", s) p := hook.GetAllPluginsName() - assert.Equal(t, 1, len(p)) + c.Assert(len(p), check.Equals, 1) + } From f9a74e3791ca5f52e86fd77e638f70eb21c49670 Mon Sep 17 00:00:00 2001 From: tsthght Date: Tue, 25 Feb 2020 17:26:00 +0800 Subject: [PATCH 11/64] modify go.mod --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index ece08e796..8d7039ec9 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,6 @@ require ( github.com/sirupsen/logrus v1.4.1 // indirect github.com/soheilhy/cmux v0.1.4 github.com/spf13/pflag v1.0.3 // indirect - github.com/stretchr/testify v1.4.0 github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d From ccfd79ba99e917f2e8dfb63c8f95fe15364bdf5f Mon Sep 17 00:00:00 2001 From: tsthght Date: Tue, 25 Feb 2020 18:18:28 +0800 Subject: [PATCH 12/64] modify --- drainer/syncer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drainer/syncer.go b/drainer/syncer.go index b1fcb96c8..0b586c098 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -448,7 +448,7 @@ ForLoop: // DDL (with version 10, commit ts 100) -> DDL (with version 9, commit ts 101) would never happen s.schema.addJob(b.job) - if !s.cfg.SyncDDL && !s.loopbackSync.SupportPlugin { + if !s.cfg.SyncDDL { log.Info("Syncer skips DDL", zap.String("sql", b.job.Query), zap.Int64("ts", b.GetCommitTs()), zap.Bool("SyncDDL", s.cfg.SyncDDL)) continue } From 861e5df1ce112cb3a295f40eb1bfa610171588d8 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 26 Feb 2020 07:01:18 +0800 Subject: [PATCH 13/64] fix a problem --- drainer/syncer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/drainer/syncer.go b/drainer/syncer.go index 0b586c098..29c329f9f 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -365,7 +365,6 @@ ForLoop: var txn *loader.Txn txn, err = translator.TiBinlogToTxn(s.schema, schema, table, binlog, preWrite) var isFilterTransaction = false - txn, err = translator.TiBinlogToTxn(s.schema, "", "", binlog, preWrite) if err != nil { err = errors.Annotate(err, "analyze transaction failed") break ForLoop From 2af503954c6dba7baf303fcdbe8698eb0ac1b705 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 26 Feb 2020 16:41:38 +0800 Subject: [PATCH 14/64] modify plugin --- pkg/loader/executor.go | 54 ++++++++++++++++++++++++---------------- pkg/loader/extend_txn.go | 6 +++-- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index 5faf54841..28f3f8ad6 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -136,29 +136,13 @@ func (e *executor) begin() (*Tx, error) { queryHistogramVec: e.queryHistogramVec, } - if e.info.SupportPlugin { - isErr := false - hook := e.info.Hooks[plugin.LoaderPlugin] - hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoopBack) - if !ok { - isErr = true - return false - } - tx = c.ExtendTxn(tx, e.info) - return true - }) - if isErr { - return nil, errors.New("type is incorrect") - } - } else { - if e.info != nil && e.info.LoopbackControl { - err1 := e.updateMark("", tx) - if err1 != nil { - return nil, errors.Trace(err1) - } + if e.info != nil && e.info.LoopbackControl && !e.info.SupportPlugin { + err1 := e.updateMark("", tx) + if err1 != nil { + return nil, errors.Trace(err1) } } + return tx, nil } @@ -236,6 +220,20 @@ func (e *executor) bulkReplace(inserts []*DML) error { // or we can simply check if it update unique index column or not, and for update change to (delete + insert) // the final result should has no duplicate entry or the origin dmls is wrong. func (e *executor) execTableBatch(ctx context.Context, dmls []*DML) error { + hook := e.info.Hooks[plugin.LoaderPlugin] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoopBack) + if !ok { + //ignore type incorrect error + return true + } + tx, dmls = c.ExtendTxn(tx, dmls, e.info) + if dmls == nil { + return false + } + return true + }) + if len(dmls) == 0 { return nil } @@ -305,6 +303,20 @@ func (e *executor) singleExec(dmls []*DML, safeMode bool) error { return errors.Trace(err) } + hook := e.info.Hooks[plugin.LoaderPlugin] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoopBack) + if !ok { + //ignore type incorrect error + return true + } + tx, dmls = c.ExtendTxn(tx, dmls, e.info) + if dmls == nil { + return false + } + return true + }) + for _, dml := range dmls { if safeMode && dml.Tp == UpdateDMLType { sql, args := dml.deleteSQL() diff --git a/pkg/loader/extend_txn.go b/pkg/loader/extend_txn.go index ac7448888..00f39047f 100644 --- a/pkg/loader/extend_txn.go +++ b/pkg/loader/extend_txn.go @@ -1,7 +1,9 @@ package loader -import "github.com/pingcap/tidb-binlog/drainer/loopbacksync" +import ( + "github.com/pingcap/tidb-binlog/drainer/loopbacksync" +) type LoopBack interface { - ExtendTxn(tx *Tx, info *loopbacksync.LoopBackSync) *Tx + ExtendTxn(tx *Tx, dmls []*DML, info *loopbacksync.LoopBackSync) (*Tx, []*DML) } From f4b62a221f595f4fa70ce0f2309f8eae685448d4 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 26 Feb 2020 17:34:09 +0800 Subject: [PATCH 15/64] modify plugin demo --- pkg/loader/executor.go | 81 +++++++++++++++++++---------------- pkg/loader/util.go | 6 ++- pkg/plugin/plugindemo/demo.go | 4 +- 3 files changed, 52 insertions(+), 39 deletions(-) diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index 28f3f8ad6..885f87a7d 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -146,24 +146,49 @@ func (e *executor) begin() (*Tx, error) { return tx, nil } +// return a wrap of sql.Tx +func (e *executor) externPoint(t *Tx, dmls []*DML) (*Tx, []*DML) { + hook := e.info.Hooks[plugin.LoaderPlugin] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoopBack) + if !ok { + //ignore type incorrect error + return true + } + t, dmls = c.ExtendTxn(t, dmls, e.info) + if dmls == nil { + return false + } + return true + }) + return t, dmls +} + func (e *executor) bulkDelete(deletes []*DML) error { if len(deletes) == 0 { return nil } var sqls strings.Builder - argss := make([]interface{}, 0, len(deletes)) + tx, err := e.begin() + if err != nil { + return errors.Trace(err) + } + + tx, deletes = e.externPoint(tx, deletes) + if len(deletes) == 0 { + return nil + } + + argss := make([]interface{}, 0, len(deletes)) for _, dml := range deletes { sql, args := dml.sql() sqls.WriteString(sql) sqls.WriteByte(';') argss = append(argss, args...) } - tx, err := e.begin() - if err != nil { - return errors.Trace(err) - } + sql := sqls.String() _, err = tx.autoRollbackExec(sql, argss...) if err != nil { @@ -194,6 +219,16 @@ func (e *executor) bulkReplace(inserts []*DML) error { builder.WriteString(holder) } + tx, err := e.begin() + if err != nil { + return errors.Trace(err) + } + + tx, inserts = e.externPoint(tx, inserts) + if len(inserts) == 0 { + return nil + } + args := make([]interface{}, 0, len(inserts)*len(info.columns)) for _, insert := range inserts { for _, name := range info.columns { @@ -201,10 +236,7 @@ func (e *executor) bulkReplace(inserts []*DML) error { args = append(args, v) } } - tx, err := e.begin() - if err != nil { - return errors.Trace(err) - } + _, err = tx.autoRollbackExec(builder.String(), args...) if err != nil { return errors.Trace(err) @@ -220,20 +252,6 @@ func (e *executor) bulkReplace(inserts []*DML) error { // or we can simply check if it update unique index column or not, and for update change to (delete + insert) // the final result should has no duplicate entry or the origin dmls is wrong. func (e *executor) execTableBatch(ctx context.Context, dmls []*DML) error { - hook := e.info.Hooks[plugin.LoaderPlugin] - hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoopBack) - if !ok { - //ignore type incorrect error - return true - } - tx, dmls = c.ExtendTxn(tx, dmls, e.info) - if dmls == nil { - return false - } - return true - }) - if len(dmls) == 0 { return nil } @@ -303,19 +321,10 @@ func (e *executor) singleExec(dmls []*DML, safeMode bool) error { return errors.Trace(err) } - hook := e.info.Hooks[plugin.LoaderPlugin] - hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoopBack) - if !ok { - //ignore type incorrect error - return true - } - tx, dmls = c.ExtendTxn(tx, dmls, e.info) - if dmls == nil { - return false - } - return true - }) + tx, dmls = e.externPoint(tx, dmls) + if len(dmls) == 0 { + return nil + } for _, dml := range dmls { if safeMode && dml.Tp == UpdateDMLType { diff --git a/pkg/loader/util.go b/pkg/loader/util.go index 996312b12..bd1175a4c 100644 --- a/pkg/loader/util.go +++ b/pkg/loader/util.go @@ -98,7 +98,11 @@ func CreateDB(user string, password string, host string, port int) (db *gosql.DB } func quoteSchema(schema string, table string) string { - return fmt.Sprintf("`%s`.`%s`", escapeName(schema), escapeName(table)) + if len(schema) == 0 { + return fmt.Sprintf("`%s`", escapeName(table)) + } else { + return fmt.Sprintf("`%s`.`%s`", escapeName(schema), escapeName(table)) + } } func quoteName(name string) string { diff --git a/pkg/plugin/plugindemo/demo.go b/pkg/plugin/plugindemo/demo.go index c488ff3a2..2172c4485 100644 --- a/pkg/plugin/plugindemo/demo.go +++ b/pkg/plugin/plugindemo/demo.go @@ -7,9 +7,9 @@ import ( type PluginDemo struct{} -func (pd PluginDemo) ExtendTxn(tx *loader.Tx, info *loopbacksync.LoopBackSync) *loader.Tx { +func (pd PluginDemo) ExtendTxn(tx *loader.Tx, dmls []*loader.DML, info *loopbacksync.LoopBackSync) (*loader.Tx, []*loader.DML) { //do sth - return nil + return nil, nil } func (pd PluginDemo) FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) bool { From 3ca49244ab6ee26b2c6243d9b28e91eb420d7c9f Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 27 Feb 2020 12:24:04 +0800 Subject: [PATCH 16/64] modify plugin pos --- drainer/syncer.go | 220 +++++++++++++++++++++++----------------------- 1 file changed, 112 insertions(+), 108 deletions(-) diff --git a/drainer/syncer.go b/drainer/syncer.go index 29c329f9f..3981067ee 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -348,28 +348,39 @@ ForLoop: if startTS == commitTS { fakeBinlogs = append(fakeBinlogs, binlog) fakeBinlogPreAddTS = append(fakeBinlogPreAddTS, lastAddComitTS) - } else { - if s.loopbackSync.SupportPlugin { - schema, table, err := s.schema.getSchemaTableAndDelete(b.job.BinlogInfo.SchemaVersion) - if err != nil { - err = errors.Trace(err) - break ForLoop - } + } else if jobID == 0 { + preWriteValue := binlog.GetPrewriteValue() + preWrite := &pb.PrewriteValue{} + err = preWrite.Unmarshal(preWriteValue) + if err != nil { + err = errors.Annotatef(err, "prewrite %s Unmarshal failed", preWriteValue) + break ForLoop + } - preWrite := &pb.PrewriteValue{} - err = preWrite.Unmarshal(binlog.GetPrewriteValue()) - if err != nil { - err = errors.Annotatef(err, "prewrite %s Unmarshal failed", binlog.GetPrewriteValue()) - break ForLoop - } - var txn *loader.Txn - txn, err = translator.TiBinlogToTxn(s.schema, schema, table, binlog, preWrite) - var isFilterTransaction = false - if err != nil { - err = errors.Annotate(err, "analyze transaction failed") - break ForLoop - } + err = s.rewriteForOldVersion(preWrite) + if err != nil { + err = errors.Annotate(err, "rewrite for old version fail") + break ForLoop + } + + log.Debug("get DML", zap.Int64("SchemaVersion", preWrite.SchemaVersion)) + if preWrite.SchemaVersion < lastDDLSchemaVersion { + log.Debug("encounter older schema dml") + } + + err = s.schema.handlePreviousDDLJobIfNeed(preWrite.SchemaVersion) + if err != nil { + err = errors.Annotate(err, "handlePreviousDDLJobIfNeed failed") + break ForLoop + } + + var isFilterTransaction = false + var err1 error + + if s.loopbackSync.SupportPlugin { hook := s.loopbackSync.Hooks[plugin.SyncerPlugin] + var txn *loader.Txn + txn, err1 = translator.TiBinlogToTxn(s.schema, "", "", binlog, preWrite) hook.Range(func(k, val interface{}) bool { c, ok := val.(LoopBack) if !ok { @@ -381,117 +392,110 @@ ForLoop: } return true }) - if isFilterTransaction { - continue - } } - if jobID == 0 { - preWriteValue := binlog.GetPrewriteValue() - preWrite := &pb.PrewriteValue{} - err = preWrite.Unmarshal(preWriteValue) - if err != nil { - err = errors.Annotatef(err, "prewrite %s Unmarshal failed", preWriteValue) - break ForLoop - } - - err = s.rewriteForOldVersion(preWrite) - if err != nil { - err = errors.Annotate(err, "rewrite for old version fail") + if s.loopbackSync != nil && s.loopbackSync.LoopbackControl && !s.loopbackSync.SupportPlugin { + isFilterTransaction, err1 = loopBackStatus(binlog, preWrite, s.schema, s.loopbackSync) + if err1 != nil { + err = errors.Annotate(err1, "analyze transaction failed") break ForLoop } + } - log.Debug("get DML", zap.Int64("SchemaVersion", preWrite.SchemaVersion)) - if preWrite.SchemaVersion < lastDDLSchemaVersion { - log.Debug("encounter older schema dml") - } + var ignore bool + ignore, err = filterTable(preWrite, s.filter, s.schema) + if err != nil { + err = errors.Annotate(err, "filterTable failed") + break ForLoop + } - err = s.schema.handlePreviousDDLJobIfNeed(preWrite.SchemaVersion) + if !ignore && !isFilterTransaction { + s.addDMLEventMetrics(preWrite.GetMutations()) + beginTime := time.Now() + lastAddComitTS = binlog.GetCommitTs() + err = s.dsyncer.Sync(&dsync.Item{Binlog: binlog, PrewriteValue: preWrite}) if err != nil { - err = errors.Annotate(err, "handlePreviousDDLJobIfNeed failed") + err = errors.Annotatef(err, "add to dsyncer, commit ts %d", binlog.CommitTs) break ForLoop } + executeHistogram.Observe(time.Since(beginTime).Seconds()) + } + } else if jobID > 0 { + log.Debug("get ddl binlog job", zap.Stringer("job", b.job)) - var isFilterTransaction = false - var err1 error - if s.loopbackSync != nil && s.loopbackSync.LoopbackControl && !s.loopbackSync.SupportPlugin { - isFilterTransaction, err1 = loopBackStatus(binlog, preWrite, s.schema, s.loopbackSync) - if err1 != nil { - err = errors.Annotate(err1, "analyze transaction failed") - break ForLoop - } - } + // Notice: the version of DDL Binlog we receive are Monotonically increasing + // DDL (with version 10, commit ts 100) -> DDL (with version 9, commit ts 101) would never happen + s.schema.addJob(b.job) - var ignore bool - ignore, err = filterTable(preWrite, s.filter, s.schema) - if err != nil { - err = errors.Annotate(err, "filterTable failed") - break ForLoop - } + if !s.cfg.SyncDDL && !s.loopbackSync.SupportPlugin { + log.Info("Syncer skips DDL", zap.String("sql", b.job.Query), zap.Int64("ts", b.GetCommitTs()), zap.Bool("SyncDDL", s.cfg.SyncDDL)) + continue + } - if !ignore && !isFilterTransaction { - s.addDMLEventMetrics(preWrite.GetMutations()) - beginTime := time.Now() - lastAddComitTS = binlog.GetCommitTs() - err = s.dsyncer.Sync(&dsync.Item{Binlog: binlog, PrewriteValue: preWrite}) - if err != nil { - err = errors.Annotatef(err, "add to dsyncer, commit ts %d", binlog.CommitTs) - break ForLoop - } - executeHistogram.Observe(time.Since(beginTime).Seconds()) - } - } else if jobID > 0 { - log.Debug("get ddl binlog job", zap.Stringer("job", b.job)) + log.Debug("get DDL", zap.Int64("SchemaVersion", b.job.BinlogInfo.SchemaVersion)) + lastDDLSchemaVersion = b.job.BinlogInfo.SchemaVersion - // Notice: the version of DDL Binlog we receive are Monotonically increasing - // DDL (with version 10, commit ts 100) -> DDL (with version 9, commit ts 101) would never happen - s.schema.addJob(b.job) + err = s.schema.handlePreviousDDLJobIfNeed(b.job.BinlogInfo.SchemaVersion) + if err != nil { + err = errors.Trace(err) + break ForLoop + } - if !s.cfg.SyncDDL { - log.Info("Syncer skips DDL", zap.String("sql", b.job.Query), zap.Int64("ts", b.GetCommitTs()), zap.Bool("SyncDDL", s.cfg.SyncDDL)) - continue - } + if b.job.SchemaState == model.StateDeleteOnly && b.job.Type == model.ActionDropColumn { + log.Info("Syncer skips DeleteOnly DDL", zap.Stringer("job", b.job), zap.Int64("ts", b.GetCommitTs())) + continue + } - log.Debug("get DDL", zap.Int64("SchemaVersion", b.job.BinlogInfo.SchemaVersion)) - lastDDLSchemaVersion = b.job.BinlogInfo.SchemaVersion + sql := b.job.Query + var schema, table string + schema, table, err = s.schema.getSchemaTableAndDelete(b.job.BinlogInfo.SchemaVersion) + if err != nil { + err = errors.Trace(err) + break ForLoop + } - err = s.schema.handlePreviousDDLJobIfNeed(b.job.BinlogInfo.SchemaVersion) - if err != nil { - err = errors.Trace(err) + if s.loopbackSync.SupportPlugin { + var isFilterTransaction = false + txn := new(loader.Txn) + txn.DDL = &loader.DDL{ + Database: schema, + Table: table, + SQL: string(binlog.GetDdlQuery()), + } + hook := s.loopbackSync.Hooks[plugin.SyncerPlugin] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoopBack) + if !ok { + return true + } + isFilterTransaction = c.FilterTxn(txn, s.loopbackSync) + if isFilterTransaction { + return false + } + return true + }) + if isFilterTransaction { break ForLoop } + } - if b.job.SchemaState == model.StateDeleteOnly && b.job.Type == model.ActionDropColumn { - log.Info("Syncer skips DeleteOnly DDL", zap.Stringer("job", b.job), zap.Int64("ts", b.GetCommitTs())) - continue - } + if s.filter.SkipSchemaAndTable(schema, table) { + log.Info("skip ddl", zap.String("schema", schema), zap.String("table", table), + zap.String("sql", sql), zap.Int64("commit ts", commitTS)) + } else if sql != "" { + s.addDDLCount() + beginTime := time.Now() + lastAddComitTS = binlog.GetCommitTs() - sql := b.job.Query - var schema, table string - schema, table, err = s.schema.getSchemaTableAndDelete(b.job.BinlogInfo.SchemaVersion) + log.Info("add ddl item to syncer, you can add this commit ts to `ignore-txn-commit-ts` to skip this ddl if needed", + zap.String("sql", sql), zap.Int64("commit ts", binlog.CommitTs)) + + err = s.dsyncer.Sync(&dsync.Item{Binlog: binlog, PrewriteValue: nil, Schema: schema, Table: table}) if err != nil { - err = errors.Trace(err) + err = errors.Annotatef(err, "add to dsyncer, commit ts %d", binlog.CommitTs) break ForLoop } - - if s.filter.SkipSchemaAndTable(schema, table) { - log.Info("skip ddl", zap.String("schema", schema), zap.String("table", table), - zap.String("sql", sql), zap.Int64("commit ts", commitTS)) - } else if sql != "" { - s.addDDLCount() - beginTime := time.Now() - lastAddComitTS = binlog.GetCommitTs() - - log.Info("add ddl item to syncer, you can add this commit ts to `ignore-txn-commit-ts` to skip this ddl if needed", - zap.String("sql", sql), zap.Int64("commit ts", binlog.CommitTs)) - - err = s.dsyncer.Sync(&dsync.Item{Binlog: binlog, PrewriteValue: nil, Schema: schema, Table: table}) - if err != nil { - err = errors.Annotatef(err, "add to dsyncer, commit ts %d", binlog.CommitTs) - break ForLoop - } - executeHistogram.Observe(time.Since(beginTime).Seconds()) - } + executeHistogram.Observe(time.Since(beginTime).Seconds()) } } } From 6ea9197edb49db4864660a8a9a5f12acdf443632 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 27 Feb 2020 14:44:15 +0800 Subject: [PATCH 17/64] modify FilterTxn ret --- drainer/filter_txn.go | 2 +- drainer/syncer.go | 17 ++++++++++++----- pkg/plugin/plugindemo/demo.go | 4 ++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/drainer/filter_txn.go b/drainer/filter_txn.go index 0721d646a..fb233cb26 100644 --- a/drainer/filter_txn.go +++ b/drainer/filter_txn.go @@ -6,5 +6,5 @@ import ( ) type LoopBack interface { - FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) bool + FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) (bool, error) } diff --git a/drainer/syncer.go b/drainer/syncer.go index 3981067ee..4b321720f 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -386,12 +386,15 @@ ForLoop: if !ok { return true } - isFilterTransaction = c.FilterTxn(txn, s.loopbackSync) - if isFilterTransaction { + isFilterTransaction, err1 = c.FilterTxn(txn, s.loopbackSync) + if isFilterTransaction || err1 != nil { return false } return true }) + if err1 != nil { + break ForLoop + } } if s.loopbackSync != nil && s.loopbackSync.LoopbackControl && !s.loopbackSync.SupportPlugin { @@ -456,6 +459,7 @@ ForLoop: if s.loopbackSync.SupportPlugin { var isFilterTransaction = false + var err1 error txn := new(loader.Txn) txn.DDL = &loader.DDL{ Database: schema, @@ -468,15 +472,18 @@ ForLoop: if !ok { return true } - isFilterTransaction = c.FilterTxn(txn, s.loopbackSync) - if isFilterTransaction { + isFilterTransaction, err1 = c.FilterTxn(txn, s.loopbackSync) + if isFilterTransaction || err1 != nil { return false } return true }) - if isFilterTransaction { + if err1 != nil { break ForLoop } + if isFilterTransaction { + continue + } } if s.filter.SkipSchemaAndTable(schema, table) { diff --git a/pkg/plugin/plugindemo/demo.go b/pkg/plugin/plugindemo/demo.go index 2172c4485..c92327fea 100644 --- a/pkg/plugin/plugindemo/demo.go +++ b/pkg/plugin/plugindemo/demo.go @@ -12,9 +12,9 @@ func (pd PluginDemo) ExtendTxn(tx *loader.Tx, dmls []*loader.DML, info *loopback return nil, nil } -func (pd PluginDemo) FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) bool { +func (pd PluginDemo) FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) (bool, error) { //do sth - return true + return true, nil } func NewPlugin() loader.LoopBack { From e4917f36e79d221d317e065ff22131403a2f2094 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 27 Feb 2020 15:48:42 +0800 Subject: [PATCH 18/64] modify value: index --- drainer/loopbacksync/loopbacksync.go | 2 ++ pkg/loader/executor.go | 7 +------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index c97c82d35..ad8c38856 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -48,6 +48,7 @@ type LoopBackSync struct { ChannelID int64 LoopbackControl bool SyncDDL bool + Index int64 PluginPath string PluginNames []string Hooks []*plugin.EventHooks @@ -60,6 +61,7 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st ChannelID: ChannelID, LoopbackControl: LoopbackControl, SyncDDL: SyncDDL, + Index: 0, PluginPath: path, PluginNames: names, SupportPlugin: SupportPlug, diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index 1e7ed4909..b2609ca44 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -37,7 +37,6 @@ import ( var ( defaultBatchSize = 128 defaultWorkerCount = 16 - index int64 ) type executor struct { @@ -128,10 +127,6 @@ func (tx *Tx) commit() error { return errors.Trace(err) } -func (e *executor) addIndex() int64 { - return atomic.AddInt64(&index, 1) % ((int64)(e.workerCount)) -} - // return a wrap of sql.Tx func (e *executor) begin() (*Tx, error) { sqlTx, err := e.db.Begin() @@ -147,7 +142,7 @@ func (e *executor) begin() (*Tx, error) { if e.info != nil && e.info.LoopbackControl && !e.info.SupportPlugin { start := time.Now() - err = loopbacksync.UpdateMark(tx.Tx, e.addIndex(), e.info.ChannelID) + err = loopbacksync.UpdateMark(tx.Tx, atomic.AddInt64(&e.info.Index, 1)%((int64)(e.workerCount)), e.info.ChannelID) if err != nil { rerr := tx.Rollback() if rerr != nil { From 8f24b8acb33d20431868673014f77aff138ba4c0 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 27 Feb 2020 19:20:03 +0800 Subject: [PATCH 19/64] add RecordId --- drainer/loopbacksync/loopbacksync.go | 1 + pkg/loader/load.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index ad8c38856..cfd92fa2e 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -53,6 +53,7 @@ type LoopBackSync struct { PluginNames []string Hooks []*plugin.EventHooks SupportPlugin bool + RecordId int } //NewLoopBackSyncInfo return LoopBackSyncInfo objec diff --git a/pkg/loader/load.go b/pkg/loader/load.go index bf51497ca..056d87b0f 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -203,6 +203,8 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { cancel: cancel, } + s.loopBackSyncInfo.RecordId = s.workerCount + db.SetMaxOpenConns(opts.workerCount) db.SetMaxIdleConns(opts.workerCount) From 24eaa9e6508b19e3b3e3311256fe7c725687cab5 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 27 Feb 2020 21:53:29 +0800 Subject: [PATCH 20/64] modify plugin --- pkg/loader/executor.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index b2609ca44..6193784ec 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -217,6 +217,16 @@ func (e *executor) bulkReplace(inserts []*DML) error { return nil } + tx, err := e.begin() + if err != nil { + return errors.Trace(err) + } + + tx, inserts = e.externPoint(tx, inserts) + if len(inserts) == 0 { + return nil + } + info := inserts[0].info var builder strings.Builder @@ -232,16 +242,6 @@ func (e *executor) bulkReplace(inserts []*DML) error { builder.WriteString(holder) } - tx, err := e.begin() - if err != nil { - return errors.Trace(err) - } - - tx, inserts = e.externPoint(tx, inserts) - if len(inserts) == 0 { - return nil - } - args := make([]interface{}, 0, len(inserts)*len(info.columns)) for _, insert := range inserts { for _, name := range info.columns { From c97f3f262c3537378e771ff702fb7d7d294ae838 Mon Sep 17 00:00:00 2001 From: tsthght Date: Tue, 3 Mar 2020 01:06:25 +0800 Subject: [PATCH 21/64] support configure the mark database name and mark table name --- drainer/config.go | 4 +++ drainer/loopbacksync/loopbacksync.go | 42 +++++++++++++++-------- drainer/loopbacksync/loopbacksync_test.go | 8 +++-- drainer/syncer.go | 3 +- pkg/loader/load.go | 2 +- 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/drainer/config.go b/drainer/config.go index c9cdd5573..bab5a51f7 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -81,6 +81,8 @@ type SyncerConfig struct { PluginPath string `toml:"plugin-path" json:"plugin-path"` PluginNames []string `toml:"plugin-names" json:"plugin-names"` SupportPlugin bool `toml:"support-plugin" json:"support-plugin"` + MarkDBName string `toml:"mark-db-name" json:"mark-db-name"` + MarkTableName string `toml:"mark-table-name" json:"mark-table-name"` } // RelayConfig is the Relay log's configuration. @@ -179,6 +181,8 @@ func NewConfig() *Config { fs.StringVar(&cfg.SyncerCfg.PluginPath, "plugin-path", "", "The path of the plugins") fs.Var(newSliceNames([]string{}, &cfg.SyncerCfg.PluginNames), "plugin-names", "The names of the plugins") fs.BoolVar(&cfg.SyncerCfg.SupportPlugin, "support-plugin", false, "Whether plugin is supported,default: false") + fs.StringVar(&cfg.SyncerCfg.MarkDBName, "mark-db-name", "rel", "mark database's name") + fs.StringVar(&cfg.SyncerCfg.MarkTableName, "mark-table-name", "_drainer_repl_mark", "mark table's name") return cfg } diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index cfd92fa2e..b9c8c397c 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -37,16 +37,12 @@ const ( ChannelInfo = "channel_info" ) -// CreateMarkTableDDL is the DDL to create the mark table. -var CreateMarkTableDDL string = fmt.Sprintf("CREATE TABLE If Not Exists %s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", MarkTableName, ID, ChannelID, Val, ChannelInfo, ID, ChannelID) - -// CreateMarkDBDDL is DDL to create the database of mark table. -var CreateMarkDBDDL = "create database IF NOT EXISTS retl;" - //LoopBackSync loopback sync info type LoopBackSync struct { ChannelID int64 LoopbackControl bool + MarkDBName string + MarkTableName string SyncDDL bool Index int64 PluginPath string @@ -57,7 +53,7 @@ type LoopBackSync struct { } //NewLoopBackSyncInfo return LoopBackSyncInfo objec -func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path string, names []string, SupportPlug bool) *LoopBackSync { +func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path string, names []string, SupportPlug bool, mdbname, mtablename string) *LoopBackSync { l := &LoopBackSync{ ChannelID: ChannelID, LoopbackControl: LoopbackControl, @@ -66,20 +62,36 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st PluginPath: path, PluginNames: names, SupportPlugin: SupportPlug, + MarkDBName: mdbname, + MarkTableName: mtablename, } return l } // CreateMarkTable create the db and table if need. -func CreateMarkTable(db *sql.DB) error { - _, err := db.Exec(CreateMarkDBDDL) - if err != nil { - return errors.Annotate(err, "failed to create mark db") - } +func CreateMarkTable(db *sql.DB, mdbname, mtablename string) error { + // CreateMarkDBDDL is DDL to create the database of mark table. + var err error + if len(mdbname) == 0 { + // CreateMarkTableDDL is the DDL to create the mark table. + var CreateMarkTableDDL string = fmt.Sprintf("CREATE TABLE If Not Exists %s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", mtablename, ID, ChannelID, Val, ChannelInfo, ID, ChannelID) + _, err = db.Exec(CreateMarkTableDDL) + if err != nil { + return errors.Annotate(err, "failed to create mark table") + } + } else { + var CreateMarkDBDDL = fmt.Sprintf("create database IF NOT EXISTS %s;", mdbname) + _, err = db.Exec(CreateMarkDBDDL) + if err != nil { + return errors.Annotate(err, "failed to create mark db") + } - _, err = db.Exec(CreateMarkTableDDL) - if err != nil { - return errors.Annotate(err, "failed to create mark table") + // CreateMarkTableDDL is the DDL to create the mark table. + var CreateMarkTableDDL string = fmt.Sprintf("CREATE TABLE If Not Exists %s.%s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", mdbname, mtablename, ID, ChannelID, Val, ChannelInfo, ID, ChannelID) + _, err = db.Exec(CreateMarkTableDDL) + if err != nil { + return errors.Annotate(err, "failed to create mark table") + } } return nil diff --git a/drainer/loopbacksync/loopbacksync_test.go b/drainer/loopbacksync/loopbacksync_test.go index 48e7efc2c..cb92fedfd 100644 --- a/drainer/loopbacksync/loopbacksync_test.go +++ b/drainer/loopbacksync/loopbacksync_test.go @@ -15,6 +15,7 @@ package loopbacksync import ( "database/sql/driver" + "fmt" "regexp" "testing" @@ -33,7 +34,7 @@ func (s *loopbackSuite) TestNewLoopBackSyncInfo(c *check.C) { var LoopbackControl = true var SyncDDL = false - l := NewLoopBackSyncInfo(ChannelID, LoopbackControl, SyncDDL, "", nil, false) + l := NewLoopBackSyncInfo(ChannelID, LoopbackControl, SyncDDL, "", nil, false, "rel", "_drainer_repl_mark") c.Assert(l, check.DeepEquals, &LoopBackSync{ ChannelID: ChannelID, @@ -46,12 +47,15 @@ func (s *loopbackSuite) TestCreateMarkTable(c *check.C) { db, mk, err := sqlmock.New() c.Assert(err, check.IsNil) + CreateMarkDBDDL := "create database IF NOT EXISTS rel;" + CreateMarkTableDDL := fmt.Sprintf("CREATE TABLE If Not Exists %s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", "_drainer_repl_mark", ID, ChannelID, Val, ChannelInfo, ID, ChannelID) + mk.ExpectExec(regexp.QuoteMeta(CreateMarkDBDDL)). WillReturnResult(sqlmock.NewResult(0, 0)) mk.ExpectExec(regexp.QuoteMeta(CreateMarkTableDDL)). WillReturnResult(sqlmock.NewResult(0, 0)) - err = CreateMarkTable(db) + err = CreateMarkTable(db, "rel", "_drainer_repl_mark") c.Assert(err, check.IsNil) err = mk.ExpectationsWereMet() diff --git a/drainer/syncer.go b/drainer/syncer.go index 8cec8267c..1ce85ee4f 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -77,7 +77,8 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( ignoreDBs = strings.Split(cfg.IgnoreSchemas, ",") } syncer.filter = filter.NewFilter(ignoreDBs, cfg.IgnoreTables, cfg.DoDBs, cfg.DoTables) - syncer.loopbackSync = loopbacksync.NewLoopBackSyncInfo(cfg.ChannelID, cfg.LoopbackControl, cfg.SyncDDL, cfg.PluginPath, cfg.PluginNames, cfg.SupportPlugin) + syncer.loopbackSync = loopbacksync.NewLoopBackSyncInfo(cfg.ChannelID, cfg.LoopbackControl, cfg.SyncDDL, cfg.PluginPath, + cfg.PluginNames, cfg.SupportPlugin, cfg.MarkDBName, cfg.MarkTableName) if syncer.loopbackSync.SupportPlugin { for _, name := range syncer.loopbackSync.PluginNames { sym, err := plugin.LoadPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 056d87b0f..8b5558007 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -512,7 +512,7 @@ func (s *loaderImpl) execDMLs(dmls []*DML) error { } func (s *loaderImpl) initMarkTable() error { - if err := loopbacksync.CreateMarkTable(s.db); err != nil { + if err := loopbacksync.CreateMarkTable(s.db, s.loopBackSyncInfo.MarkDBName, s.loopBackSyncInfo.MarkTableName); err != nil { return errors.Trace(err) } return loopbacksync.InitMarkTableData(s.db, s.workerCount, s.loopBackSyncInfo.ChannelID) From 536943a266d935d876122a38d1afdc092fc1e379 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 4 Mar 2020 16:13:09 +0800 Subject: [PATCH 22/64] add some testcase --- drainer/config.go | 1 + drainer/config_test.go | 62 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/drainer/config.go b/drainer/config.go index bab5a51f7..f30ea7a46 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -178,6 +178,7 @@ func NewConfig() *Config { fs.IntVar(&maxBinlogItemCount, "cache-binlog-count", defaultBinlogItemCount, "blurry count of binlogs in cache, limit cache size") fs.IntVar(&cfg.SyncedCheckTime, "synced-check-time", defaultSyncedCheckTime, "if we can't detect new binlog after many minute, we think the all binlog is all synced") fs.StringVar(new(string), "log-rotate", "", "DEPRECATED") + fs.StringVar(&cfg.SyncerCfg.PluginPath, "plugin-path", "", "The path of the plugins") fs.Var(newSliceNames([]string{}, &cfg.SyncerCfg.PluginNames), "plugin-names", "The names of the plugins") fs.BoolVar(&cfg.SyncerCfg.SupportPlugin, "support-plugin", false, "Whether plugin is supported,default: false") diff --git a/drainer/config_test.go b/drainer/config_test.go index 156ebba24..f63038459 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -55,9 +55,6 @@ func (t *testDrainerSuite) TestConfig(c *C) { "-config", "../cmd/drainer/drainer.toml", "-addr", "192.168.15.10:8257", "-advertise-addr", "192.168.15.10:8257", - "-plugin-path", "/opt/drainer/plugin", - "-plugin-names", "p1, p2", - "-support-plugin", } cfg := NewConfig() @@ -71,9 +68,6 @@ func (t *testDrainerSuite) TestConfig(c *C) { var strSQLMode *string c.Assert(cfg.SyncerCfg.StrSQLMode, Equals, strSQLMode) c.Assert(cfg.SyncerCfg.SQLMode, Equals, mysql.SQLMode(0)) - c.Assert(cfg.SyncerCfg.PluginPath, Equals, "/opt/drainer/plugin") - c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 2) - c.Assert(cfg.SyncerCfg.SupportPlugin, Equals, true) } func (t *testDrainerSuite) TestValidateFilter(c *C) { @@ -299,3 +293,59 @@ func (t *testKafkaSuite) TestConfigDestDBTypeKafka(c *C) { c.Assert(cfg.SyncerCfg.To.KafkaVersion, Equals, defaultKafkaVersion) c.Assert(cfg.SyncerCfg.To.KafkaMaxMessages, Equals, 1024) } + +func (t *testKafkaSuite) TestConfigPlugin(c *C) { + args := []string{} + + cfg := NewConfig() + err := cfg.Parse(args) + c.Assert(err, IsNil) + + c.Assert(len(cfg.SyncerCfg.PluginPath), Equals, 0) + c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 0) + c.Assert(cfg.SyncerCfg.SupportPlugin, Equals, false) + c.Assert(cfg.SyncerCfg.MarkDBName, Equals, "rel") + c.Assert(cfg.SyncerCfg.MarkTableName, Equals, "_drainer_repl_mark") + + args = []string{ + "-plugin-path", "/tmp/drainer/plugin", + "-plugin-names", "demo1", + "-support-plugin", + "-mark-db-name", "db1", + "-mark-table-name", "tb1", + } + + cfg = NewConfig() + err = cfg.Parse(args) + c.Assert(err, IsNil) + + c.Assert(cfg.SyncerCfg.PluginPath, Equals, "/tmp/drainer/plugin") + c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 1) + c.Assert(cfg.SyncerCfg.PluginNames[0], Equals, "demo1") + c.Assert(cfg.SyncerCfg.SupportPlugin, Equals, true) + c.Assert(cfg.SyncerCfg.MarkDBName, Equals, "db1") + c.Assert(cfg.SyncerCfg.MarkTableName, Equals, "tb1") + + args = []string{ + "-plugin-names", "demo1,demo2", + } + + cfg = NewConfig() + err = cfg.Parse(args) + c.Assert(err, IsNil) + + c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 2) + c.Assert(cfg.SyncerCfg.PluginNames[0], Equals, "demo1") + c.Assert(cfg.SyncerCfg.PluginNames[1], Equals, "demo2") + + args = []string{ + "-plugin-names", "", + } + + cfg = NewConfig() + err = cfg.Parse(args) + c.Assert(err, IsNil) + + c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 1) + c.Assert(cfg.SyncerCfg.PluginNames[0], Equals, "") +} From af0558131e032f65a2a86b0c90c085cec47426c7 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 4 Mar 2020 16:15:15 +0800 Subject: [PATCH 23/64] add testcase --- drainer/config_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drainer/config_test.go b/drainer/config_test.go index f63038459..4ff93401d 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -348,4 +348,16 @@ func (t *testKafkaSuite) TestConfigPlugin(c *C) { c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 1) c.Assert(cfg.SyncerCfg.PluginNames[0], Equals, "") + + args = []string{ + "-plugin-names", ",", + } + + cfg = NewConfig() + err = cfg.Parse(args) + c.Assert(err, IsNil) + + c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 2) + c.Assert(cfg.SyncerCfg.PluginNames[0], Equals, "") + c.Assert(cfg.SyncerCfg.PluginNames[1], Equals, "") } From 1d4e857ea6914eea8ed00eef7f1f362e460ef3b8 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 4 Mar 2020 16:18:40 +0800 Subject: [PATCH 24/64] add testcase --- drainer/config_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drainer/config_test.go b/drainer/config_test.go index 4ff93401d..445615c4c 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -328,6 +328,8 @@ func (t *testKafkaSuite) TestConfigPlugin(c *C) { args = []string{ "-plugin-names", "demo1,demo2", + "-mark-db-name", "", + "-mark-table-name", "", } cfg = NewConfig() @@ -337,6 +339,8 @@ func (t *testKafkaSuite) TestConfigPlugin(c *C) { c.Assert(len(cfg.SyncerCfg.PluginNames), Equals, 2) c.Assert(cfg.SyncerCfg.PluginNames[0], Equals, "demo1") c.Assert(cfg.SyncerCfg.PluginNames[1], Equals, "demo2") + c.Assert(cfg.SyncerCfg.MarkDBName, Equals, "") + c.Assert(cfg.SyncerCfg.MarkTableName, Equals, "") args = []string{ "-plugin-names", "", From fc3644f8bbf1f8adaa6d30f86764af86e3caec68 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 4 Mar 2020 16:31:19 +0800 Subject: [PATCH 25/64] modify some error --- drainer/loopbacksync/loopbacksync_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drainer/loopbacksync/loopbacksync_test.go b/drainer/loopbacksync/loopbacksync_test.go index cb92fedfd..ca3fb7145 100644 --- a/drainer/loopbacksync/loopbacksync_test.go +++ b/drainer/loopbacksync/loopbacksync_test.go @@ -40,6 +40,11 @@ func (s *loopbackSuite) TestNewLoopBackSyncInfo(c *check.C) { ChannelID: ChannelID, LoopbackControl: LoopbackControl, SyncDDL: SyncDDL, + PluginPath: "", + PluginNames: nil, + SupportPlugin: false, + MarkDBName: "rel", + MarkTableName: "_drainer_repl_mark", }) } @@ -48,7 +53,7 @@ func (s *loopbackSuite) TestCreateMarkTable(c *check.C) { c.Assert(err, check.IsNil) CreateMarkDBDDL := "create database IF NOT EXISTS rel;" - CreateMarkTableDDL := fmt.Sprintf("CREATE TABLE If Not Exists %s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", "_drainer_repl_mark", ID, ChannelID, Val, ChannelInfo, ID, ChannelID) + CreateMarkTableDDL := fmt.Sprintf("CREATE TABLE If Not Exists %s.%s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", "rel", "_drainer_repl_mark", ID, ChannelID, Val, ChannelInfo, ID, ChannelID) mk.ExpectExec(regexp.QuoteMeta(CreateMarkDBDDL)). WillReturnResult(sqlmock.NewResult(0, 0)) From 8525698180d4a6c1b86943d6926383e33a1c82d7 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 4 Mar 2020 17:59:36 +0800 Subject: [PATCH 26/64] add some tests --- pkg/plugin/plugins_test.go | 48 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/pkg/plugin/plugins_test.go b/pkg/plugin/plugins_test.go index 2e835a5cf..40453784e 100644 --- a/pkg/plugin/plugins_test.go +++ b/pkg/plugin/plugins_test.go @@ -1,9 +1,14 @@ package plugin import ( + "fmt" + "testing" + "github.com/pingcap/check" ) +func Test(t *testing.T) { check.TestingT(t) } + type PluginSuite struct { } @@ -28,10 +33,49 @@ func (s *STest1) Do() int { func (ps *PluginSuite) TestRegisterPlugin(c *check.C) { hook := &EventHooks{} - s := STest1{32} + s1 := STest1{32} - RegisterPlugin(hook, "test1", s) + RegisterPlugin(hook, "test1", s1) p := hook.GetAllPluginsName() c.Assert(len(p), check.Equals, 1) + RegisterPlugin(hook, "test1", s1) + p = hook.GetAllPluginsName() + c.Assert(len(p), check.Equals, 1) + + s2 := STest1{64} + RegisterPlugin(hook, "test2", s2) + p = hook.GetAllPluginsName() + c.Assert(len(p), check.Equals, 2) + c.Assert(p[0], check.Equals, "test1") + c.Assert(p[1], check.Equals, "test2") +} + +func (ps *PluginSuite) TestTraversePlugin(c *check.C) { + hook := &EventHooks{} + + s1 := STest1{32} + RegisterPlugin(hook, "test1", &s1) + + s2 := STest1{64} + RegisterPlugin(hook, "test2", &s2) + + s3 := STest1{128} + RegisterPlugin(hook, "test3", &s3) + + p := hook.GetAllPluginsName() + c.Assert(len(p), check.Equals, 3) + + ret := 0 + hook.Range(func(k, val interface{}) bool { + c, ok := val.(ITest1) + if !ok { + //ignore type incorrect error + fmt.Printf("ok : %v\n", ok) + return true + } + ret += c.Do() + return true + }) + c.Assert(ret, check.Equals, 32+64+128) } From 3aa0f2ea9e0eecb08b9bb82bef7e29cb672176d9 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 4 Mar 2020 18:02:21 +0800 Subject: [PATCH 27/64] modify test --- pkg/plugin/plugins_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/plugin/plugins_test.go b/pkg/plugin/plugins_test.go index 40453784e..89bc3c9c2 100644 --- a/pkg/plugin/plugins_test.go +++ b/pkg/plugin/plugins_test.go @@ -27,7 +27,7 @@ type STest1 struct { a int } -func (s *STest1) Do() int { +func (s STest1) Do() int { return s.a } @@ -55,13 +55,13 @@ func (ps *PluginSuite) TestTraversePlugin(c *check.C) { hook := &EventHooks{} s1 := STest1{32} - RegisterPlugin(hook, "test1", &s1) + RegisterPlugin(hook, "test1", s1) s2 := STest1{64} - RegisterPlugin(hook, "test2", &s2) + RegisterPlugin(hook, "test2", s2) s3 := STest1{128} - RegisterPlugin(hook, "test3", &s3) + RegisterPlugin(hook, "test3", s3) p := hook.GetAllPluginsName() c.Assert(len(p), check.Equals, 3) From 9f0ae6a61535050c38798955a464629ee9dc048f Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 4 Mar 2020 19:44:26 +0800 Subject: [PATCH 28/64] add some log --- drainer/syncer.go | 2 ++ pkg/loader/load.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/drainer/syncer.go b/drainer/syncer.go index 1ce85ee4f..773c2f91e 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -88,10 +88,12 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( } newPlugin, ok := sym.(func() LoopBack) if !ok { + log.Info("Load plugin error: type is not match.", zap.String("plugin name", name)) continue } plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], name, newPlugin()) + log.Info("Load plugin success.", zap.String("plugin name", name)) } } diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 8b5558007..8fd914b62 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -217,10 +217,12 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { } newPlugin, ok := sym.(func() LoopBack) if !ok { + log.Info("Load plugin error: type is not match.", zap.String("plugin name", name)) continue } plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], name, newPlugin()) + log.Info("Load plugin success.", zap.String("plugin name", name)) } } From b04f2565f023331e67efb9ed4e8507d69ad904c6 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 4 Mar 2020 20:10:54 +0800 Subject: [PATCH 29/64] modify log --- drainer/syncer.go | 4 ++-- pkg/loader/load.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drainer/syncer.go b/drainer/syncer.go index 773c2f91e..3913f3f22 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -88,12 +88,12 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( } newPlugin, ok := sym.(func() LoopBack) if !ok { - log.Info("Load plugin error: type is not match.", zap.String("plugin name", name)) + log.Info("Load plugin error: type is not match.", zap.String("plugin name", name), zap.String("type", "syncer plugin")) continue } plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], name, newPlugin()) - log.Info("Load plugin success.", zap.String("plugin name", name)) + log.Info("Load plugin success.", zap.String("plugin name", name), zap.String("type", "syncer plugin")) } } diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 8fd914b62..e5da52c1e 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -217,12 +217,12 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { } newPlugin, ok := sym.(func() LoopBack) if !ok { - log.Info("Load plugin error: type is not match.", zap.String("plugin name", name)) + log.Info("Load plugin error: type is not match.", zap.String("plugin name", name), zap.String("type", "loader plugin")) continue } plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], name, newPlugin()) - log.Info("Load plugin success.", zap.String("plugin name", name)) + log.Info("Load plugin success.", zap.String("plugin name", name), zap.String("type", "loader plugin")) } } From f0cbd855d48fa6cf6f670af62efea4ce9b5888e5 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 13:12:35 +0800 Subject: [PATCH 30/64] refine code --- drainer/loopbacksync/loopbacksync.go | 5 +++++ drainer/syncer.go | 16 ++++++++++++---- pkg/loader/load.go | 9 ++++++--- pkg/plugin/plugindemo/demo.go | 2 +- pkg/plugin/plugins.go | 2 +- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index b9c8c397c..b14e3babc 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -65,6 +65,11 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st MarkDBName: mdbname, MarkTableName: mtablename, } + if l.SupportPlugin { + l.Hooks = make([]*plugin.EventHooks, 2) + l.Hooks[plugin.SyncerPlugin] = &plugin.EventHooks{} + l.Hooks[plugin.LoaderPlugin] = &plugin.EventHooks{} + } return l } diff --git a/drainer/syncer.go b/drainer/syncer.go index 3913f3f22..63a0eb29b 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -80,23 +80,31 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( syncer.loopbackSync = loopbacksync.NewLoopBackSyncInfo(cfg.ChannelID, cfg.LoopbackControl, cfg.SyncDDL, cfg.PluginPath, cfg.PluginNames, cfg.SupportPlugin, cfg.MarkDBName, cfg.MarkTableName) if syncer.loopbackSync.SupportPlugin { + log.Info("Begin to Load syncer-plugins.") for _, name := range syncer.loopbackSync.PluginNames { sym, err := plugin.LoadPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], syncer.loopbackSync.PluginPath, name) if err != nil { - return nil, err + log.Error("Load plugin failed.", zap.String("plugin name", name), + zap.String("error", err.Error())) + continue } - newPlugin, ok := sym.(func() LoopBack) + + newPlugin, ok := sym.(func() interface{}) if !ok { - log.Info("Load plugin error: type is not match.", zap.String("plugin name", name), zap.String("type", "syncer plugin")) + log.Error("The correct new-function is not provided.", zap.String("plugin name", name), zap.String("type", "syncer plugin")) continue } + plg := newPlugin() + _, ok = plg.(LoopBack) + if !ok { + log.Info("syncer plugin's interface is not implemented.", zap.String("plugin name", name), zap.String("type", "syncer plugin")) + } plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], name, newPlugin()) log.Info("Load plugin success.", zap.String("plugin name", name), zap.String("type", "syncer plugin")) } } - var err error // create schema syncer.schema, err = NewSchema(jobs, false) diff --git a/pkg/loader/load.go b/pkg/loader/load.go index e5da52c1e..bba12d05c 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -209,15 +209,18 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { db.SetMaxIdleConns(opts.workerCount) if s.loopBackSyncInfo.SupportPlugin { + log.Info("Begin to Load loader-plugins.") for _, name := range s.loopBackSyncInfo.PluginNames { sym, err := plugin.LoadPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], s.loopBackSyncInfo.PluginPath, name) if err != nil { - return nil, err + log.Error("Load plugin failed.", zap.String("plugin name", name), + zap.String("error", err.Error())) + continue } - newPlugin, ok := sym.(func() LoopBack) + newPlugin, ok := sym.(func() interface{}) if !ok { - log.Info("Load plugin error: type is not match.", zap.String("plugin name", name), zap.String("type", "loader plugin")) + log.Error("The correct new-function is not provided.", zap.String("plugin name", name), zap.String("type", "loader plugin")) continue } plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], diff --git a/pkg/plugin/plugindemo/demo.go b/pkg/plugin/plugindemo/demo.go index c92327fea..3a089d139 100644 --- a/pkg/plugin/plugindemo/demo.go +++ b/pkg/plugin/plugindemo/demo.go @@ -17,6 +17,6 @@ func (pd PluginDemo) FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) return true, nil } -func NewPlugin() loader.LoopBack { +func NewPlugin() interface{} { return PluginDemo{} } diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 2411863e3..2733e736b 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -51,7 +51,7 @@ func LoadPlugin(eh *EventHooks, path, name string) (plugin.Symbol, error) { fp := path + "/" + name p, err := plugin.Open(fp) if err != nil { - return nil, errors.New(fmt.Sprintf("Open %s failed, err: %s", fp, err.Error())) + return nil, errors.New(fmt.Sprintf("Open %s failed. err: %s", fp, err.Error())) } return p.Lookup(FactorFunc) From 7782196431442ad6e43b0a820e4894d59ab9fe34 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 14:36:36 +0800 Subject: [PATCH 31/64] add some log --- pkg/plugin/plugindemo/demo.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/plugin/plugindemo/demo.go b/pkg/plugin/plugindemo/demo.go index 3a089d139..a66eea933 100644 --- a/pkg/plugin/plugindemo/demo.go +++ b/pkg/plugin/plugindemo/demo.go @@ -1,6 +1,7 @@ package main import ( + "github.com/pingcap/log" "github.com/pingcap/tidb-binlog/drainer/loopbacksync" "github.com/pingcap/tidb-binlog/pkg/loader" ) @@ -9,11 +10,13 @@ type PluginDemo struct{} func (pd PluginDemo) ExtendTxn(tx *loader.Tx, dmls []*loader.DML, info *loopbacksync.LoopBackSync) (*loader.Tx, []*loader.DML) { //do sth + log.Info("i am ExtendTxn") return nil, nil } func (pd PluginDemo) FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) (bool, error) { //do sth + log.Info("i am FilterTxn") return true, nil } From ce64ee40a54a99efc7fbeb0a17039f110a06b016 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 14:59:09 +0800 Subject: [PATCH 32/64] trim string --- drainer/loopbacksync/loopbacksync.go | 4 ++-- drainer/syncer.go | 13 +++++++------ pkg/loader/load.go | 12 +++++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index b14e3babc..b5e90d1c2 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -62,8 +62,8 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st PluginPath: path, PluginNames: names, SupportPlugin: SupportPlug, - MarkDBName: mdbname, - MarkTableName: mtablename, + MarkDBName: strings.TrimSpace(mdbname), + MarkTableName: strings.TrimSpace(mtablename), } if l.SupportPlugin { l.Hooks = make([]*plugin.EventHooks, 2) diff --git a/drainer/syncer.go b/drainer/syncer.go index 63a0eb29b..ce86a4a5e 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -82,27 +82,28 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( if syncer.loopbackSync.SupportPlugin { log.Info("Begin to Load syncer-plugins.") for _, name := range syncer.loopbackSync.PluginNames { + n := strings.TrimSpace(name) sym, err := plugin.LoadPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], - syncer.loopbackSync.PluginPath, name) + syncer.loopbackSync.PluginPath, n) if err != nil { - log.Error("Load plugin failed.", zap.String("plugin name", name), + log.Error("Load plugin failed.", zap.String("plugin name", n), zap.String("error", err.Error())) continue } newPlugin, ok := sym.(func() interface{}) if !ok { - log.Error("The correct new-function is not provided.", zap.String("plugin name", name), zap.String("type", "syncer plugin")) + log.Error("The correct new-function is not provided.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) continue } plg := newPlugin() _, ok = plg.(LoopBack) if !ok { - log.Info("syncer plugin's interface is not implemented.", zap.String("plugin name", name), zap.String("type", "syncer plugin")) + log.Info("syncer plugin's interface is not implemented.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) } plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], - name, newPlugin()) - log.Info("Load plugin success.", zap.String("plugin name", name), zap.String("type", "syncer plugin")) + n, newPlugin()) + log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) } } var err error diff --git a/pkg/loader/load.go b/pkg/loader/load.go index bba12d05c..9a57280a4 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -17,6 +17,7 @@ import ( "context" gosql "database/sql" "fmt" + "strings" "sync" "sync/atomic" "time" @@ -211,21 +212,22 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { if s.loopBackSyncInfo.SupportPlugin { log.Info("Begin to Load loader-plugins.") for _, name := range s.loopBackSyncInfo.PluginNames { + n := strings.TrimSpace(name) sym, err := plugin.LoadPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], - s.loopBackSyncInfo.PluginPath, name) + s.loopBackSyncInfo.PluginPath, n) if err != nil { - log.Error("Load plugin failed.", zap.String("plugin name", name), + log.Error("Load plugin failed.", zap.String("plugin name", n), zap.String("error", err.Error())) continue } newPlugin, ok := sym.(func() interface{}) if !ok { - log.Error("The correct new-function is not provided.", zap.String("plugin name", name), zap.String("type", "loader plugin")) + log.Error("The correct new-function is not provided.", zap.String("plugin name", n), zap.String("type", "loader plugin")) continue } plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], - name, newPlugin()) - log.Info("Load plugin success.", zap.String("plugin name", name), zap.String("type", "loader plugin")) + n, newPlugin()) + log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "loader plugin")) } } From 2df8841e10a4b4547b672554d3500e2146ce8fc5 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 15:04:42 +0800 Subject: [PATCH 33/64] should not conflict with LoopbackControl --- drainer/syncer.go | 4 ++-- pkg/loader/executor.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drainer/syncer.go b/drainer/syncer.go index ce86a4a5e..0dd6e330f 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -408,7 +408,7 @@ ForLoop: } } - if s.loopbackSync != nil && s.loopbackSync.LoopbackControl && !s.loopbackSync.SupportPlugin { + if s.loopbackSync != nil && s.loopbackSync.LoopbackControl { isFilterTransaction, err1 = loopBackStatus(binlog, preWrite, s.schema, s.loopbackSync) if err1 != nil { err = errors.Annotate(err1, "analyze transaction failed") @@ -500,7 +500,7 @@ ForLoop: shouldSkip := false - if !s.cfg.SyncDDL && !s.loopbackSync.SupportPlugin { + if !s.cfg.SyncDDL { log.Info("skip ddl by SyncDDL setting to false", zap.String("schema", schema), zap.String("table", table), zap.String("sql", sql), zap.Int64("commit ts", commitTS)) // A empty sql force it to evict the downstream table info. diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index 6193784ec..8ce84a335 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -139,7 +139,7 @@ func (e *executor) begin() (*Tx, error) { queryHistogramVec: e.queryHistogramVec, } - if e.info != nil && e.info.LoopbackControl && !e.info.SupportPlugin { + if e.info != nil && e.info.LoopbackControl { start := time.Now() err = loopbacksync.UpdateMark(tx.Tx, atomic.AddInt64(&e.info.Index, 1)%((int64)(e.workerCount)), e.info.ChannelID) From 0c764b1fa720d778dd09d2eb73678776a741aab1 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 15:06:41 +0800 Subject: [PATCH 34/64] refine comment --- pkg/plugin/plugins.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 2733e736b..d53ef9b04 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -46,7 +46,7 @@ func (ehs *EventHooks) GetAllPluginsName() []string { return ns } -//LoadPlugin() can load plugin by plugin's name +//LoadPlugin can load plugin by plugin's name func LoadPlugin(eh *EventHooks, path, name string) (plugin.Symbol, error) { fp := path + "/" + name p, err := plugin.Open(fp) @@ -57,7 +57,7 @@ func LoadPlugin(eh *EventHooks, path, name string) (plugin.Symbol, error) { return p.Lookup(FactorFunc) } -//RegisterPlugin() register plugin to EventHooks +//RegisterPlugin register plugin to EventHooks func RegisterPlugin(ehs *EventHooks, name string, plg interface{}) { ehs.SetPlugin(name, plg) } From 76061b58c5fcbd5a7a292f0eeda9ee070ef2adf1 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 15:30:57 +0800 Subject: [PATCH 35/64] add comment --- drainer/filter_txn.go | 1 + pkg/loader/extend_txn.go | 1 + 2 files changed, 2 insertions(+) diff --git a/drainer/filter_txn.go b/drainer/filter_txn.go index fb233cb26..0c26e80f1 100644 --- a/drainer/filter_txn.go +++ b/drainer/filter_txn.go @@ -5,6 +5,7 @@ import ( "github.com/pingcap/tidb-binlog/pkg/loader" ) +// LoopBack is the interface that for syncer-plugin type LoopBack interface { FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) (bool, error) } diff --git a/pkg/loader/extend_txn.go b/pkg/loader/extend_txn.go index 00f39047f..db28589ff 100644 --- a/pkg/loader/extend_txn.go +++ b/pkg/loader/extend_txn.go @@ -4,6 +4,7 @@ import ( "github.com/pingcap/tidb-binlog/drainer/loopbacksync" ) +// LoopBack is the interface that for loader-plugin type LoopBack interface { ExtendTxn(tx *Tx, dmls []*DML, info *loopbacksync.LoopBackSync) (*Tx, []*DML) } From 78e91537d681bb8a150d593e42e7621882c629d2 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 15:45:01 +0800 Subject: [PATCH 36/64] replace errors.New(fmt.Sprintf(...)) with fmt.Errorf(...) --- pkg/plugin/plugins.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index d53ef9b04..e562bd23f 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -1,7 +1,6 @@ package plugin import ( - "errors" "fmt" "plugin" "sync" @@ -51,7 +50,7 @@ func LoadPlugin(eh *EventHooks, path, name string) (plugin.Symbol, error) { fp := path + "/" + name p, err := plugin.Open(fp) if err != nil { - return nil, errors.New(fmt.Sprintf("Open %s failed. err: %s", fp, err.Error())) + return nil, fmt.Errorf("Open %s failed. err: %s", fp, err.Error()) } return p.Lookup(FactorFunc) From 59ad9eed2c3d8ef2a506273aa09f72049539cc1c Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 16:10:44 +0800 Subject: [PATCH 37/64] add comment --- pkg/plugin/plugins.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index e562bd23f..9161cfd46 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -29,6 +29,7 @@ func (ehs *EventHooks) SetPlugin(name string, plg interface{}) *EventHooks { return ehs } +//GetAllPluginsName is get all names of plugin func (ehs *EventHooks) GetAllPluginsName() []string { if ehs == nil { return nil From 8859fd79c0342589d8a4378e2563a7a8f63343d8 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 16:14:28 +0800 Subject: [PATCH 38/64] change RecordId to RecordID --- drainer/loopbacksync/loopbacksync.go | 2 +- pkg/loader/load.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index b5e90d1c2..78835998f 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -49,7 +49,7 @@ type LoopBackSync struct { PluginNames []string Hooks []*plugin.EventHooks SupportPlugin bool - RecordId int + RecordID int } //NewLoopBackSyncInfo return LoopBackSyncInfo objec diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 9a57280a4..4a79f6556 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -204,7 +204,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { cancel: cancel, } - s.loopBackSyncInfo.RecordId = s.workerCount + s.loopBackSyncInfo.RecordID = s.workerCount db.SetMaxOpenConns(opts.workerCount) db.SetMaxIdleConns(opts.workerCount) From f06e03fd9ef64efaffbae0da939e659966b3f06b Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 16:19:39 +0800 Subject: [PATCH 39/64] change SetPlugin to setPlugin --- pkg/plugin/plugins.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 9161cfd46..383887200 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -21,7 +21,7 @@ type EventHooks struct { sync.Map } -func (ehs *EventHooks) SetPlugin(name string, plg interface{}) *EventHooks { +func (ehs *EventHooks) setPlugin(name string, plg interface{}) *EventHooks { if len(name) == 0 || ehs == nil { return ehs } @@ -59,5 +59,5 @@ func LoadPlugin(eh *EventHooks, path, name string) (plugin.Symbol, error) { //RegisterPlugin register plugin to EventHooks func RegisterPlugin(ehs *EventHooks, name string, plg interface{}) { - ehs.SetPlugin(name, plg) + ehs.setPlugin(name, plg) } From 355b6793e044e5c58b7187bc726492c400132908 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 16:22:03 +0800 Subject: [PATCH 40/64] add comment --- pkg/plugin/plugins.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 383887200..cdc4c5524 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -10,7 +10,9 @@ import ( type Kind uint8 const ( + //SyncerPlugin is one kind of Plugin for syncer SyncerPlugin Kind = iota + //LoaderPlugin is one kind of Plugin for loader LoaderPlugin FactorFunc = "NewPlugin" From 9d8f6e3600cb0fbca750eaadf4c3cc74d37d415c Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 16:38:24 +0800 Subject: [PATCH 41/64] refine --- pkg/loader/util.go | 3 +-- pkg/plugin/plugindemo/demo.go | 4 ++++ pkg/plugin/plugins.go | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/loader/util.go b/pkg/loader/util.go index 159f70a16..26f085dbf 100644 --- a/pkg/loader/util.go +++ b/pkg/loader/util.go @@ -115,9 +115,8 @@ func CreateDB(user string, password string, host string, port int, tls *tls.Conf func quoteSchema(schema string, table string) string { if len(schema) == 0 { return fmt.Sprintf("`%s`", escapeName(table)) - } else { - return fmt.Sprintf("`%s`.`%s`", escapeName(schema), escapeName(table)) } + return fmt.Sprintf("`%s`.`%s`", escapeName(schema), escapeName(table)) } func quoteName(name string) string { diff --git a/pkg/plugin/plugindemo/demo.go b/pkg/plugin/plugindemo/demo.go index a66eea933..e991fa848 100644 --- a/pkg/plugin/plugindemo/demo.go +++ b/pkg/plugin/plugindemo/demo.go @@ -6,20 +6,24 @@ import ( "github.com/pingcap/tidb-binlog/pkg/loader" ) +//PluginDemo is a demo struct type PluginDemo struct{} +//ExtendTxn is one of the Hook func (pd PluginDemo) ExtendTxn(tx *loader.Tx, dmls []*loader.DML, info *loopbacksync.LoopBackSync) (*loader.Tx, []*loader.DML) { //do sth log.Info("i am ExtendTxn") return nil, nil } +//FilterTxn is one of the Hook func (pd PluginDemo) FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) (bool, error) { //do sth log.Info("i am FilterTxn") return true, nil } +//NewPlugin is the Factory function of plugin func NewPlugin() interface{} { return PluginDemo{} } diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index cdc4c5524..f9fad533c 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -6,7 +6,7 @@ import ( "sync" ) -//Plugin type we supported currently +//Kind is the plugin's type thant we supported currently type Kind uint8 const ( @@ -14,7 +14,7 @@ const ( SyncerPlugin Kind = iota //LoaderPlugin is one kind of Plugin for loader LoaderPlugin - + //FactorFunc is the factory of all plugins FactorFunc = "NewPlugin" ) From 7a125712bb8536fc31f2c07fcea6b9380764ea34 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 16:40:25 +0800 Subject: [PATCH 42/64] modify comment --- pkg/loader/executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index 8ce84a335..630dfa5d6 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -87,7 +87,7 @@ func (e *executor) execTableBatchRetry(ctx context.Context, dmls []*DML, retryNu return errors.Trace(err) } -// a wrap of *sql.Tx with metrics +// Tx is a wrap of *sql.Tx with metrics type Tx struct { *gosql.Tx queryHistogramVec *prometheus.HistogramVec From 5e2e3ac8b80e140d58c51ddfbe4a5bd2bbb9de10 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 16:49:46 +0800 Subject: [PATCH 43/64] refine --- pkg/loader/executor.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index 630dfa5d6..74f4c45ff 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -169,10 +169,7 @@ func (e *executor) externPoint(t *Tx, dmls []*DML) (*Tx, []*DML) { return true } t, dmls = c.ExtendTxn(t, dmls, e.info) - if dmls == nil { - return false - } - return true + return dmls != nil }) return t, dmls } From d385b8932ea9192f00a919e41e29849e9fd77814 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 17:11:06 +0800 Subject: [PATCH 44/64] refine code --- drainer/config.go | 16 ---------------- drainer/util.go | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/drainer/config.go b/drainer/config.go index f30ea7a46..8664c5827 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -120,22 +120,6 @@ type Config struct { tls *tls.Config } -type sliceNames []string - -func newSliceNames(vals []string, p *[]string) *sliceNames { - *p = vals - return (*sliceNames)(p) -} - -func (s *sliceNames) Set(val string) error { - *s = sliceNames(strings.Split(val, ",")) - return nil -} - -func (s *sliceNames) Get() interface{} { return []string(*s) } - -func (s *sliceNames) String() string { return strings.Join([]string(*s), ",") } - // NewConfig return an instance of configuration func NewConfig() *Config { cfg := &Config{ diff --git a/drainer/util.go b/drainer/util.go index 9543cdf3c..872cd82d5 100644 --- a/drainer/util.go +++ b/drainer/util.go @@ -20,6 +20,7 @@ import ( "os" "path" "sort" + "strings" "sync" "github.com/Shopify/sarama" @@ -191,3 +192,19 @@ func genDrainerID(listenAddr string) (string, error) { return fmt.Sprintf("%s:%s", hostname, port), nil } + +type sliceNames []string + +func newSliceNames(vals []string, p *[]string) *sliceNames { + *p = vals + return (*sliceNames)(p) +} + +func (s *sliceNames) Set(val string) error { + *s = sliceNames(strings.Split(val, ",")) + return nil +} + +func (s *sliceNames) Get() interface{} { return []string(*s) } + +func (s *sliceNames) String() string { return strings.Join([]string(*s), ",") } From f102f7c42b5f8cd22a7a253b16e6788410fdde07 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 5 Mar 2020 17:12:30 +0800 Subject: [PATCH 45/64] handle ci error --- pkg/plugin/plugindemo/demo.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/plugin/plugindemo/demo.go b/pkg/plugin/plugindemo/demo.go index e991fa848..856d92c4b 100644 --- a/pkg/plugin/plugindemo/demo.go +++ b/pkg/plugin/plugindemo/demo.go @@ -27,3 +27,6 @@ func (pd PluginDemo) FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) func NewPlugin() interface{} { return PluginDemo{} } + +var _ PluginDemo +var _ = NewPlugin() From 90c5126a614d71d507693e6ad76b7401b3dfc8c8 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 11:33:26 +0800 Subject: [PATCH 46/64] add init interface --- drainer/filter_txn.go | 4 +++ drainer/loopbacksync/loopbacksync.go | 1 + drainer/syncer.go | 41 ++++++++++++++++++++++++---- pkg/plugin/plugins.go | 2 ++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/drainer/filter_txn.go b/drainer/filter_txn.go index 0c26e80f1..a6223d1f4 100644 --- a/drainer/filter_txn.go +++ b/drainer/filter_txn.go @@ -9,3 +9,7 @@ import ( type LoopBack interface { FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) (bool, error) } + +type SyncerInit interface { + SyncerInit(s *Syncer) error +} diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index 78835998f..aff3c6354 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -69,6 +69,7 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st l.Hooks = make([]*plugin.EventHooks, 2) l.Hooks[plugin.SyncerPlugin] = &plugin.EventHooks{} l.Hooks[plugin.LoaderPlugin] = &plugin.EventHooks{} + l.Hooks[plugin.SyncerInit] = &plugin.EventHooks{} } return l } diff --git a/drainer/syncer.go b/drainer/syncer.go index 0dd6e330f..977392800 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -99,11 +99,23 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( plg := newPlugin() _, ok = plg.(LoopBack) if !ok { - log.Info("syncer plugin's interface is not implemented.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) + log.Info("LoopBack interface is not implemented.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) + } else { + plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], + n, newPlugin()) + log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "syncer plugin"), + zap.String("interface", "LoopBack")) + } + + _, ok = plg.(SyncerInit) + if !ok { + log.Info("SyncerInit interface is not implemented.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) + } else { + plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerInit], + n, newPlugin()) + log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "syncer plugin"), + zap.String("interface", "SyncerInit")) } - plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], - n, newPlugin()) - log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) } } var err error @@ -156,7 +168,26 @@ func createDSyncer(cfg *SyncerConfig, schema *Schema, info *loopbacksync.LoopBac // Start starts to sync. func (s *Syncer) Start() error { - err := s.run() + var err error + if s.loopbackSync.SupportPlugin { + hook := s.loopbackSync.Hooks[plugin.SyncerInit] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(SyncerInit) + if !ok { + return true + } + err = c.SyncerInit(s) + if err != nil { + return false + } + return true + }) + if err != nil { + return errors.Trace(err) + } + } + + err = s.run() return errors.Trace(err) } diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index f9fad533c..bcdad8554 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -12,6 +12,8 @@ type Kind uint8 const ( //SyncerPlugin is one kind of Plugin for syncer SyncerPlugin Kind = iota + //SyncerInit is one kind of Plugin for syncer + SyncerInit //LoaderPlugin is one kind of Plugin for loader LoaderPlugin //FactorFunc is the factory of all plugins From fbd274830c12b71dfc5164dcc54770d7ad12b1d6 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 11:40:25 +0800 Subject: [PATCH 47/64] modify SyncerPlugin to --- drainer/loopbacksync/loopbacksync.go | 2 +- drainer/syncer.go | 8 ++++---- pkg/plugin/plugins.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index aff3c6354..7662b4a73 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -67,7 +67,7 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st } if l.SupportPlugin { l.Hooks = make([]*plugin.EventHooks, 2) - l.Hooks[plugin.SyncerPlugin] = &plugin.EventHooks{} + l.Hooks[plugin.SyncerFilter] = &plugin.EventHooks{} l.Hooks[plugin.LoaderPlugin] = &plugin.EventHooks{} l.Hooks[plugin.SyncerInit] = &plugin.EventHooks{} } diff --git a/drainer/syncer.go b/drainer/syncer.go index 977392800..dda343cf7 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -83,7 +83,7 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( log.Info("Begin to Load syncer-plugins.") for _, name := range syncer.loopbackSync.PluginNames { n := strings.TrimSpace(name) - sym, err := plugin.LoadPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], + sym, err := plugin.LoadPlugin(syncer.loopbackSync.Hooks[plugin.SyncerFilter], syncer.loopbackSync.PluginPath, n) if err != nil { log.Error("Load plugin failed.", zap.String("plugin name", n), @@ -101,7 +101,7 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( if !ok { log.Info("LoopBack interface is not implemented.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) } else { - plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerPlugin], + plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerFilter], n, newPlugin()) log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "syncer plugin"), zap.String("interface", "LoopBack")) @@ -420,7 +420,7 @@ ForLoop: var err1 error if s.loopbackSync.SupportPlugin { - hook := s.loopbackSync.Hooks[plugin.SyncerPlugin] + hook := s.loopbackSync.Hooks[plugin.SyncerFilter] var txn *loader.Txn txn, err1 = translator.TiBinlogToTxn(s.schema, "", "", binlog, preWrite, false) hook.Range(func(k, val interface{}) bool { @@ -503,7 +503,7 @@ ForLoop: Table: table, SQL: string(binlog.GetDdlQuery()), } - hook := s.loopbackSync.Hooks[plugin.SyncerPlugin] + hook := s.loopbackSync.Hooks[plugin.SyncerFilter] hook.Range(func(k, val interface{}) bool { c, ok := val.(LoopBack) if !ok { diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index bcdad8554..1d5111d3e 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -11,7 +11,7 @@ type Kind uint8 const ( //SyncerPlugin is one kind of Plugin for syncer - SyncerPlugin Kind = iota + SyncerFilter Kind = iota //SyncerInit is one kind of Plugin for syncer SyncerInit //LoaderPlugin is one kind of Plugin for loader From 7df02f6c84834c89680d747b088f3997f9f7fcfc Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 11:41:42 +0800 Subject: [PATCH 48/64] modify SyncerPlugin to SyncerFilter --- pkg/plugin/plugins.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 1d5111d3e..3199fa381 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -10,7 +10,7 @@ import ( type Kind uint8 const ( - //SyncerPlugin is one kind of Plugin for syncer + //SyncerFilter is one kind of Plugin for syncer SyncerFilter Kind = iota //SyncerInit is one kind of Plugin for syncer SyncerInit From 2893d2fe1c8f7dbe90069854ae3048047baf0383 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 11:48:40 +0800 Subject: [PATCH 49/64] modify Loopback to SyncerFilter --- drainer/filter_txn.go | 4 ++-- drainer/syncer.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drainer/filter_txn.go b/drainer/filter_txn.go index a6223d1f4..2d5beffdd 100644 --- a/drainer/filter_txn.go +++ b/drainer/filter_txn.go @@ -5,8 +5,8 @@ import ( "github.com/pingcap/tidb-binlog/pkg/loader" ) -// LoopBack is the interface that for syncer-plugin -type LoopBack interface { +// SyncerFilter is the interface that for syncer-plugin +type SyncerFilter interface { FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) (bool, error) } diff --git a/drainer/syncer.go b/drainer/syncer.go index dda343cf7..371116bc6 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -97,14 +97,14 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( continue } plg := newPlugin() - _, ok = plg.(LoopBack) + _, ok = plg.(SyncerFilter) if !ok { - log.Info("LoopBack interface is not implemented.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) + log.Info("SyncerFilter interface is not implemented.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) } else { plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerFilter], n, newPlugin()) log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "syncer plugin"), - zap.String("interface", "LoopBack")) + zap.String("interface", "SyncerFilter")) } _, ok = plg.(SyncerInit) @@ -424,7 +424,7 @@ ForLoop: var txn *loader.Txn txn, err1 = translator.TiBinlogToTxn(s.schema, "", "", binlog, preWrite, false) hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoopBack) + c, ok := val.(SyncerFilter) if !ok { return true } @@ -505,7 +505,7 @@ ForLoop: } hook := s.loopbackSync.Hooks[plugin.SyncerFilter] hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoopBack) + c, ok := val.(SyncerFilter) if !ok { return true } From ed5852fae6456cbb74e525700f6af06255050fae Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 11:57:38 +0800 Subject: [PATCH 50/64] modify --- drainer/loopbacksync/loopbacksync.go | 2 +- pkg/loader/executor.go | 2 +- pkg/loader/load.go | 4 ++-- pkg/plugin/plugins.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index 7662b4a73..306d0293b 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -68,7 +68,7 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st if l.SupportPlugin { l.Hooks = make([]*plugin.EventHooks, 2) l.Hooks[plugin.SyncerFilter] = &plugin.EventHooks{} - l.Hooks[plugin.LoaderPlugin] = &plugin.EventHooks{} + l.Hooks[plugin.ExecutorExtend] = &plugin.EventHooks{} l.Hooks[plugin.SyncerInit] = &plugin.EventHooks{} } return l diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index 74f4c45ff..ef4fb7ec5 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -161,7 +161,7 @@ func (e *executor) begin() (*Tx, error) { // return a wrap of sql.Tx func (e *executor) externPoint(t *Tx, dmls []*DML) (*Tx, []*DML) { - hook := e.info.Hooks[plugin.LoaderPlugin] + hook := e.info.Hooks[plugin.ExecutorExtend] hook.Range(func(k, val interface{}) bool { c, ok := val.(LoopBack) if !ok { diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 4a79f6556..e6f5ba189 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -213,7 +213,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { log.Info("Begin to Load loader-plugins.") for _, name := range s.loopBackSyncInfo.PluginNames { n := strings.TrimSpace(name) - sym, err := plugin.LoadPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], + sym, err := plugin.LoadPlugin(s.loopBackSyncInfo.Hooks[plugin.ExecutorExtend], s.loopBackSyncInfo.PluginPath, n) if err != nil { log.Error("Load plugin failed.", zap.String("plugin name", n), @@ -225,7 +225,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { log.Error("The correct new-function is not provided.", zap.String("plugin name", n), zap.String("type", "loader plugin")) continue } - plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderPlugin], + plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.ExecutorExtend], n, newPlugin()) log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "loader plugin")) } diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 3199fa381..c08ec2971 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -14,8 +14,8 @@ const ( SyncerFilter Kind = iota //SyncerInit is one kind of Plugin for syncer SyncerInit - //LoaderPlugin is one kind of Plugin for loader - LoaderPlugin + //ExecutorExtend is one kind of Plugin for loader + ExecutorExtend //FactorFunc is the factory of all plugins FactorFunc = "NewPlugin" ) From 518415db41f8af3fc47b75821ac9f69dda9288c6 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 12:19:27 +0800 Subject: [PATCH 51/64] add --- drainer/{filter_txn.go => isyncer.go} | 0 pkg/loader/executor.go | 2 +- pkg/loader/{extend_txn.go => iloader.go} | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename drainer/{filter_txn.go => isyncer.go} (100%) rename pkg/loader/{extend_txn.go => iloader.go} (87%) diff --git a/drainer/filter_txn.go b/drainer/isyncer.go similarity index 100% rename from drainer/filter_txn.go rename to drainer/isyncer.go diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index ef4fb7ec5..bfb762e14 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -163,7 +163,7 @@ func (e *executor) begin() (*Tx, error) { func (e *executor) externPoint(t *Tx, dmls []*DML) (*Tx, []*DML) { hook := e.info.Hooks[plugin.ExecutorExtend] hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoopBack) + c, ok := val.(ExecutorExtend) if !ok { //ignore type incorrect error return true diff --git a/pkg/loader/extend_txn.go b/pkg/loader/iloader.go similarity index 87% rename from pkg/loader/extend_txn.go rename to pkg/loader/iloader.go index db28589ff..e9ea810a2 100644 --- a/pkg/loader/extend_txn.go +++ b/pkg/loader/iloader.go @@ -5,6 +5,6 @@ import ( ) // LoopBack is the interface that for loader-plugin -type LoopBack interface { +type ExecutorExtend interface { ExtendTxn(tx *Tx, dmls []*DML, info *loopbacksync.LoopBackSync) (*Tx, []*DML) } From 6ac7d53ad39dd913cc10c0c9f9c19f29fa0f666e Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 12:21:56 +0800 Subject: [PATCH 52/64] refine --- drainer/isyncer.go | 1 + pkg/loader/iloader.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/drainer/isyncer.go b/drainer/isyncer.go index 2d5beffdd..2a78ee84b 100644 --- a/drainer/isyncer.go +++ b/drainer/isyncer.go @@ -10,6 +10,7 @@ type SyncerFilter interface { FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) (bool, error) } +// SyncerInit is the interface that for syncer-plugin type SyncerInit interface { SyncerInit(s *Syncer) error } diff --git a/pkg/loader/iloader.go b/pkg/loader/iloader.go index e9ea810a2..5a5fd4a58 100644 --- a/pkg/loader/iloader.go +++ b/pkg/loader/iloader.go @@ -4,7 +4,7 @@ import ( "github.com/pingcap/tidb-binlog/drainer/loopbacksync" ) -// LoopBack is the interface that for loader-plugin +// ExecutorExtend is the interface that for loader-plugin type ExecutorExtend interface { ExtendTxn(tx *Tx, dmls []*DML, info *loopbacksync.LoopBackSync) (*Tx, []*DML) } From 0929e8da216a5e10ecb6b7bb395dda86d4755389 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 14:41:20 +0800 Subject: [PATCH 53/64] add LoaderInit --- drainer/syncer.go | 14 ++++++-------- pkg/loader/iloader.go | 5 +++++ pkg/loader/load.go | 41 ++++++++++++++++++++++++++++++++++++++--- pkg/plugin/plugins.go | 2 ++ 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/drainer/syncer.go b/drainer/syncer.go index 371116bc6..253e96163 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -99,22 +99,20 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( plg := newPlugin() _, ok = plg.(SyncerFilter) if !ok { - log.Info("SyncerFilter interface is not implemented.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) + log.Info("SyncerFilter interface is not implemented.", zap.String("plugin name", n)) } else { plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerFilter], - n, newPlugin()) - log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "syncer plugin"), - zap.String("interface", "SyncerFilter")) + n, plg) + log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "SyncerFilter")) } _, ok = plg.(SyncerInit) if !ok { - log.Info("SyncerInit interface is not implemented.", zap.String("plugin name", n), zap.String("type", "syncer plugin")) + log.Info("SyncerInit interface is not implemented.", zap.String("plugin name", n)) } else { plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerInit], - n, newPlugin()) - log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "syncer plugin"), - zap.String("interface", "SyncerInit")) + n, plg) + log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "SyncerInit")) } } } diff --git a/pkg/loader/iloader.go b/pkg/loader/iloader.go index 5a5fd4a58..5d9b691b2 100644 --- a/pkg/loader/iloader.go +++ b/pkg/loader/iloader.go @@ -8,3 +8,8 @@ import ( type ExecutorExtend interface { ExtendTxn(tx *Tx, dmls []*DML, info *loopbacksync.LoopBackSync) (*Tx, []*DML) } + +// LoaderInit is the interface that for syncer-plugin +type LoaderInit interface { + LoaderInit(s *loaderImpl) error +} diff --git a/pkg/loader/load.go b/pkg/loader/load.go index e6f5ba189..64afcf259 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -225,9 +225,25 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { log.Error("The correct new-function is not provided.", zap.String("plugin name", n), zap.String("type", "loader plugin")) continue } - plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.ExecutorExtend], - n, newPlugin()) - log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("type", "loader plugin")) + + plg := newPlugin() + _, ok = plg.(ExecutorExtend) + if !ok { + log.Info("ExecutorExtend interface is not implemented.", zap.String("plugin name", n)) + } else { + plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.ExecutorExtend], + n, plg) + log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "ExecutorExtend")) + } + + _, ok = plg.(LoaderInit) + if !ok { + log.Info("LoaderInit interface is not implemented.", zap.String("plugin name", n)) + } else { + plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.ExecutorExtend], + n, plg) + log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "LoaderInit")) + } } } @@ -532,6 +548,25 @@ func (s *loaderImpl) Run() error { close(s.successTxn) }() + var err error + if s.loopBackSyncInfo.SupportPlugin { + hook := s.loopBackSyncInfo.Hooks[plugin.SyncerInit] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoaderInit) + if !ok { + return true + } + err = c.LoaderInit(s) + if err != nil { + return false + } + return true + }) + if err != nil { + return errors.Trace(err) + } + } + if s.loopBackSyncInfo != nil && s.loopBackSyncInfo.LoopbackControl { if err := s.initMarkTable(); err != nil { return errors.Trace(err) diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index c08ec2971..171a31eee 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -16,6 +16,8 @@ const ( SyncerInit //ExecutorExtend is one kind of Plugin for loader ExecutorExtend + //LoaderInit is one kind of Plugin for loader + LoaderInit //FactorFunc is the factory of all plugins FactorFunc = "NewPlugin" ) From 124a885439dc6961a90190fe0df13c89f3746fed Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 14:55:16 +0800 Subject: [PATCH 54/64] add loaderdestroy --- drainer/loopbacksync/loopbacksync.go | 36 ++++++++++++---------------- drainer/syncer.go | 30 +---------------------- pkg/loader/iloader.go | 5 ++++ pkg/loader/load.go | 30 ++++++++++++++++++++++- pkg/plugin/plugins.go | 4 ++-- 5 files changed, 52 insertions(+), 53 deletions(-) diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index 306d0293b..32f1b5109 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -66,10 +66,13 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st MarkTableName: strings.TrimSpace(mtablename), } if l.SupportPlugin { - l.Hooks = make([]*plugin.EventHooks, 2) + l.Hooks = make([]*plugin.EventHooks, 4) l.Hooks[plugin.SyncerFilter] = &plugin.EventHooks{} + l.Hooks[plugin.ExecutorExtend] = &plugin.EventHooks{} - l.Hooks[plugin.SyncerInit] = &plugin.EventHooks{} + l.Hooks[plugin.LoaderInit] = &plugin.EventHooks{} + l.Hooks[plugin.LoaderDestroy] = &plugin.EventHooks{} + } return l } @@ -78,26 +81,17 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st func CreateMarkTable(db *sql.DB, mdbname, mtablename string) error { // CreateMarkDBDDL is DDL to create the database of mark table. var err error - if len(mdbname) == 0 { - // CreateMarkTableDDL is the DDL to create the mark table. - var CreateMarkTableDDL string = fmt.Sprintf("CREATE TABLE If Not Exists %s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", mtablename, ID, ChannelID, Val, ChannelInfo, ID, ChannelID) - _, err = db.Exec(CreateMarkTableDDL) - if err != nil { - return errors.Annotate(err, "failed to create mark table") - } - } else { - var CreateMarkDBDDL = fmt.Sprintf("create database IF NOT EXISTS %s;", mdbname) - _, err = db.Exec(CreateMarkDBDDL) - if err != nil { - return errors.Annotate(err, "failed to create mark db") - } + var CreateMarkDBDDL = fmt.Sprintf("create database IF NOT EXISTS %s;", mdbname) + _, err = db.Exec(CreateMarkDBDDL) + if err != nil { + return errors.Annotate(err, "failed to create mark db") + } - // CreateMarkTableDDL is the DDL to create the mark table. - var CreateMarkTableDDL string = fmt.Sprintf("CREATE TABLE If Not Exists %s.%s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", mdbname, mtablename, ID, ChannelID, Val, ChannelInfo, ID, ChannelID) - _, err = db.Exec(CreateMarkTableDDL) - if err != nil { - return errors.Annotate(err, "failed to create mark table") - } + // CreateMarkTableDDL is the DDL to create the mark table. + var CreateMarkTableDDL string = fmt.Sprintf("CREATE TABLE If Not Exists %s.%s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", mdbname, mtablename, ID, ChannelID, Val, ChannelInfo, ID, ChannelID) + _, err = db.Exec(CreateMarkTableDDL) + if err != nil { + return errors.Annotate(err, "failed to create mark table") } return nil diff --git a/drainer/syncer.go b/drainer/syncer.go index 253e96163..5e463b4f0 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -105,15 +105,6 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( n, plg) log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "SyncerFilter")) } - - _, ok = plg.(SyncerInit) - if !ok { - log.Info("SyncerInit interface is not implemented.", zap.String("plugin name", n)) - } else { - plugin.RegisterPlugin(syncer.loopbackSync.Hooks[plugin.SyncerInit], - n, plg) - log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "SyncerInit")) - } } } var err error @@ -166,26 +157,7 @@ func createDSyncer(cfg *SyncerConfig, schema *Schema, info *loopbacksync.LoopBac // Start starts to sync. func (s *Syncer) Start() error { - var err error - if s.loopbackSync.SupportPlugin { - hook := s.loopbackSync.Hooks[plugin.SyncerInit] - hook.Range(func(k, val interface{}) bool { - c, ok := val.(SyncerInit) - if !ok { - return true - } - err = c.SyncerInit(s) - if err != nil { - return false - } - return true - }) - if err != nil { - return errors.Trace(err) - } - } - - err = s.run() + err := s.run() return errors.Trace(err) } diff --git a/pkg/loader/iloader.go b/pkg/loader/iloader.go index 5d9b691b2..686d795d5 100644 --- a/pkg/loader/iloader.go +++ b/pkg/loader/iloader.go @@ -13,3 +13,8 @@ type ExecutorExtend interface { type LoaderInit interface { LoaderInit(s *loaderImpl) error } + +// LoaderInit is the interface that for syncer-plugin +type LoaderDestroy interface { + LoaderDestroy(s *loaderImpl) error +} diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 64afcf259..627809974 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -244,6 +244,15 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { n, plg) log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "LoaderInit")) } + + _, ok = plg.(LoaderDestroy) + if !ok { + log.Info("LoaderDestroy interface is not implemented.", zap.String("plugin name", n)) + } else { + plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderDestroy], + n, plg) + log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "LoaderDestroy")) + } } } @@ -550,7 +559,7 @@ func (s *loaderImpl) Run() error { var err error if s.loopBackSyncInfo.SupportPlugin { - hook := s.loopBackSyncInfo.Hooks[plugin.SyncerInit] + hook := s.loopBackSyncInfo.Hooks[plugin.LoaderInit] hook.Range(func(k, val interface{}) bool { c, ok := val.(LoaderInit) if !ok { @@ -625,6 +634,25 @@ func (s *loaderImpl) Run() error { } } } + + if s.loopBackSyncInfo.SupportPlugin { + hook := s.loopBackSyncInfo.Hooks[plugin.LoaderDestroy] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(LoaderDestroy) + if !ok { + return true + } + err = c.LoaderDestroy(s) + if err != nil { + return false + } + return true + }) + if err != nil { + return errors.Trace(err) + } + } + return nil } // groupDMLs group DMLs by table in batchByTbls and diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 171a31eee..dc74b5260 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -12,12 +12,12 @@ type Kind uint8 const ( //SyncerFilter is one kind of Plugin for syncer SyncerFilter Kind = iota - //SyncerInit is one kind of Plugin for syncer - SyncerInit //ExecutorExtend is one kind of Plugin for loader ExecutorExtend //LoaderInit is one kind of Plugin for loader LoaderInit + //LoaderDestroy is one kind of Plugin for loader + LoaderDestroy //FactorFunc is the factory of all plugins FactorFunc = "NewPlugin" ) From ce067e31ffd9eb43501bb7760a49abd67047ed9c Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 14:59:03 +0800 Subject: [PATCH 55/64] refine --- pkg/loader/iloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/loader/iloader.go b/pkg/loader/iloader.go index 686d795d5..b27e427ff 100644 --- a/pkg/loader/iloader.go +++ b/pkg/loader/iloader.go @@ -14,7 +14,7 @@ type LoaderInit interface { LoaderInit(s *loaderImpl) error } -// LoaderInit is the interface that for syncer-plugin +// LoaderDestroy is the interface that for syncer-plugin type LoaderDestroy interface { LoaderDestroy(s *loaderImpl) error } From 8b5fbd9bba92e1bf0d3cdbf2a22e66a0b4165f90 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 15:05:34 +0800 Subject: [PATCH 56/64] modify --- pkg/loader/iloader.go | 10 +++++----- pkg/loader/load.go | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/loader/iloader.go b/pkg/loader/iloader.go index b27e427ff..3f40b14bc 100644 --- a/pkg/loader/iloader.go +++ b/pkg/loader/iloader.go @@ -4,17 +4,17 @@ import ( "github.com/pingcap/tidb-binlog/drainer/loopbacksync" ) -// ExecutorExtend is the interface that for loader-plugin +// ExecutorExtend is the interface for loader plugin type ExecutorExtend interface { ExtendTxn(tx *Tx, dmls []*DML, info *loopbacksync.LoopBackSync) (*Tx, []*DML) } -// LoaderInit is the interface that for syncer-plugin -type LoaderInit interface { +// Init is the interface for loader plugin +type Init interface { LoaderInit(s *loaderImpl) error } -// LoaderDestroy is the interface that for syncer-plugin -type LoaderDestroy interface { +// Destroy is the interface that for loader-plugin +type Destroy interface { LoaderDestroy(s *loaderImpl) error } diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 627809974..3a0e9b2e3 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -236,7 +236,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "ExecutorExtend")) } - _, ok = plg.(LoaderInit) + _, ok = plg.(Init) if !ok { log.Info("LoaderInit interface is not implemented.", zap.String("plugin name", n)) } else { @@ -245,7 +245,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "LoaderInit")) } - _, ok = plg.(LoaderDestroy) + _, ok = plg.(Destroy) if !ok { log.Info("LoaderDestroy interface is not implemented.", zap.String("plugin name", n)) } else { @@ -561,7 +561,7 @@ func (s *loaderImpl) Run() error { if s.loopBackSyncInfo.SupportPlugin { hook := s.loopBackSyncInfo.Hooks[plugin.LoaderInit] hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoaderInit) + c, ok := val.(Init) if !ok { return true } @@ -638,7 +638,7 @@ func (s *loaderImpl) Run() error { if s.loopBackSyncInfo.SupportPlugin { hook := s.loopBackSyncInfo.Hooks[plugin.LoaderDestroy] hook.Range(func(k, val interface{}) bool { - c, ok := val.(LoaderDestroy) + c, ok := val.(Destroy) if !ok { return true } From 8a2d0a711f384eb7fed8623b5bd2594a2ab47adf Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 15:09:52 +0800 Subject: [PATCH 57/64] refine --- pkg/loader/load.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 3a0e9b2e3..35c222e2b 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -566,10 +566,7 @@ func (s *loaderImpl) Run() error { return true } err = c.LoaderInit(s) - if err != nil { - return false - } - return true + return err == nil }) if err != nil { return errors.Trace(err) @@ -643,10 +640,7 @@ func (s *loaderImpl) Run() error { return true } err = c.LoaderDestroy(s) - if err != nil { - return false - } - return true + return err == nil }) if err != nil { return errors.Trace(err) From 8c71d4a5ee4ae82579426f02769db6973435896c Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 15:14:29 +0800 Subject: [PATCH 58/64] reine --- pkg/loader/load.go | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 35c222e2b..191318fb7 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -557,6 +557,24 @@ func (s *loaderImpl) Run() error { close(s.successTxn) }() + defer func() { + if s.loopBackSyncInfo.SupportPlugin { + var err error + hook := s.loopBackSyncInfo.Hooks[plugin.LoaderDestroy] + hook.Range(func(k, val interface{}) bool { + c, ok := val.(Destroy) + if !ok { + return true + } + err = c.LoaderDestroy(s) + return err == nil + }) + if err != nil { + log.Error(errors.Trace(err).Error()) + } + } + }() + var err error if s.loopBackSyncInfo.SupportPlugin { hook := s.loopBackSyncInfo.Hooks[plugin.LoaderInit] @@ -631,22 +649,6 @@ func (s *loaderImpl) Run() error { } } } - - if s.loopBackSyncInfo.SupportPlugin { - hook := s.loopBackSyncInfo.Hooks[plugin.LoaderDestroy] - hook.Range(func(k, val interface{}) bool { - c, ok := val.(Destroy) - if !ok { - return true - } - err = c.LoaderDestroy(s) - return err == nil - }) - if err != nil { - return errors.Trace(err) - } - } - return nil } // groupDMLs group DMLs by table in batchByTbls and From 1d33874e5c7ecddb7d649c320283e1e85825fba4 Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 16:03:35 +0800 Subject: [PATCH 59/64] refine --- drainer/isyncer.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drainer/isyncer.go b/drainer/isyncer.go index 2a78ee84b..9e22fbb9f 100644 --- a/drainer/isyncer.go +++ b/drainer/isyncer.go @@ -9,8 +9,3 @@ import ( type SyncerFilter interface { FilterTxn(txn *loader.Txn, info *loopbacksync.LoopBackSync) (bool, error) } - -// SyncerInit is the interface that for syncer-plugin -type SyncerInit interface { - SyncerInit(s *Syncer) error -} From c99c28c8c7aa4a12831cdd6bb4644f87ee6a358b Mon Sep 17 00:00:00 2001 From: tsthght Date: Wed, 11 Mar 2020 18:34:10 +0800 Subject: [PATCH 60/64] modify --- pkg/loader/iloader.go | 5 +++-- pkg/loader/load.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/loader/iloader.go b/pkg/loader/iloader.go index 3f40b14bc..514c1cb73 100644 --- a/pkg/loader/iloader.go +++ b/pkg/loader/iloader.go @@ -1,6 +1,7 @@ package loader import ( + gosql "database/sql" "github.com/pingcap/tidb-binlog/drainer/loopbacksync" ) @@ -11,10 +12,10 @@ type ExecutorExtend interface { // Init is the interface for loader plugin type Init interface { - LoaderInit(s *loaderImpl) error + LoaderInit(db *gosql.DB, info *loopbacksync.LoopBackSync) error } // Destroy is the interface that for loader-plugin type Destroy interface { - LoaderDestroy(s *loaderImpl) error + LoaderDestroy(db *gosql.DB, info *loopbacksync.LoopBackSync) error } diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 191318fb7..6ba53d2f5 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -566,7 +566,7 @@ func (s *loaderImpl) Run() error { if !ok { return true } - err = c.LoaderDestroy(s) + err = c.LoaderDestroy(s.db, s.loopBackSyncInfo) return err == nil }) if err != nil { @@ -583,7 +583,7 @@ func (s *loaderImpl) Run() error { if !ok { return true } - err = c.LoaderInit(s) + err = c.LoaderInit(s.db, s.loopBackSyncInfo) return err == nil }) if err != nil { From cd06a340afcc6563d64cffccdf85f03c2894aa6d Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 12 Mar 2020 11:51:00 +0800 Subject: [PATCH 61/64] refine --- pkg/loader/executor.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/loader/executor.go b/pkg/loader/executor.go index bfb762e14..92485bba4 100644 --- a/pkg/loader/executor.go +++ b/pkg/loader/executor.go @@ -186,9 +186,11 @@ func (e *executor) bulkDelete(deletes []*DML) error { return errors.Trace(err) } - tx, deletes = e.externPoint(tx, deletes) - if len(deletes) == 0 { - return nil + if e.info.SupportPlugin { + tx, deletes = e.externPoint(tx, deletes) + if len(deletes) == 0 { + return nil + } } argss := make([]interface{}, 0, len(deletes)) @@ -219,9 +221,11 @@ func (e *executor) bulkReplace(inserts []*DML) error { return errors.Trace(err) } - tx, inserts = e.externPoint(tx, inserts) - if len(inserts) == 0 { - return nil + if e.info.SupportPlugin { + tx, inserts = e.externPoint(tx, inserts) + if len(inserts) == 0 { + return nil + } } info := inserts[0].info @@ -375,9 +379,11 @@ func (e *executor) singleExec(dmls []*DML, safeMode bool) error { return errors.Trace(err) } - tx, dmls = e.externPoint(tx, dmls) - if len(dmls) == 0 { - return nil + if e.info.SupportPlugin { + tx, dmls = e.externPoint(tx, dmls) + if len(dmls) == 0 { + return nil + } } for _, dml := range dmls { From 613567d2ff430750de1a3d7bea6a1e8351a60798 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 12 Mar 2020 11:56:48 +0800 Subject: [PATCH 62/64] add log --- drainer/syncer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drainer/syncer.go b/drainer/syncer.go index 5e463b4f0..243f36387 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -405,6 +405,7 @@ ForLoop: return true }) if err1 != nil { + log.Warn("FilterTxn return error", zap.String("error", err1.Error())) break ForLoop } } @@ -486,6 +487,7 @@ ForLoop: return true }) if err1 != nil { + log.Warn("FilterTxn return error", zap.String("error", err1.Error())) break ForLoop } if isFilterTransaction { From edecd470ba682d2f78ad8e6412c5168e3ed9c177 Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 12 Mar 2020 11:58:23 +0800 Subject: [PATCH 63/64] add logs --- drainer/loopbacksync/loopbacksync.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drainer/loopbacksync/loopbacksync.go b/drainer/loopbacksync/loopbacksync.go index 32f1b5109..1a59f071e 100644 --- a/drainer/loopbacksync/loopbacksync.go +++ b/drainer/loopbacksync/loopbacksync.go @@ -79,17 +79,16 @@ func NewLoopBackSyncInfo(ChannelID int64, LoopbackControl, SyncDDL bool, path st // CreateMarkTable create the db and table if need. func CreateMarkTable(db *sql.DB, mdbname, mtablename string) error { - // CreateMarkDBDDL is DDL to create the database of mark table. var err error - var CreateMarkDBDDL = fmt.Sprintf("create database IF NOT EXISTS %s;", mdbname) - _, err = db.Exec(CreateMarkDBDDL) + var createMarkDBDDL = fmt.Sprintf("create database IF NOT EXISTS %s;", mdbname) + _, err = db.Exec(createMarkDBDDL) if err != nil { return errors.Annotate(err, "failed to create mark db") } // CreateMarkTableDDL is the DDL to create the mark table. - var CreateMarkTableDDL string = fmt.Sprintf("CREATE TABLE If Not Exists %s.%s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", mdbname, mtablename, ID, ChannelID, Val, ChannelInfo, ID, ChannelID) - _, err = db.Exec(CreateMarkTableDDL) + var createMarkTableDDL string = fmt.Sprintf("CREATE TABLE If Not Exists %s.%s (%s bigint not null,%s bigint not null DEFAULT 0, %s bigint DEFAULT 0, %s varchar(64) ,PRIMARY KEY (%s,%s));", mdbname, mtablename, ID, ChannelID, Val, ChannelInfo, ID, ChannelID) + _, err = db.Exec(createMarkTableDDL) if err != nil { return errors.Annotate(err, "failed to create mark table") } From 016b67f89dfdf9293437323c9bab3fcfdaf82bee Mon Sep 17 00:00:00 2001 From: tsthght Date: Thu, 12 Mar 2020 15:09:46 +0800 Subject: [PATCH 64/64] Get rid of useless parameters --- drainer/syncer.go | 3 +-- pkg/loader/load.go | 5 ++--- pkg/plugin/plugins.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/drainer/syncer.go b/drainer/syncer.go index 243f36387..8e9badbe8 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -83,8 +83,7 @@ func NewSyncer(cp checkpoint.CheckPoint, cfg *SyncerConfig, jobs []*model.Job) ( log.Info("Begin to Load syncer-plugins.") for _, name := range syncer.loopbackSync.PluginNames { n := strings.TrimSpace(name) - sym, err := plugin.LoadPlugin(syncer.loopbackSync.Hooks[plugin.SyncerFilter], - syncer.loopbackSync.PluginPath, n) + sym, err := plugin.LoadPlugin(syncer.loopbackSync.PluginPath, n) if err != nil { log.Error("Load plugin failed.", zap.String("plugin name", n), zap.String("error", err.Error())) diff --git a/pkg/loader/load.go b/pkg/loader/load.go index 6ba53d2f5..66c39983c 100644 --- a/pkg/loader/load.go +++ b/pkg/loader/load.go @@ -213,8 +213,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { log.Info("Begin to Load loader-plugins.") for _, name := range s.loopBackSyncInfo.PluginNames { n := strings.TrimSpace(name) - sym, err := plugin.LoadPlugin(s.loopBackSyncInfo.Hooks[plugin.ExecutorExtend], - s.loopBackSyncInfo.PluginPath, n) + sym, err := plugin.LoadPlugin(s.loopBackSyncInfo.PluginPath, n) if err != nil { log.Error("Load plugin failed.", zap.String("plugin name", n), zap.String("error", err.Error())) @@ -240,7 +239,7 @@ func NewLoader(db *gosql.DB, opt ...Option) (Loader, error) { if !ok { log.Info("LoaderInit interface is not implemented.", zap.String("plugin name", n)) } else { - plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.ExecutorExtend], + plugin.RegisterPlugin(s.loopBackSyncInfo.Hooks[plugin.LoaderInit], n, plg) log.Info("Load plugin success.", zap.String("plugin name", n), zap.String("interface", "LoaderInit")) } diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index dc74b5260..5f0a66596 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -53,7 +53,7 @@ func (ehs *EventHooks) GetAllPluginsName() []string { } //LoadPlugin can load plugin by plugin's name -func LoadPlugin(eh *EventHooks, path, name string) (plugin.Symbol, error) { +func LoadPlugin(path, name string) (plugin.Symbol, error) { fp := path + "/" + name p, err := plugin.Open(fp) if err != nil {