diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4ffe8f7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,28 @@ +name: release + +on: + push: + tags: + - 'v*' + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: '1.17' + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..5a4e57e --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,52 @@ +env: + - GO111MODULE=on +before: + hooks: + - go mod tidy +builds: + - env: + - CGO_ENABLED=0 + - GO111MODULE=on + goos: + - linux + - darwin + - windows + goarch: + - '386' + - amd64 + - arm + - arm64 + goarm: + - '7' + ignore: + - goos: darwin + goarch: arm + - goos: darwin + goarch: '386' + - goos: windows + goarch: arm64 + mod_timestamp: '{{ .CommitTimestamp }}' + flags: + - -trimpath + ldflags: + - -s -w + +checksum: + name_template: '{{ .ProjectName }}_checksums.txt' +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + - fix typo + - Merge pull request + - Merge branch + - Merge remote-tracking + - go mod tidy + +archives: + - name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' + format_overrides: + - goos: windows + format: zip diff --git a/README.md b/README.md new file mode 100644 index 0000000..e7d3b4d --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +go-cqhttp leveldb v3 迁移工具 +=============================== + +## 安装 + +你可以下载 [已经编译好的二进制文件](https://github.com/RomiChan/gocq-leveldb-migrate/releases). + +从源码安装: +```bash +$ go install github.com/RomiChan/gocq-leveldb-migrate@latest +``` + +## 使用方法 + +```bash +./gocq-leveldb-migrate -from xxx -to yyy +``` +默认值: + * from: `data/leveldb-v2` + * to: `data/leveldb-v3` \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..ddbb374 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,99 @@ +package cmd + +import ( + "bytes" + "encoding/gob" + "flag" + "fmt" + "os" + + "github.com/Mrs4s/go-cqhttp/db" + "github.com/Mrs4s/go-cqhttp/global" + "github.com/pkg/errors" + "github.com/syndtr/goleveldb/leveldb" +) + +func handle(err error) { + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func Main() { + v2 := flag.String("from", "data/leveldb-v2", "leveldb v2 path") + v3 := flag.String("to", "data/leveldb-v3", "leveldb v3 path") + flag.Parse() + if flag.Arg(0) == "help" { + flag.Usage() + return + } + gob.Register(db.StoredMessageAttribute{}) // register gob types + gob.Register(db.StoredGuildMessageAttribute{}) + gob.Register(db.QuotedInfo{}) + gob.Register(global.MSG{}) + gob.Register(db.StoredGroupMessage{}) + gob.Register(db.StoredPrivateMessage{}) + gob.Register(db.StoredGuildChannelMessage{}) + + v2db, err := leveldb.OpenFile(*v2, nil) + handle(err) + defer v2db.Close() + { + v3db, err := leveldb.OpenFile(*v3, nil) + handle(err) + _ = v3db.Close() + } + v3db, err := leveldb.OpenFile(*v3, nil) + handle(err) + + iter := v2db.NewIterator(nil, nil) + var errs []error + for iter.Next() { + key := iter.Key() + value := iter.Value() + if len(value) == 0 { + continue + } + writer := newWriter() + data := value[1:] + writer.uvarint(uint64(value[0])) + switch value[0] { + case group: + x := &db.StoredGroupMessage{} + if err = gob.NewDecoder(bytes.NewReader(data)).Decode(x); err != nil { + errs = append(errs, errors.Wrap(err, "decode group message failed")) + continue + } + writer.writeStoredGroupMessage(x) + case private: + x := &db.StoredPrivateMessage{} + if err = gob.NewDecoder(bytes.NewReader(data)).Decode(x); err != nil { + errs = append(errs, errors.Wrap(err, "decode private message failed")) + continue + } + writer.writeStoredPrivateMessage(x) + case guildChannel: + x := &db.StoredGuildChannelMessage{} + if err = gob.NewDecoder(bytes.NewReader(data)).Decode(x); err != nil { + errs = append(errs, errors.Wrap(err, "decode guild channel message failed")) + continue + } + writer.writeStoredGuildChannelMessage(x) + } + err := v3db.Put(key, writer.bytes(), nil) + if err != nil { + errs = append(errs, errors.Wrap(err, "put to v3 failed")) + } + } + iter.Release() + if len(errs) > 0 { + for _, err := range errs { + fmt.Printf("%v\n", err) + } + } + tr, err := v3db.OpenTransaction() + handle(err) + handle(tr.Commit()) + handle(v3db.Close()) +} diff --git a/cmd/writer.go b/cmd/writer.go new file mode 100644 index 0000000..4c6535d --- /dev/null +++ b/cmd/writer.go @@ -0,0 +1,257 @@ +package cmd + +import ( + "bytes" + "io" + + "github.com/Mrs4s/go-cqhttp/db" + "github.com/Mrs4s/go-cqhttp/global" +) + +const dataVersion = 1 + +const ( + group = 0x0 + private = 0x1 + guildChannel = 0x2 +) + +type coder byte + +const ( + coderNil coder = iota + coderInt + coderUint + coderInt32 + coderUint32 + coderInt64 + coderUint64 + coderString + coderMSG // global.MSG + coderArrayMSG // []global.MSG + coderStruct // struct{} +) + +type intWriter struct { + bytes.Buffer +} + +func (w *intWriter) varint(x int64) { + w.uvarint(uint64(x)<<1 ^ uint64(x>>63)) +} + +func (w *intWriter) uvarint(x uint64) { + for x >= 0x80 { + w.WriteByte(byte(x) | 0x80) + x >>= 7 + } + w.WriteByte(byte(x)) +} + +// writer implements the index write. +// data format(use uvarint to encode integers): +// | version | string data length | index data length | string data | index data | +// for string data part, each string is encoded as: +// | string length | string | +// for index data part, each value is encoded as: +// | coder | value | +// * coder is the identifier of value's type. +// * specially for string, it's value is the offset in string data part. +type writer struct { + data intWriter + strings intWriter + stringIndex map[string]uint64 +} + +func newWriter() *writer { + return &writer{ + stringIndex: make(map[string]uint64), + } +} + +func (w *writer) coder(o coder) { w.data.WriteByte(byte(o)) } +func (w *writer) varint(x int64) { w.data.varint(x) } +func (w *writer) uvarint(x uint64) { w.data.uvarint(x) } +func (w *writer) nil() { w.coder(coderNil) } + +func (w *writer) int(i int) { + w.varint(int64(i)) +} + +func (w *writer) uint(i uint) { + w.uvarint(uint64(i)) +} + +func (w *writer) int32(i int32) { + w.varint(int64(i)) +} + +func (w *writer) uint32(i uint32) { + w.uvarint(uint64(i)) +} + +func (w *writer) int64(i int64) { + w.varint(i) +} + +func (w *writer) uint64(i uint64) { + w.uvarint(i) +} + +func (w *writer) string(s string) { + off, ok := w.stringIndex[s] + if !ok { + // not found write to string data part + // | string length | string | + off = uint64(w.strings.Len()) + w.strings.uvarint(uint64(len(s))) + _, _ = w.strings.WriteString(s) + w.stringIndex[s] = off + } + // write offset to index data part + w.uvarint(off) +} + +func (w *writer) msg(m global.MSG) { + w.uvarint(uint64(len(m))) + for s, obj := range m { + w.string(s) + w.obj(obj) + } +} + +func (w *writer) arrayMsg(a []global.MSG) { + w.uvarint(uint64(len(a))) + for _, v := range a { + w.msg(v) + } +} + +func (w *writer) obj(o interface{}) { + switch x := o.(type) { + case nil: + w.nil() + case int: + w.coder(coderInt) + w.int(x) + case int32: + w.coder(coderInt32) + w.int32(x) + case int64: + w.coder(coderInt64) + w.int64(x) + case uint: + w.coder(coderUint) + w.uint(x) + case uint32: + w.coder(coderUint32) + w.uint32(x) + case uint64: + w.coder(coderUint64) + w.uint64(x) + case string: + w.coder(coderString) + w.string(x) + case global.MSG: + w.coder(coderMSG) + w.msg(x) + case []global.MSG: + w.coder(coderArrayMSG) + w.arrayMsg(x) + default: + panic("unsupported type") + } +} + +func (w *writer) bytes() []byte { + var out intWriter + out.uvarint(dataVersion) + out.uvarint(uint64(w.strings.Len())) + out.uvarint(uint64(w.data.Len())) + _, _ = io.Copy(&out, &w.strings) + _, _ = io.Copy(&out, &w.data) + return out.Bytes() +} + +func (w *writer) writeStoredGroupMessage(x *db.StoredGroupMessage) { + if x == nil { + w.nil() + return + } + w.coder(coderStruct) + w.string(x.ID) + w.int32(x.GlobalID) + w.writeStoredMessageAttribute(x.Attribute) + w.string(x.SubType) + w.writeQuotedInfo(x.QuotedInfo) + w.int64(x.GroupCode) + w.string(x.AnonymousID) + w.arrayMsg(x.Content) +} + +func (w *writer) writeStoredPrivateMessage(x *db.StoredPrivateMessage) { + if x == nil { + w.nil() + return + } + w.coder(coderStruct) + w.string(x.ID) + w.int32(x.GlobalID) + w.writeStoredMessageAttribute(x.Attribute) + w.string(x.SubType) + w.writeQuotedInfo(x.QuotedInfo) + w.int64(x.SessionUin) + w.int64(x.TargetUin) + w.arrayMsg(x.Content) +} + +func (w *writer) writeStoredGuildChannelMessage(x *db.StoredGuildChannelMessage) { + if x == nil { + w.nil() + return + } + w.coder(coderStruct) + w.string(x.ID) + w.writeStoredGuildMessageAttribute(x.Attribute) + w.uint64(x.GuildID) + w.uint64(x.ChannelID) + w.writeQuotedInfo(x.QuotedInfo) + w.arrayMsg(x.Content) +} + +func (w *writer) writeStoredMessageAttribute(x *db.StoredMessageAttribute) { + if x == nil { + w.nil() + return + } + w.coder(coderStruct) + w.int32(x.MessageSeq) + w.int32(x.InternalID) + w.int64(x.SenderUin) + w.string(x.SenderName) + w.int64(x.Timestamp) +} + +func (w *writer) writeStoredGuildMessageAttribute(x *db.StoredGuildMessageAttribute) { + if x == nil { + w.nil() + return + } + w.coder(coderStruct) + w.uint64(x.MessageSeq) + w.uint64(x.InternalID) + w.uint64(x.SenderTinyID) + w.string(x.SenderName) + w.int64(x.Timestamp) +} + +func (w *writer) writeQuotedInfo(x *db.QuotedInfo) { + if x == nil { + w.nil() + return + } + w.coder(coderStruct) + w.string(x.PrevID) + w.int32(x.PrevGlobalID) + w.arrayMsg(x.QuotedContent) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2db9797 --- /dev/null +++ b/go.mod @@ -0,0 +1,24 @@ +module github.com/RomiChan/gocq-leveldb-migrate + +go 1.17 + +require ( + github.com/Mrs4s/go-cqhttp v1.0.0-rc1 + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 +) + +require ( + github.com/Microsoft/go-winio v0.5.1 // indirect + github.com/Mrs4s/MiraiGo v0.0.0-20220209092529-5d071b034c17 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/segmentio/asm v1.1.3 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/tidwall/gjson v1.12.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect +) diff --git a/main.go b/main.go new file mode 100644 index 0000000..52c1870 --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/RomiChan/gocq-leveldb-migrate/cmd" + +func main() { + cmd.Main() +}