diff --git a/.gitignore b/.gitignore index ffaee62..5ddb6e7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,9 +12,11 @@ *.out # Intermediate build artifacts +/*.zip /assets /bindata.go /go.sum +/vendor # RWTxt databases *db diff --git a/cmd/rwtxt/main.go b/cmd/rwtxt/main.go index 10031ef..bbed947 100644 --- a/cmd/rwtxt/main.go +++ b/cmd/rwtxt/main.go @@ -21,6 +21,7 @@ var ( func main() { var ( err error + export = flag.Bool("export", false, "export uploads to {{TIMESTAMP}}-uploads.zip and posts to {{TIMESTAMP}}-posts.zip") resizeWidth = flag.Int("resizewidth", -1, "image width to resize on the fly") resizeOnUpload = flag.Bool("resizeonupload", false, "resize on upload") resizeOnRequest = flag.Bool("resizeonrequest", false, "resize on request") @@ -70,6 +71,18 @@ func main() { panic(err) } + if *export { + err = fs.ExportPosts() + if err != nil { + panic(err) + } + err = fs.ExportUploads() + if err != nil { + panic(err) + } + return + } + config := rwtxt.Config{Private: *private, ResizeWidth: *resizeWidth, ResizeOnRequest: *resizeOnRequest, diff --git a/pkg/db/db.go b/pkg/db/db.go index 407854c..b575ad9 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -2,11 +2,16 @@ package db import ( "bufio" + "bytes" "compress/gzip" "database/sql" "encoding/json" + "fmt" "html/template" + "io/ioutil" "os" + "path/filepath" + "strconv" "strings" "sync" "time" @@ -263,6 +268,150 @@ func (fs *FileSystem) SaveBlob(id string, name string, blob []byte) (err error) return } +// ExportPosts will save posts to {{TIMESTAMP}}-posts.gz +func (fs *FileSystem) ExportPosts() error { + domains, err := fs.GetDomains() + if err != nil { + return err + } + + dir := os.TempDir() + postPaths := []string{} + for _, domain := range domains { + files, err := fs.GetAll(domain) + if err != nil { + return err + } + for _, file := range files { + fname := (fmt.Sprintf("%s-%s.md", file.Slug, file.ID)) + r := strings.NewReader(file.Data) + if err != nil { + return err + } + var buf bytes.Buffer + _, err = buf.ReadFrom(r) + if err != nil { + return err + } + err = os.MkdirAll(filepath.Join(dir, domain), os.ModePerm) + if err != nil { + return err + } + fpath := filepath.Join(dir, domain, fname) + err = ioutil.WriteFile(fpath, buf.Bytes(), os.ModePerm) + if err != nil { + return err + } + + postPaths = append(postPaths, fpath) + } + } + timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) + for _, f := range postPaths { + log.Debug(f) + } + utils.ZipFiles(fmt.Sprintf("%s-posts.zip", timestamp), postPaths) + return nil + +} + +// ExportUploads will save uploads to {{TIMESTAMP}}-uploads.gz +func (fs *FileSystem) ExportUploads() error { + dir := os.TempDir() + files := []string{} + + ids, err := fs.GetBlobIDs() + if err != nil { + return err + } + + for _, id := range ids { + name, data, _, err := fs.GetBlob(id) + if err != nil { + return err + } + fname := fmt.Sprintf("%s-%s", id, name) + + r, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return err + } + var buf bytes.Buffer + _, err = buf.ReadFrom(r) + if err != nil { + return err + } + fpath := filepath.Join(dir, fname) + err = ioutil.WriteFile(fpath, buf.Bytes(), os.ModePerm) + if err != nil { + return err + } + + files = append(files, fpath) + } + + timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) + for _, f := range files { + log.Debug(f) + } + utils.ZipFiles(fmt.Sprintf("%s-uploads.zip", timestamp), files) + return nil +} + +// GetBlobIDs will return a list of blob ids +func (fs *FileSystem) GetBlobIDs() ([]string, error) { + fs.Lock() + defer fs.Unlock() + stmt, err := fs.DB.Prepare(`SELECT id FROM blobs`) + if err != nil { + return nil, err + } + defer stmt.Close() + + result := []string{} + rows, err := stmt.Query() + if err != nil { + return nil, err + } + for rows.Next() { + var id string + err = rows.Scan(&id) + if err != nil { + return nil, err + } + result = append(result, id) + } + + return result, nil +} + +// GetDomains will return a list of domains +func (fs *FileSystem) GetDomains() ([]string, error) { + fs.Lock() + defer fs.Unlock() + stmt, err := fs.DB.Prepare(`SELECT name FROM domains`) + if err != nil { + return nil, err + } + defer stmt.Close() + + result := []string{} + rows, err := stmt.Query() + if err != nil { + return nil, err + } + for rows.Next() { + var domain string + err = rows.Scan(&domain) + if err != nil { + return nil, err + } + result = append(result, domain) + } + + return result, nil +} + // SaveResizedImage will save a resized image func (fs *FileSystem) SaveResizedImage(id string, name string, blob []byte) (err error) { fs.Lock() diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index d2b8054..26a648a 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -1,11 +1,14 @@ package utils import ( + "archive/zip" "crypto/hmac" "crypto/sha512" "encoding/hex" "html/template" + "io" "math/rand" + "os" "strings" "time" @@ -14,6 +17,56 @@ import ( blackfriday "gopkg.in/russross/blackfriday.v2" ) +// ZipFiles will zip files to filename +func ZipFiles(filename string, files []string) error { + + newZipFile, err := os.Create(filename) + if err != nil { + return err + } + defer newZipFile.Close() + + zipWriter := zip.NewWriter(newZipFile) + defer zipWriter.Close() + + // Add files to zip + for _, file := range files { + + zipfile, err := os.Open(file) + if err != nil { + return err + } + defer zipfile.Close() + + // Get the file information + info, err := zipfile.Stat() + if err != nil { + return err + } + + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + + // Using FileInfoHeader() above only uses the basename of the file. If we want + // to preserve the folder structure we can overwrite this with the full path. + header.Name = file + + // Change to deflate to gain better compression + // see http://golang.org/pkg/archive/zip/#pkg-constants + header.Method = zip.Deflate + + writer, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + if _, err = io.Copy(writer, zipfile); err != nil { + return err + } + } + return nil +} func RenderMarkdownToHTML(markdown string) template.HTML { html := string(blackfriday.Run([]byte(markdown), blackfriday.WithExtensions(