From 8bad8a9df2ee1ae44a11039e1271b4f39cf2375b Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Mon, 16 Oct 2023 16:24:49 +0530 Subject: [PATCH 1/8] scaffold(backend): add initial files for new command --- backend/cmd/user/export.go | 32 ++++++++++++++++++++++++++++++++ backend/cmd/user/import.go | 13 +++++++------ backend/cmd/user/root.go | 1 + 3 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 backend/cmd/user/export.go diff --git a/backend/cmd/user/export.go b/backend/cmd/user/export.go new file mode 100644 index 000000000..46a5b9305 --- /dev/null +++ b/backend/cmd/user/export.go @@ -0,0 +1,32 @@ +package user + +import ( + "log" + + "github.com/spf13/cobra" + "github.com/teamhanko/hanko/backend/config" +) + +func NewExportCommand() *cobra.Command { + var ( + configFile string + outputFile string + ) + + cmd := &cobra.Command{ + Use: "export", + Short: "Export users from database into a Json file", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + log.Println("Exporting users...") + }, + } + + cmd.Flags().StringVar(&configFile, "config", config.DefaultConfigFilePath, "config file") + cmd.Flags().StringVarP(&outputFile, "outputFile", "o", "", "The path of the output file.") + err := cmd.MarkFlagRequired("outputFile") + if err != nil { + log.Println(err) + } + return cmd +} diff --git a/backend/cmd/user/import.go b/backend/cmd/user/import.go index a9ca51c97..260de53ee 100644 --- a/backend/cmd/user/import.go +++ b/backend/cmd/user/import.go @@ -4,18 +4,19 @@ import ( "encoding/json" "errors" "fmt" - "github.com/gobuffalo/pop/v6" - "github.com/gofrs/uuid" - "github.com/spf13/cobra" - "github.com/teamhanko/hanko/backend/config" - "github.com/teamhanko/hanko/backend/persistence" - "github.com/teamhanko/hanko/backend/persistence/models" "io" "log" "net/http" "os" "strings" "time" + + "github.com/gobuffalo/pop/v6" + "github.com/gofrs/uuid" + "github.com/spf13/cobra" + "github.com/teamhanko/hanko/backend/config" + "github.com/teamhanko/hanko/backend/persistence" + "github.com/teamhanko/hanko/backend/persistence/models" ) func NewImportCommand() *cobra.Command { diff --git a/backend/cmd/user/root.go b/backend/cmd/user/root.go index 3065bc108..66742f09c 100644 --- a/backend/cmd/user/root.go +++ b/backend/cmd/user/root.go @@ -17,4 +17,5 @@ func RegisterCommands(parent *cobra.Command) { parent.AddCommand(command) command.AddCommand(NewImportCommand()) command.AddCommand(NewGenerateCommand()) + command.AddCommand(NewExportCommand()) } From e4b79c37e5b1d5e196648491322c4ee3df6cbc36 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Mon, 16 Oct 2023 17:05:32 +0530 Subject: [PATCH 2/8] refactor(backend): change name of struct --- backend/cmd/user/format.go | 19 +++++++------ backend/cmd/user/format_test.go | 17 ++++++------ backend/cmd/user/generate.go | 13 +++++---- backend/cmd/user/import.go | 8 +++--- backend/cmd/user/import_test.go | 37 +++++++++++++------------ backend/json_schema/schema_generator.go | 5 ++-- backend/persistence/user_persister.go | 15 ++++++++++ 7 files changed, 67 insertions(+), 47 deletions(-) diff --git a/backend/cmd/user/format.go b/backend/cmd/user/format.go index a47222125..07e788146 100644 --- a/backend/cmd/user/format.go +++ b/backend/cmd/user/format.go @@ -3,12 +3,13 @@ package user import ( "errors" "fmt" - "github.com/gofrs/uuid" "time" + + "github.com/gofrs/uuid" ) -// ImportEmail The import format for a user's email -type ImportEmail struct { +// ImportOrExportEmail The import/export format for a user's email +type ImportOrExportEmail struct { // Address Valid email address Address string `json:"address" yaml:"address"` // IsPrimary indicates if this is the primary email of the users. In the Emails array there has to be exactly one primary email. @@ -18,10 +19,10 @@ type ImportEmail struct { } // Emails Array of email addresses -type Emails []ImportEmail +type Emails []ImportOrExportEmail -// ImportEntry represents a user to be imported to the Hanko database -type ImportEntry struct { +// ImportOrExportEntry represents a user to be imported/export to the Hanko database +type ImportOrExportEntry struct { // UserID optional uuid.v4. If not provided a new one will be generated for the user UserID string `json:"user_id" yaml:"user_id"` // Emails List of emails @@ -32,10 +33,10 @@ type ImportEntry struct { UpdatedAt *time.Time `json:"updated_at" yaml:"updated_at"` } -// ImportList a list of ImportEntries -type ImportList []ImportEntry +// ImportOrExportList a list of ImportEntries +type ImportOrExportList []ImportOrExportEntry -func (entry *ImportEntry) validate() error { +func (entry *ImportOrExportEntry) validate() error { if len(entry.Emails) == 0 { return errors.New(fmt.Sprintf("Entry with id: %v has got no Emails.", entry.UserID)) } diff --git a/backend/cmd/user/format_test.go b/backend/cmd/user/format_test.go index cecdf4902..4d0a4f5ee 100644 --- a/backend/cmd/user/format_test.go +++ b/backend/cmd/user/format_test.go @@ -2,9 +2,10 @@ package user import ( "fmt" - "github.com/stretchr/testify/assert" "testing" "time" + + "github.com/stretchr/testify/assert" ) const validUUID = "62418053-a2cd-47a8-9b61-4426380d263a" @@ -27,7 +28,7 @@ func TestImportEntry_validate(t *testing.T) { fields: fields{ UserID: "", Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary@hanko.io", IsPrimary: true, IsVerified: false, @@ -43,7 +44,7 @@ func TestImportEntry_validate(t *testing.T) { fields: fields{ UserID: validUUID, Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary@hanko.io", IsPrimary: true, IsVerified: false, @@ -59,7 +60,7 @@ func TestImportEntry_validate(t *testing.T) { fields: fields{ UserID: invalidUUID, Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary@hanko.io", IsPrimary: true, IsVerified: false, @@ -85,7 +86,7 @@ func TestImportEntry_validate(t *testing.T) { fields: fields{ UserID: "", Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary@hanko.io", IsPrimary: false, IsVerified: false, @@ -101,12 +102,12 @@ func TestImportEntry_validate(t *testing.T) { fields: fields{ UserID: "", Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary@hanko.io", IsPrimary: true, IsVerified: false, }, - ImportEmail{ + ImportOrExportEmail{ Address: "primary2@hanko.io", IsPrimary: true, IsVerified: false, @@ -120,7 +121,7 @@ func TestImportEntry_validate(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - entry := &ImportEntry{ + entry := &ImportOrExportEntry{ UserID: tt.fields.UserID, Emails: tt.fields.Emails, CreatedAt: tt.fields.CreatedAt, diff --git a/backend/cmd/user/generate.go b/backend/cmd/user/generate.go index cd64050fd..3f991d516 100644 --- a/backend/cmd/user/generate.go +++ b/backend/cmd/user/generate.go @@ -2,12 +2,13 @@ package user import ( "encoding/json" - "github.com/brianvoe/gofakeit/v6" - "github.com/gofrs/uuid" - "github.com/spf13/cobra" "log" "os" "time" + + "github.com/brianvoe/gofakeit/v6" + "github.com/gofrs/uuid" + "github.com/spf13/cobra" ) var outputFile string @@ -36,18 +37,18 @@ func NewGenerateCommand() *cobra.Command { } func generate() error { - var entries []ImportEntry + var entries []ImportOrExportEntry for i := 0; i < count; i++ { now := time.Now().UTC() id, _ := uuid.NewV4() - emails := []ImportEmail{ + emails := []ImportOrExportEmail{ { Address: gofakeit.Email(), IsPrimary: true, IsVerified: true, }, } - entry := ImportEntry{ + entry := ImportOrExportEntry{ UserID: id.String(), Emails: emails, CreatedAt: &now, diff --git a/backend/cmd/user/import.go b/backend/cmd/user/import.go index 260de53ee..b403e2dd9 100644 --- a/backend/cmd/user/import.go +++ b/backend/cmd/user/import.go @@ -104,7 +104,7 @@ func NewImportCommand() *cobra.Command { // loadAndValidate reads json from an io.Reader so we read every entry separate and validate it. We go through the whole // array to print out every validation error in the input data. -func loadAndValidate(input io.Reader) ([]ImportEntry, error) { +func loadAndValidate(input io.Reader) ([]ImportOrExportEntry, error) { dec := json.NewDecoder(input) // read the open bracket @@ -113,14 +113,14 @@ func loadAndValidate(input io.Reader) ([]ImportEntry, error) { return nil, err } - users := []ImportEntry{} + users := []ImportOrExportEntry{} numErrors := 0 index := 0 // while the array contains values for dec.More() { index = index + 1 - var userEntry ImportEntry + var userEntry ImportOrExportEntry // decode one ImportEntry err := dec.Decode(&userEntry) if err != nil { @@ -153,7 +153,7 @@ func loadAndValidate(input io.Reader) ([]ImportEntry, error) { } // commits the list of ImportEntries to the database. Wrapped in a transaction so if something fails no new users are added. -func addToDatabase(entries []ImportEntry, persister persistence.Persister) error { +func addToDatabase(entries []ImportOrExportEntry, persister persistence.Persister) error { tx := persister.GetConnection() err := tx.Transaction(func(tx *pop.Connection) error { for i, v := range entries { diff --git a/backend/cmd/user/import_test.go b/backend/cmd/user/import_test.go index b577e17ed..cf30d4f83 100644 --- a/backend/cmd/user/import_test.go +++ b/backend/cmd/user/import_test.go @@ -2,16 +2,17 @@ package user import ( "fmt" - "github.com/gofrs/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - "github.com/teamhanko/hanko/backend/persistence" - "github.com/teamhanko/hanko/backend/test" "io" "log" "strings" "testing" "time" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/teamhanko/hanko/backend/persistence" + "github.com/teamhanko/hanko/backend/test" ) const validUUID2 = "799e95f0-4cc7-4bd7-9f01-5fdc4fa26ea3" @@ -33,7 +34,7 @@ func (s *importSuite) Test_loadAndValidate() { tests := []struct { name string args args - want []ImportEntry + want []ImportOrExportEntry wantErr assert.ErrorAssertionFunc }{ { @@ -42,7 +43,7 @@ func (s *importSuite) Test_loadAndValidate() { input: strings.NewReader("[]"), }, wantErr: assert.NoError, - want: []ImportEntry{}, + want: []ImportOrExportEntry{}, }, { name: "empty file -> nil result", @@ -58,11 +59,11 @@ func (s *importSuite) Test_loadAndValidate() { input: strings.NewReader("[{\"user_id\":\"799e95f0-4cc7-4bd7-9f01-5fdc4fa26ea3\",\"emails\":[{\"address\":\"koreyrath@wolff.name\",\"is_primary\":true,\"is_verified\":true}],\"created_at\":\"2023-06-07T13:42:49.369489Z\",\"updated_at\":\"2023-06-07T13:42:49.369489Z\"}]\n"), }, wantErr: assert.NoError, - want: []ImportEntry{ + want: []ImportOrExportEntry{ { UserID: validUUID2, Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "koreyrath@wolff.name", IsPrimary: true, IsVerified: true, @@ -107,7 +108,7 @@ func (s *importSuite) Test_addToDatabase() { } type args struct { - entries []ImportEntry + entries []ImportOrExportEntry persister persistence.Persister } tests := []struct { @@ -119,11 +120,11 @@ func (s *importSuite) Test_addToDatabase() { { name: "Positive", args: args{ - entries: []ImportEntry{ + entries: []ImportOrExportEntry{ { UserID: "", Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary@hanko.io", IsPrimary: true, IsVerified: false, @@ -141,11 +142,11 @@ func (s *importSuite) Test_addToDatabase() { { name: "Double uuid", args: args{ - entries: []ImportEntry{ + entries: []ImportOrExportEntry{ { UserID: validUUID, Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary1@hanko.io", IsPrimary: true, IsVerified: false, @@ -157,7 +158,7 @@ func (s *importSuite) Test_addToDatabase() { { UserID: validUUID, Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary2@hanko.io", IsPrimary: true, IsVerified: false, @@ -175,11 +176,11 @@ func (s *importSuite) Test_addToDatabase() { { name: "Double primary email", args: args{ - entries: []ImportEntry{ + entries: []ImportOrExportEntry{ { UserID: validUUID, Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary@hanko.io", IsPrimary: true, IsVerified: false, @@ -191,7 +192,7 @@ func (s *importSuite) Test_addToDatabase() { { UserID: validUUID, Emails: Emails{ - ImportEmail{ + ImportOrExportEmail{ Address: "primary@hanko.io", IsPrimary: true, IsVerified: false, diff --git a/backend/json_schema/schema_generator.go b/backend/json_schema/schema_generator.go index 61a162cbd..ffd55c191 100644 --- a/backend/json_schema/schema_generator.go +++ b/backend/json_schema/schema_generator.go @@ -3,17 +3,18 @@ package main import ( "encoding/json" "fmt" + "os" + "github.com/invopop/jsonschema" "github.com/teamhanko/hanko/backend/cmd/user" "github.com/teamhanko/hanko/backend/config" - "os" ) func main() { if err := generateSchema("./config", "./json_schema/hanko.config.json", &config.Config{}); err != nil { panic(err) } - if err := generateSchema("./cmd/user", "./json_schema/hanko.user_import.json", &user.ImportList{}); err != nil { + if err := generateSchema("./cmd/user", "./json_schema/hanko.user_import.json", &user.ImportOrExportList{}); err != nil { panic(err) } diff --git a/backend/persistence/user_persister.go b/backend/persistence/user_persister.go index e73338ce6..74d57ff7f 100644 --- a/backend/persistence/user_persister.go +++ b/backend/persistence/user_persister.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/gobuffalo/pop/v6" "github.com/gofrs/uuid" "github.com/teamhanko/hanko/backend/persistence/models" @@ -15,6 +16,7 @@ type UserPersister interface { Update(models.User) error Delete(models.User) error List(page int, perPage int, userId uuid.UUID, email string, sortDirection string) ([]models.User, error) + All() ([]models.User, error) Count(userId uuid.UUID, email string) (int, error) } @@ -26,6 +28,19 @@ func NewUserPersister(db *pop.Connection) UserPersister { return &userPersister{db: db} } +func (p *userPersister) All() ([]models.User, error) { + users := []models.User{} + err := p.db.EagerPreload("Emails", "Emails.PrimaryEmail", "Emails.Identity", "WebauthnCredentials").All(&users) + if err != nil && errors.Is(err, sql.ErrNoRows) { + return users, nil + } + if err != nil { + return nil, fmt.Errorf("failed to fetch users: %w", err) + } + + return users, nil +} + func (p *userPersister) Get(id uuid.UUID) (*models.User, error) { user := models.User{} err := p.db.EagerPreload("Emails", "Emails.PrimaryEmail", "Emails.Identity", "WebauthnCredentials").Find(&user, id) From e87616ed5e2a3db6f4368809d872382e0a7fa7ae Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Mon, 16 Oct 2023 17:08:11 +0530 Subject: [PATCH 3/8] feat(backend): add export command --- backend/cmd/user/export.go | 52 +++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/backend/cmd/user/export.go b/backend/cmd/user/export.go index 46a5b9305..09f69336e 100644 --- a/backend/cmd/user/export.go +++ b/backend/cmd/user/export.go @@ -1,10 +1,14 @@ package user import ( + "encoding/json" + "fmt" "log" + "os" "github.com/spf13/cobra" "github.com/teamhanko/hanko/backend/config" + "github.com/teamhanko/hanko/backend/persistence" ) func NewExportCommand() *cobra.Command { @@ -18,7 +22,19 @@ func NewExportCommand() *cobra.Command { Short: "Export users from database into a Json file", Long: ``, Run: func(cmd *cobra.Command, args []string) { - log.Println("Exporting users...") + cfg, err := config.Load(&configFile) + if err != nil { + log.Fatal(err) + } + persister, err := persistence.New(cfg.Database) + if err != nil { + log.Fatal(err) + } + err = export(persister, outputFile) + if err != nil { + log.Fatal(err) + } + log.Println(fmt.Sprintf("Successfully exported users to %s", outputFile)) }, } @@ -30,3 +46,37 @@ func NewExportCommand() *cobra.Command { } return cmd } + +func export(persister persistence.Persister, outFile string) error { + var entries []ImportOrExportEntry + users, err := persister.GetUserPersister().All() + if err != nil { + return fmt.Errorf("failed to get list of users: %w", err) + } + for _, user := range users { + var emails []ImportOrExportEmail + for _, email := range user.Emails { + emails = append(emails, ImportOrExportEmail{ + Address: email.Address, + IsPrimary: email.IsPrimary(), + IsVerified: email.Verified, + }) + } + entry := ImportOrExportEntry{ + UserID: user.ID.String(), + Emails: emails, + CreatedAt: &user.CreatedAt, + UpdatedAt: &user.UpdatedAt, + } + entries = append(entries, entry) + } + bytes, err := json.Marshal(entries) + if err != nil { + return err + } + err = os.WriteFile(outputFile, bytes, 0600) + if err != nil { + return err + } + return nil +} From 5b06caca9276c56ec3665ace75ce7f2fcb280444 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Mon, 16 Oct 2023 17:11:45 +0530 Subject: [PATCH 4/8] fix(backend): remove todo string --- backend/cmd/user/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cmd/user/root.go b/backend/cmd/user/root.go index 66742f09c..d1305e8ba 100644 --- a/backend/cmd/user/root.go +++ b/backend/cmd/user/root.go @@ -7,7 +7,7 @@ import ( func NewUserCommand() *cobra.Command { return &cobra.Command{ Use: "user", - Short: "User import/export(TODO) tools", + Short: "User import/export tools", Long: `Add the ability to import users into the hanko database.`, } } From 364ff7c493498275019508fbbeccc2859ded541a Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Mon, 16 Oct 2023 17:15:00 +0530 Subject: [PATCH 5/8] fix(backend): write file to correct location --- backend/cmd/user/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cmd/user/export.go b/backend/cmd/user/export.go index 09f69336e..608b7ba99 100644 --- a/backend/cmd/user/export.go +++ b/backend/cmd/user/export.go @@ -74,7 +74,7 @@ func export(persister persistence.Persister, outFile string) error { if err != nil { return err } - err = os.WriteFile(outputFile, bytes, 0600) + err = os.WriteFile(outFile, bytes, 0600) if err != nil { return err } From cd2015f85e9bae12f49d8d0fa99fb2c91b61b897 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Mon, 16 Oct 2023 18:10:34 +0530 Subject: [PATCH 6/8] tests(backend): add missing impl --- backend/test/user_persister.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/test/user_persister.go b/backend/test/user_persister.go index e40be6cb2..6f1045580 100644 --- a/backend/test/user_persister.go +++ b/backend/test/user_persister.go @@ -81,6 +81,10 @@ func (p *userPersister) List(page int, perPage int, userId uuid.UUID, email stri return result[page-1], nil } +func (p *userPersister) All() ([]models.User, error) { + return p.users, nil +} + func (p *userPersister) Count(userId uuid.UUID, email string) (int, error) { return len(p.users), nil } From 611556f99ced0ac53e5910422a73ae87a8588d15 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Mon, 16 Oct 2023 18:12:21 +0530 Subject: [PATCH 7/8] refactor(backend): change order of funcs --- backend/persistence/user_persister.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/persistence/user_persister.go b/backend/persistence/user_persister.go index 74d57ff7f..07b3def40 100644 --- a/backend/persistence/user_persister.go +++ b/backend/persistence/user_persister.go @@ -28,19 +28,6 @@ func NewUserPersister(db *pop.Connection) UserPersister { return &userPersister{db: db} } -func (p *userPersister) All() ([]models.User, error) { - users := []models.User{} - err := p.db.EagerPreload("Emails", "Emails.PrimaryEmail", "Emails.Identity", "WebauthnCredentials").All(&users) - if err != nil && errors.Is(err, sql.ErrNoRows) { - return users, nil - } - if err != nil { - return nil, fmt.Errorf("failed to fetch users: %w", err) - } - - return users, nil -} - func (p *userPersister) Get(id uuid.UUID) (*models.User, error) { user := models.User{} err := p.db.EagerPreload("Emails", "Emails.PrimaryEmail", "Emails.Identity", "WebauthnCredentials").Find(&user, id) @@ -126,6 +113,19 @@ func (p *userPersister) List(page int, perPage int, userId uuid.UUID, email stri return users, nil } +func (p *userPersister) All() ([]models.User, error) { + users := []models.User{} + err := p.db.EagerPreload("Emails", "Emails.PrimaryEmail", "Emails.Identity", "WebauthnCredentials").All(&users) + if err != nil && errors.Is(err, sql.ErrNoRows) { + return users, nil + } + if err != nil { + return nil, fmt.Errorf("failed to fetch users: %w", err) + } + + return users, nil +} + func (p *userPersister) Count(userId uuid.UUID, email string) (int, error) { query := p.db. Q(). From 10ba93619150d103e2a93fcdce08b1e56d797b04 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Tue, 17 Oct 2023 12:58:56 +0530 Subject: [PATCH 8/8] fix(backend): add long description for user export --- backend/cmd/user/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cmd/user/root.go b/backend/cmd/user/root.go index d1305e8ba..c019cbea4 100644 --- a/backend/cmd/user/root.go +++ b/backend/cmd/user/root.go @@ -8,7 +8,7 @@ func NewUserCommand() *cobra.Command { return &cobra.Command{ Use: "user", Short: "User import/export tools", - Long: `Add the ability to import users into the hanko database.`, + Long: `Add the ability to import/export users into/from the hanko database.`, } }