diff --git a/api/publish.go b/api/publish.go index e3de9b4c7..4d9d39fc6 100644 --- a/api/publish.go +++ b/api/publish.go @@ -3,6 +3,7 @@ package api import ( "fmt" "net/http" + "path/filepath" "strings" "github.com/aptly-dev/aptly/aptly" @@ -394,3 +395,86 @@ func apiPublishDrop(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil }) } + +// POST /publish/:prefix/:distribution/remove +func apiPublishRemove(c *gin.Context) { + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := c.Params.ByName("distribution") + + var b struct { + ForceOverwrite bool + Signing SigningOptions + SkipCleanup *bool + Components []string `binding:"required"` + MultiDist bool + } + + if c.Bind(&b) != nil { + return + } + + signer, err := getSigner(&b.Signing) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to initialize GPG signer: %s", err)) + return + } + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err)) + return + } + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err)) + return + } + + components := b.Components + + for _, component := range components { + _, exists := published.Sources[component] + if !exists { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: '%s' is not a published component", component)) + return + } + published.DropComponent(component) + } + + resources := []string{string(published.Key())} + + taskName := fmt.Sprintf("Remove components '%s' from publish %s (%s)", strings.Join(components, ","), prefix, distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, b.MultiDist) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + if b.SkipCleanup == nil || !*b.SkipCleanup { + publishedStorage := context.GetPublishedStorage(storage) + + err = collection.CleanupPrefixComponentFiles(published.Prefix, components, publishedStorage, collectionFactory, out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + for _, component := range components { + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + } + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil + }) +} diff --git a/api/router.go b/api/router.go index 148b0f4af..cac1764fc 100644 --- a/api/router.go +++ b/api/router.go @@ -189,6 +189,7 @@ func Router(c *ctx.AptlyContext) http.Handler { api.POST("/publish/:prefix", apiPublishRepoOrSnapshot) api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch) api.DELETE("/publish/:prefix/:distribution", apiPublishDrop) + api.POST("/publish/:prefix/:distribution/remove", apiPublishRemove) } { diff --git a/cmd/publish.go b/cmd/publish.go index d74384e06..9ab944afb 100644 --- a/cmd/publish.go +++ b/cmd/publish.go @@ -38,6 +38,7 @@ func makeCmdPublish() *commander.Command { makeCmdPublishSwitch(), makeCmdPublishUpdate(), makeCmdPublishShow(), + makeCmdPublishRemove(), }, } } diff --git a/cmd/publish_remove.go b/cmd/publish_remove.go new file mode 100644 index 000000000..5a664107a --- /dev/null +++ b/cmd/publish_remove.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/aptly-dev/aptly/deb" + "github.com/smira/commander" + "github.com/smira/flag" +) + +func aptlyPublishRemove(cmd *commander.Command, args []string) error { + var err error + + if len(args) < 2 { + cmd.Usage() + return commander.ErrCommandError + } + + distribution := args[0] + components := args[1:] + + param := context.Flags().Lookup("prefix").Value.String() + if param == "" { + param = "." + } + storage, prefix := deb.ParsePrefix(param) + + collectionFactory := context.NewCollectionFactory() + published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + + err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + + multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool) + + for _, component := range components { + _, exists := published.Sources[component] + if !exists { + return fmt.Errorf("unable to update: '%s' is not a published component", component) + } + published.DropComponent(component) + } + + signer, err := getSigner(context.Flags()) + if err != nil { + return fmt.Errorf("unable to initialize GPG signer: %s", err) + } + + forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool) + if forceOverwrite { + context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " + + "the same package pool.\n") + } + + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist) + if err != nil { + return fmt.Errorf("unable to publish: %s", err) + } + + err = collectionFactory.PublishedRepoCollection().Update(published) + if err != nil { + return fmt.Errorf("unable to save to DB: %s", err) + } + + skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) + if !skipCleanup { + publishedStorage := context.GetPublishedStorage(storage) + + err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components, + publishedStorage, collectionFactory, context.Progress()) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + + for _, component := range components { + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", published.Distribution, component), context.Progress()) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + } + } + + return err +} + +func makeCmdPublishRemove() *commander.Command { + cmd := &commander.Command{ + Run: aptlyPublishRemove, + UsageLine: "remove ...", + Short: "remove component from published repository", + Long: ` +Command removes one or multiple components from a published repository. + +Example: + + $ aptly publish remove -prefix=filesystem:symlink:/debian wheezy contrib non-free +`, + Flag: *flag.NewFlagSet("aptly-publish-remove", flag.ExitOnError), + } + cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release") + cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") + cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") + cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") + cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)") + cmd.Flag.Bool("batch", false, "run GPG with detached tty") + cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG") + cmd.Flag.String("prefix", "", "publishing prefix in the form of [:]") + cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") + cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component") + cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") + + return cmd +} diff --git a/deb/publish.go b/deb/publish.go index 2bdd608de..78faaf2d4 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -439,6 +439,14 @@ func (p *PublishedRepo) SourceNames() []string { return sources } +// DropComponent removes component from published repo +func (p *PublishedRepo) DropComponent(component string) { + delete(p.Sources, component) + delete(p.sourceItems, component) + + p.rePublishing = true +} + // UpdateLocalRepo updates content from local repo in component func (p *PublishedRepo) UpdateLocalRepo(component string) { if p.SourceKind != SourceLocalRepo { diff --git a/system/t06_publish/PublishRemove1Test_gold b/system/t06_publish/PublishRemove1Test_gold new file mode 100644 index 000000000..301aceecb --- /dev/null +++ b/system/t06_publish/PublishRemove1Test_gold @@ -0,0 +1,7 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components c... +Removing ${HOME}/.aptly/public/dists/maverick/c... diff --git a/system/t06_publish/PublishRemove2Test_gold b/system/t06_publish/PublishRemove2Test_gold new file mode 100644 index 000000000..0207b726c --- /dev/null +++ b/system/t06_publish/PublishRemove2Test_gold @@ -0,0 +1,8 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components b, c... +Removing ${HOME}/.aptly/public/dists/maverick/b... +Removing ${HOME}/.aptly/public/dists/maverick/c... diff --git a/system/t06_publish/PublishRemove3Test_gold b/system/t06_publish/PublishRemove3Test_gold new file mode 100644 index 000000000..9cd8bc8bf --- /dev/null +++ b/system/t06_publish/PublishRemove3Test_gold @@ -0,0 +1 @@ +ERROR: unable to update: 'not-existent' is not a published component diff --git a/system/t06_publish/PublishRemove4Test_gold b/system/t06_publish/PublishRemove4Test_gold new file mode 100644 index 000000000..61f950296 --- /dev/null +++ b/system/t06_publish/PublishRemove4Test_gold @@ -0,0 +1,27 @@ +Usage: aptly publish remove ... + +aptly publish remove - remove component from published repository + + +Options: + -architectures="": list of architectures to consider during (comma-separated), default to all available + -batch: run GPG with detached tty + -config="": location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf) + -db-open-attempts=10: number of attempts to open DB if it's locked by other instance + -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b' + -dep-follow-recommends: when processing dependencies, follow Recommends + -dep-follow-source: when processing dependencies, follow from binary to Source packages + -dep-follow-suggests: when processing dependencies, follow Suggests + -dep-verbose-resolve: when processing dependencies, print detailed logs + -force-overwrite: overwrite files in package pool in case of mismatch + -gpg-key="": GPG key ID to use when signing the release + -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation) + -keyring=: GPG keyring to use (instead of default) + -multi-dist: enable multiple packages with the same filename in different distributions + -passphrase="": GPG passphrase for the key (warning: could be insecure) + -passphrase-file="": GPG passphrase-file for the key (warning: could be insecure) + -prefix="": publishing prefix in the form of [:] + -secret-keyring="": GPG secret keyring to use (instead of default) + -skip-cleanup: don't remove unreferenced files in prefix/component + -skip-signing: don't sign Release files with GPG +ERROR: unable to parse command diff --git a/system/t06_publish/remove.py b/system/t06_publish/remove.py new file mode 100644 index 000000000..88291ed22 --- /dev/null +++ b/system/t06_publish/remove.py @@ -0,0 +1,87 @@ +from lib import BaseTest + + +class PublishRemove1Test(BaseTest): + """ + publish remove: remove single component from published repository + """ + fixtureCmds = [ + "aptly snapshot create snap1 empty", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b,c snap1 snap2 snap3" + ] + runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick c" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishRemove1Test, self).check() + + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/a/binary-i386/Packages') + self.check_exists('public/dists/maverick/b/binary-i386/Packages') + self.check_not_exists('public/dists/maverick/c/binary-i386/Packages') + + release = self.read_file('public/dists/maverick/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + + if ['a', 'b'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'a b')) + + +class PublishRemove2Test(BaseTest): + """ + publish remove: remove multiple components from published repository + """ + fixtureCmds = [ + "aptly snapshot create snap1 empty", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b,c snap1 snap2 snap3" + ] + runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick b c" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishRemove2Test, self).check() + + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/a/binary-i386/Packages') + self.check_not_exists('public/dists/maverick/b/binary-i386/Packages') + self.check_not_exists('public/dists/maverick/c/binary-i386/Packages') + + release = self.read_file('public/dists/maverick/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + + if ['a'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'a')) + + +class PublishRemove3Test(BaseTest): + """ + publish remove: remove not existing component from published repository + """ + fixtureCmds = [ + "aptly snapshot create snap1 empty", + "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a snap1" + ] + runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick not-existent" + expectedCode = 1 + gold_processor = BaseTest.expand_environ + + +class PublishRemove4Test(BaseTest): + """ + publish remove: unspecified components + """ + fixtureCmds = [ + "aptly snapshot create snap1 empty", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b,c snap1 snap2 snap3" + ] + runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" + expectedCode = 2 + gold_processor = BaseTest.expand_environ diff --git a/system/t12_api/publish.py b/system/t12_api/publish.py index 4ecffbc1c..b06aa6c1b 100644 --- a/system/t12_api/publish.py +++ b/system/t12_api/publish.py @@ -873,3 +873,71 @@ def check(self): get = self.get("/repos/apiandserve/pool/main/b/boost-defaults/i-dont-exist") if get.status_code != 404: raise Exception(f"Expected status 404 != {get.status_code}") + + +class PublishRemoveAPITestRepo(APITest): + """ + POST /publish/:prefix/:distribution/remove + """ + fixtureGpg = True + + def check(self): + snapshot1_name = self.random_name() + snapshot2_name = self.random_name() + snapshot3_name = self.random_name() + + self.check_equal( + self.post("/api/snapshots", json={"Name": snapshot1_name}).status_code, 201) + self.check_equal( + self.post("/api/snapshots", json={"Name": snapshot2_name}).status_code, 201) + self.check_equal( + self.post("/api/snapshots", json={"Name": snapshot3_name}).status_code, 201) + + prefix = self.random_name() + distribution = "stable" + task = self.post_task( + "/api/publish/" + prefix, + json={ + "AcquireByHash": True, + "Architectures": ["i386"], + "SourceKind": "snapshot", + "Sources": [{"Component": "A", "Name": snapshot1_name}, {"Component": "B", "Name": snapshot2_name}, {"Component": "C", "Name": snapshot3_name}], + "Signing": DefaultSigningOptions, + "Distribution": distribution, + "NotAutomatic": "yes", + "ButAutomaticUpgrades": "yes", + "Origin": "earth", + "Label": "fun", + "SkipContents": True, + } + ) + + task = self.post_task("/api/publish/" + prefix + '/' + distribution + '/remove', json={"Components": ["B", "C"]}) + self.check_task(task) + + repo_expected = { + "AcquireByHash": True, + "Architectures": ["i386"], + "Codename": "", + "Distribution": distribution, + "Label": "fun", + "Origin": "earth", + "NotAutomatic": "yes", + "ButAutomaticUpgrades": "yes", + "Path": prefix + '/' + distribution, + "Prefix": prefix, + "SkipContents": True, + "SourceKind": "snapshot", + "Sources": [{"Component": "A", "Name": snapshot1_name}], + "Storage": "", + "Suite": "", + } + + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) + + self.check_exists("public/" + prefix + "/dists/" + distribution + "/Release") + self.check_exists("public/" + prefix + "/dists/" + distribution + "/A/binary-i386/Packages") + self.check_not_exists("public/" + prefix + "/dists/" + distribution + "/B/binary-i386/Packages") + self.check_not_exists("public/" + prefix + "/dists/" + distribution + "/C/binary-i386/Packages")