From 85195b7ecf1d3f5a14f132466f4512878fcc49ab Mon Sep 17 00:00:00 2001 From: Tatiana Nesterenko Date: Wed, 20 Dec 2023 16:07:34 +0000 Subject: [PATCH] api: add HTTP-like object uploader Add `POST /upload/{cid}` API. Signed-off-by: Tatiana Nesterenko --- gen/models/address_for_upload.go | 89 +++++++++ gen/restapi/doc.go | 1 + gen/restapi/embedded_spec.go | 150 +++++++++++++++ gen/restapi/operations/neofs_rest_gw_api.go | 23 ++- .../operations/upload_container_object.go | 71 +++++++ .../upload_container_object_parameters.go | 111 +++++++++++ .../upload_container_object_responses.go | 124 +++++++++++++ handlers/api.go | 1 + handlers/objects.go | 173 ++++++++++++++++++ handlers/util.go | 76 +++++++- handlers/util_test.go | 48 ++++- spec/rest.yaml | 46 +++++ 12 files changed, 908 insertions(+), 5 deletions(-) create mode 100644 gen/models/address_for_upload.go create mode 100644 gen/restapi/operations/upload_container_object.go create mode 100644 gen/restapi/operations/upload_container_object_parameters.go create mode 100644 gen/restapi/operations/upload_container_object_responses.go diff --git a/gen/models/address_for_upload.go b/gen/models/address_for_upload.go new file mode 100644 index 0000000..0a30ced --- /dev/null +++ b/gen/models/address_for_upload.go @@ -0,0 +1,89 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// AddressForUpload Address of the object in NeoFS. +// Example: {"container_id":"5HZTn5qkRnmgSz9gSrw22CEdPPk6nQhkwf2Mgzyvkikv","object_id":"8N3o7Dtr6T1xteCt6eRwhpmJ7JhME58Hyu1dvaswuTDd"} +// +// swagger:model AddressForUpload +type AddressForUpload struct { + + // container id + // Required: true + ContainerID *string `json:"container_id"` + + // object id + // Required: true + ObjectID *string `json:"object_id"` +} + +// Validate validates this address for upload +func (m *AddressForUpload) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateContainerID(formats); err != nil { + res = append(res, err) + } + + if err := m.validateObjectID(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *AddressForUpload) validateContainerID(formats strfmt.Registry) error { + + if err := validate.Required("container_id", "body", m.ContainerID); err != nil { + return err + } + + return nil +} + +func (m *AddressForUpload) validateObjectID(formats strfmt.Registry) error { + + if err := validate.Required("object_id", "body", m.ObjectID); err != nil { + return err + } + + return nil +} + +// ContextValidate validates this address for upload based on context it is used +func (m *AddressForUpload) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *AddressForUpload) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *AddressForUpload) UnmarshalBinary(b []byte) error { + var res AddressForUpload + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/gen/restapi/doc.go b/gen/restapi/doc.go index 67ffad3..b87030c 100644 --- a/gen/restapi/doc.go +++ b/gen/restapi/doc.go @@ -11,6 +11,7 @@ // // Consumes: // - application/json +// - multipart/form-data // // Produces: // - application/octet-stream diff --git a/gen/restapi/embedded_spec.go b/gen/restapi/embedded_spec.go index 56e5b22..b4a839b 100644 --- a/gen/restapi/embedded_spec.go +++ b/gen/restapi/embedded_spec.go @@ -979,6 +979,59 @@ func init() { "$ref": "#/parameters/objectId" } ] + }, + "/upload/{containerId}": { + "post": { + "security": [ + {}, + { + "BearerAuth": [] + }, + { + "CookieAuth": [] + } + ], + "consumes": [ + "multipart/form-data" + ], + "produces": [ + "application/json" + ], + "summary": "Upload object to NeoFS", + "operationId": "uploadContainerObject", + "parameters": [ + { + "type": "file", + "description": "The file to upload. If no file is present in this field, any other field name will be accepted, except for an empty one.", + "name": "payload", + "in": "formData" + } + ], + "responses": { + "200": { + "description": "Address of uploaded objects.", + "schema": { + "$ref": "#/definitions/AddressForUpload" + }, + "headers": { + "Access-Control-Allow-Origin": { + "type": "string" + } + } + }, + "400": { + "description": "Bad request.", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/containerId" + } + ] } }, "definitions": { @@ -1010,6 +1063,26 @@ func init() { "objectId": "8N3o7Dtr6T1xteCt6eRwhpmJ7JhME58Hyu1dvaswuTDd" } }, + "AddressForUpload": { + "description": "Address of the object in NeoFS.", + "type": "object", + "required": [ + "object_id", + "container_id" + ], + "properties": { + "container_id": { + "type": "string" + }, + "object_id": { + "type": "string" + } + }, + "example": { + "container_id": "5HZTn5qkRnmgSz9gSrw22CEdPPk6nQhkwf2Mgzyvkikv", + "object_id": "8N3o7Dtr6T1xteCt6eRwhpmJ7JhME58Hyu1dvaswuTDd" + } + }, "Attribute": { "description": "Attribute is a pair of strings that can be attached to a container or an object.", "type": "object", @@ -2924,6 +2997,63 @@ func init() { "required": true } ] + }, + "/upload/{containerId}": { + "post": { + "security": [ + {}, + { + "BearerAuth": [] + }, + { + "CookieAuth": [] + } + ], + "consumes": [ + "multipart/form-data" + ], + "produces": [ + "application/json" + ], + "summary": "Upload object to NeoFS", + "operationId": "uploadContainerObject", + "parameters": [ + { + "type": "file", + "description": "The file to upload. If no file is present in this field, any other field name will be accepted, except for an empty one.", + "name": "payload", + "in": "formData" + } + ], + "responses": { + "200": { + "description": "Address of uploaded objects.", + "schema": { + "$ref": "#/definitions/AddressForUpload" + }, + "headers": { + "Access-Control-Allow-Origin": { + "type": "string" + } + } + }, + "400": { + "description": "Bad request.", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + }, + "parameters": [ + { + "type": "string", + "description": "Base58 encoded container id.", + "name": "containerId", + "in": "path", + "required": true + } + ] } }, "definitions": { @@ -2955,6 +3085,26 @@ func init() { "objectId": "8N3o7Dtr6T1xteCt6eRwhpmJ7JhME58Hyu1dvaswuTDd" } }, + "AddressForUpload": { + "description": "Address of the object in NeoFS.", + "type": "object", + "required": [ + "object_id", + "container_id" + ], + "properties": { + "container_id": { + "type": "string" + }, + "object_id": { + "type": "string" + } + }, + "example": { + "container_id": "5HZTn5qkRnmgSz9gSrw22CEdPPk6nQhkwf2Mgzyvkikv", + "object_id": "8N3o7Dtr6T1xteCt6eRwhpmJ7JhME58Hyu1dvaswuTDd" + } + }, "Attribute": { "description": "Attribute is a pair of strings that can be attached to a container or an object.", "type": "object", diff --git a/gen/restapi/operations/neofs_rest_gw_api.go b/gen/restapi/operations/neofs_rest_gw_api.go index c052e48..bf53976 100644 --- a/gen/restapi/operations/neofs_rest_gw_api.go +++ b/gen/restapi/operations/neofs_rest_gw_api.go @@ -40,7 +40,8 @@ func NewNeofsRestGwAPI(spec *loads.Document) *NeofsRestGwAPI { APIKeyAuthenticator: security.APIKeyAuth, BearerAuthenticator: security.BearerAuth, - JSONConsumer: runtime.JSONConsumer(), + JSONConsumer: runtime.JSONConsumer(), + MultipartformConsumer: runtime.DiscardConsumer, BinProducer: runtime.ByteStreamProducer(), JSONProducer: runtime.JSONProducer(), @@ -114,6 +115,9 @@ func NewNeofsRestGwAPI(spec *loads.Document) *NeofsRestGwAPI { SearchObjectsHandler: SearchObjectsHandlerFunc(func(params SearchObjectsParams, principal *models.Principal) middleware.Responder { return middleware.NotImplemented("operation SearchObjects has not yet been implemented") }), + UploadContainerObjectHandler: UploadContainerObjectHandlerFunc(func(params UploadContainerObjectParams, principal *models.Principal) middleware.Responder { + return middleware.NotImplemented("operation UploadContainerObject has not yet been implemented") + }), // Applies when the "Authorization" header is set BearerAuthAuth: func(token string) (*models.Principal, error) { @@ -156,6 +160,9 @@ type NeofsRestGwAPI struct { // JSONConsumer registers a consumer for the following mime types: // - application/json JSONConsumer runtime.Consumer + // MultipartformConsumer registers a consumer for the following mime types: + // - multipart/form-data + MultipartformConsumer runtime.Consumer // BinProducer registers a producer for the following mime types: // - application/octet-stream @@ -221,6 +228,8 @@ type NeofsRestGwAPI struct { PutObjectHandler PutObjectHandler // SearchObjectsHandler sets the operation handler for the search objects operation SearchObjectsHandler SearchObjectsHandler + // UploadContainerObjectHandler sets the operation handler for the upload container object operation + UploadContainerObjectHandler UploadContainerObjectHandler // ServeError is called when an error is received, there is a default handler // but you can set your own with this @@ -293,6 +302,9 @@ func (o *NeofsRestGwAPI) Validate() error { if o.JSONConsumer == nil { unregistered = append(unregistered, "JSONConsumer") } + if o.MultipartformConsumer == nil { + unregistered = append(unregistered, "MultipartformConsumer") + } if o.BinProducer == nil { unregistered = append(unregistered, "BinProducer") @@ -377,6 +389,9 @@ func (o *NeofsRestGwAPI) Validate() error { if o.SearchObjectsHandler == nil { unregistered = append(unregistered, "SearchObjectsHandler") } + if o.UploadContainerObjectHandler == nil { + unregistered = append(unregistered, "UploadContainerObjectHandler") + } if len(unregistered) > 0 { return fmt.Errorf("missing registration: %s", strings.Join(unregistered, ", ")) @@ -425,6 +440,8 @@ func (o *NeofsRestGwAPI) ConsumersFor(mediaTypes []string) map[string]runtime.Co switch mt { case "application/json": result["application/json"] = o.JSONConsumer + case "multipart/form-data": + result["multipart/form-data"] = o.MultipartformConsumer } if c, ok := o.customConsumers[mt]; ok { @@ -576,6 +593,10 @@ func (o *NeofsRestGwAPI) initHandlerCache() { o.handlers["POST"] = make(map[string]http.Handler) } o.handlers["POST"]["/objects/{containerId}/search"] = NewSearchObjects(o.context, o.SearchObjectsHandler) + if o.handlers["POST"] == nil { + o.handlers["POST"] = make(map[string]http.Handler) + } + o.handlers["POST"]["/upload/{containerId}"] = NewUploadContainerObject(o.context, o.UploadContainerObjectHandler) } // Serve creates a http handler to serve the API over HTTP diff --git a/gen/restapi/operations/upload_container_object.go b/gen/restapi/operations/upload_container_object.go new file mode 100644 index 0000000..4fe2270 --- /dev/null +++ b/gen/restapi/operations/upload_container_object.go @@ -0,0 +1,71 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime/middleware" + + "github.com/nspcc-dev/neofs-rest-gw/gen/models" +) + +// UploadContainerObjectHandlerFunc turns a function with the right signature into a upload container object handler +type UploadContainerObjectHandlerFunc func(UploadContainerObjectParams, *models.Principal) middleware.Responder + +// Handle executing the request and returning a response +func (fn UploadContainerObjectHandlerFunc) Handle(params UploadContainerObjectParams, principal *models.Principal) middleware.Responder { + return fn(params, principal) +} + +// UploadContainerObjectHandler interface for that can handle valid upload container object params +type UploadContainerObjectHandler interface { + Handle(UploadContainerObjectParams, *models.Principal) middleware.Responder +} + +// NewUploadContainerObject creates a new http.Handler for the upload container object operation +func NewUploadContainerObject(ctx *middleware.Context, handler UploadContainerObjectHandler) *UploadContainerObject { + return &UploadContainerObject{Context: ctx, Handler: handler} +} + +/* UploadContainerObject swagger:route POST /upload/{containerId} uploadContainerObject + +Upload object to NeoFS + +*/ +type UploadContainerObject struct { + Context *middleware.Context + Handler UploadContainerObjectHandler +} + +func (o *UploadContainerObject) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + route, rCtx, _ := o.Context.RouteInfo(r) + if rCtx != nil { + *r = *rCtx + } + var Params = NewUploadContainerObjectParams() + uprinc, aCtx, err := o.Context.Authorize(r, route) + if err != nil { + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + if aCtx != nil { + *r = *aCtx + } + var principal *models.Principal + if uprinc != nil { + principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise + } + + if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + + res := o.Handler.Handle(Params, principal) // actually handle the request + o.Context.Respond(rw, r, route.Produces, route, res) + +} diff --git a/gen/restapi/operations/upload_container_object_parameters.go b/gen/restapi/operations/upload_container_object_parameters.go new file mode 100644 index 0000000..36a38dc --- /dev/null +++ b/gen/restapi/operations/upload_container_object_parameters.go @@ -0,0 +1,111 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "io" + "mime/multipart" + "net/http" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/strfmt" +) + +// UploadContainerObjectMaxParseMemory sets the maximum size in bytes for +// the multipart form parser for this operation. +// +// The default value is 32 MB. +// The multipart parser stores up to this + 10MB. +var UploadContainerObjectMaxParseMemory int64 = 32 << 20 + +// NewUploadContainerObjectParams creates a new UploadContainerObjectParams object +// +// There are no default values defined in the spec. +func NewUploadContainerObjectParams() UploadContainerObjectParams { + + return UploadContainerObjectParams{} +} + +// UploadContainerObjectParams contains all the bound params for the upload container object operation +// typically these are obtained from a http.Request +// +// swagger:parameters uploadContainerObject +type UploadContainerObjectParams struct { + + // HTTP Request Object + HTTPRequest *http.Request `json:"-"` + + /*Base58 encoded container id. + Required: true + In: path + */ + ContainerID string + /*The file to upload. If no file is present in this field, any other field name will be accepted, except for an empty one. + In: formData + */ + Payload io.ReadCloser +} + +// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface +// for simple values it will use straight method calls. +// +// To ensure default values, the struct must have been initialized with NewUploadContainerObjectParams() beforehand. +func (o *UploadContainerObjectParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { + var res []error + + o.HTTPRequest = r + + if err := r.ParseMultipartForm(UploadContainerObjectMaxParseMemory); err != nil { + if err != http.ErrNotMultipart { + return errors.New(400, "%v", err) + } else if err := r.ParseForm(); err != nil { + return errors.New(400, "%v", err) + } + } + + rContainerID, rhkContainerID, _ := route.Params.GetOK("containerId") + if err := o.bindContainerID(rContainerID, rhkContainerID, route.Formats); err != nil { + res = append(res, err) + } + + payload, payloadHeader, err := r.FormFile("payload") + if err != nil && err != http.ErrMissingFile { + res = append(res, errors.New(400, "reading file %q failed: %v", "payload", err)) + } else if err == http.ErrMissingFile { + // no-op for missing but optional file parameter + } else if err := o.bindPayload(payload, payloadHeader); err != nil { + res = append(res, err) + } else { + o.Payload = &runtime.File{Data: payload, Header: payloadHeader} + } + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +// bindContainerID binds and validates parameter ContainerID from path. +func (o *UploadContainerObjectParams) bindContainerID(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: true + // Parameter is provided by construction from the route + o.ContainerID = raw + + return nil +} + +// bindPayload binds file parameter Payload. +// +// The only supported validations on files are MinLength and MaxLength +func (o *UploadContainerObjectParams) bindPayload(file multipart.File, header *multipart.FileHeader) error { + return nil +} diff --git a/gen/restapi/operations/upload_container_object_responses.go b/gen/restapi/operations/upload_container_object_responses.go new file mode 100644 index 0000000..719d9a2 --- /dev/null +++ b/gen/restapi/operations/upload_container_object_responses.go @@ -0,0 +1,124 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime" + + "github.com/nspcc-dev/neofs-rest-gw/gen/models" +) + +// UploadContainerObjectOKCode is the HTTP code returned for type UploadContainerObjectOK +const UploadContainerObjectOKCode int = 200 + +/*UploadContainerObjectOK Address of uploaded objects. + +swagger:response uploadContainerObjectOK +*/ +type UploadContainerObjectOK struct { + /* + + */ + AccessControlAllowOrigin string `json:"Access-Control-Allow-Origin"` + + /* + In: Body + */ + Payload *models.AddressForUpload `json:"body,omitempty"` +} + +// NewUploadContainerObjectOK creates UploadContainerObjectOK with default headers values +func NewUploadContainerObjectOK() *UploadContainerObjectOK { + + return &UploadContainerObjectOK{} +} + +// WithAccessControlAllowOrigin adds the accessControlAllowOrigin to the upload container object o k response +func (o *UploadContainerObjectOK) WithAccessControlAllowOrigin(accessControlAllowOrigin string) *UploadContainerObjectOK { + o.AccessControlAllowOrigin = accessControlAllowOrigin + return o +} + +// SetAccessControlAllowOrigin sets the accessControlAllowOrigin to the upload container object o k response +func (o *UploadContainerObjectOK) SetAccessControlAllowOrigin(accessControlAllowOrigin string) { + o.AccessControlAllowOrigin = accessControlAllowOrigin +} + +// WithPayload adds the payload to the upload container object o k response +func (o *UploadContainerObjectOK) WithPayload(payload *models.AddressForUpload) *UploadContainerObjectOK { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the upload container object o k response +func (o *UploadContainerObjectOK) SetPayload(payload *models.AddressForUpload) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *UploadContainerObjectOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + // response header Access-Control-Allow-Origin + + accessControlAllowOrigin := o.AccessControlAllowOrigin + if accessControlAllowOrigin != "" { + rw.Header().Set("Access-Control-Allow-Origin", accessControlAllowOrigin) + } + + rw.WriteHeader(200) + if o.Payload != nil { + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } + } +} + +// UploadContainerObjectBadRequestCode is the HTTP code returned for type UploadContainerObjectBadRequest +const UploadContainerObjectBadRequestCode int = 400 + +/*UploadContainerObjectBadRequest Bad request. + +swagger:response uploadContainerObjectBadRequest +*/ +type UploadContainerObjectBadRequest struct { + + /* + In: Body + */ + Payload *models.ErrorResponse `json:"body,omitempty"` +} + +// NewUploadContainerObjectBadRequest creates UploadContainerObjectBadRequest with default headers values +func NewUploadContainerObjectBadRequest() *UploadContainerObjectBadRequest { + + return &UploadContainerObjectBadRequest{} +} + +// WithPayload adds the payload to the upload container object bad request response +func (o *UploadContainerObjectBadRequest) WithPayload(payload *models.ErrorResponse) *UploadContainerObjectBadRequest { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the upload container object bad request response +func (o *UploadContainerObjectBadRequest) SetPayload(payload *models.ErrorResponse) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *UploadContainerObjectBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(400) + if o.Payload != nil { + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } + } +} diff --git a/handlers/api.go b/handlers/api.go index 2e44538..e65c5f3 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -129,6 +129,7 @@ func (a *API) Configure(api *operations.NeofsRestGwAPI) http.Handler { api.GetContainerObjectHandler = operations.GetContainerObjectHandlerFunc(a.GetContainerObject) api.HeadContainerObjectHandler = operations.HeadContainerObjectHandlerFunc(a.HeadContainerObject) + api.UploadContainerObjectHandler = operations.UploadContainerObjectHandlerFunc(a.UploadContainerObject) api.BearerAuthAuth = func(s string) (*models.Principal, error) { if !strings.HasPrefix(s, BearerPrefix) { diff --git a/handlers/objects.go b/handlers/objects.go index 55f8b38..c27d835 100644 --- a/handlers/objects.go +++ b/handlers/objects.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "mime/multipart" "net/http" "path" "strconv" @@ -18,6 +19,7 @@ import ( "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/swag" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-api-go/v2/acl" "github.com/nspcc-dev/neofs-api-go/v2/container" @@ -786,3 +788,174 @@ func attachOwner(obj *object.Object, btoken *bearer.Token) { obj.SetOwnerID(&owner) } } + +// UploadContainerObject handler that upload file as object with attributes to NeoFS. +func (a *API) UploadContainerObject(params operations.UploadContainerObjectParams, principal *models.Principal) middleware.Responder { + var ( + header *multipart.FileHeader + file multipart.File + err error + idObj oid.ID + addr oid.Address + btoken *bearer.Token + ) + errorResponse := operations.NewUploadContainerObjectBadRequest() + ctx := params.HTTPRequest.Context() + + var idCnr cid.ID + if err := idCnr.DecodeString(params.ContainerID); err != nil { + resp := a.logAndGetErrorResponse("invalid container id", err) + return errorResponse.WithPayload(resp) + } + + if principal != nil { + btoken, err = getBearerTokenFromString(string(*principal)) + if err != nil { + resp := a.logAndGetErrorResponse("get bearer token", err) + return errorResponse.WithPayload(resp) + } + } + + if swagFile, ok := params.Payload.(*swag.File); ok { + header = swagFile.Header + file = swagFile.Data + } else { + var fileKey string + for fileKey = range params.HTTPRequest.MultipartForm.File { + file, header, err = params.HTTPRequest.FormFile(fileKey) + if err != nil { + resp := a.logAndGetErrorResponse(fmt.Sprintf("get file %q from HTTP request", fileKey), err) + return errorResponse.WithPayload(resp) + } + break + } + if fileKey == "" { + resp := a.logAndGetErrorResponse("no multipart/form file", http.ErrMissingFile) + return errorResponse.WithPayload(resp) + } + } + + defer func() { + if file == nil { + return + } + err := file.Close() + a.log.Debug( + "close temporary multipart/form file", + zap.Stringer("address", addr), + zap.String("filename", header.Filename), + zap.Error(err), + ) + }() + + filtered, err := filterHeaders(a.log, params.HTTPRequest.Header) + if err != nil { + resp := a.logAndGetErrorResponse("could not process headers", err) + return errorResponse.WithPayload(resp) + } + + if needParseExpiration(filtered) { + epochDuration, err := getEpochDurations(ctx, a.pool) + if err != nil { + resp := a.logAndGetErrorResponse("could not get epoch durations from network info", err) + return errorResponse.WithPayload(resp) + } + + now := time.Now() + if rawHeader := params.HTTPRequest.Header.Get("Date"); rawHeader != "" { + if parsed, err := time.Parse(http.TimeFormat, rawHeader); err != nil { + a.log.Warn("could not parse client time", zap.String("Date header", rawHeader), zap.Error(err)) + } else { + now = parsed + } + } + + if err = prepareExpirationHeader(filtered, epochDuration, now); err != nil { + resp := a.logAndGetErrorResponse("could not parse expiration header", err) + return errorResponse.WithPayload(resp) + } + } + + attributes := make([]object.Attribute, 0, len(filtered)) + // prepares attributes from filtered headers + for key, val := range filtered { + attribute := object.NewAttribute() + attribute.SetKey(key) + attribute.SetValue(val) + attributes = append(attributes, *attribute) + } + // sets FileName attribute if it wasn't set from header + if _, ok := filtered[object.AttributeFileName]; !ok { + filename := object.NewAttribute() + filename.SetKey(object.AttributeFileName) + filename.SetValue(header.Filename) + attributes = append(attributes, *filename) + } + // sets Content-Type attribute if it wasn't set from header + if _, ok := filtered[object.AttributeContentType]; !ok { + if contentTypes, ok := header.Header["Content-Type"]; ok && len(contentTypes) > 0 { + contentType := contentTypes[0] + cType := object.NewAttribute() + cType.SetKey(object.AttributeContentType) + cType.SetValue(contentType) + attributes = append(attributes, *cType) + } + } + // sets Timestamp attribute if it wasn't set from header and enabled by settings + if _, ok := filtered[object.AttributeTimestamp]; !ok && a.defaultTimestamp { + timestamp := object.NewAttribute() + timestamp.SetKey(object.AttributeTimestamp) + timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) + attributes = append(attributes, *timestamp) + } + + var obj object.Object + obj.SetContainerID(idCnr) + a.setOwner(&obj, btoken) + obj.SetAttributes(attributes...) + + var prmPutInit client.PrmObjectPutInit + if btoken != nil { + prmPutInit.WithBearerToken(*btoken) + } + + writer, err := a.pool.ObjectPutInit(ctx, obj, a.signer, prmPutInit) + if err != nil { + resp := a.logAndGetErrorResponse("put object init", err) + return errorResponse.WithPayload(resp) + } + + chunk := make([]byte, a.maxObjectSize) + _, err = io.CopyBuffer(writer, file, chunk) + if err != nil { + resp := a.logAndGetErrorResponse("write", err) + return errorResponse.WithPayload(resp) + } + + if err = writer.Close(); err != nil { + resp := a.logAndGetErrorResponse("writer close", err) + return errorResponse.WithPayload(resp) + } + + idObj = writer.GetResult().StoredObjectID() + addr.SetObject(idObj) + addr.SetContainer(idCnr) + + var resp models.AddressForUpload + resp.ContainerID = ¶ms.ContainerID + resp.ObjectID = util.NewString(idObj.String()) + + return operations.NewUploadContainerObjectOK(). + WithPayload(&resp). + WithAccessControlAllowOrigin("*") +} + +func (a *API) setOwner(obj *object.Object, btoken *bearer.Token) { + if btoken != nil { + owner := btoken.ResolveIssuer() + obj.SetOwnerID(&owner) + } else { + ownerID := a.signer.UserID() + obj.SetOwnerID(&ownerID) + } +} diff --git a/handlers/util.go b/handlers/util.go index 62b0b68..77a7292 100644 --- a/handlers/util.go +++ b/handlers/util.go @@ -1,14 +1,17 @@ package handlers import ( + "bytes" "context" "errors" "fmt" "math" + "net/http" "strconv" "strings" "time" + "github.com/nspcc-dev/neofs-api-go/v2/container" objectv2 "github.com/nspcc-dev/neofs-api-go/v2/object" sessionv2 "github.com/nspcc-dev/neofs-api-go/v2/session" "github.com/nspcc-dev/neofs-rest-gw/gen/models" @@ -16,6 +19,7 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/container/acl" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/pool" + "go.uber.org/zap" ) // PrmAttributes groups parameters to form attributes from request headers. @@ -38,6 +42,8 @@ const ( ExpirationRFC3339Attr = SystemAttributePrefix + "EXPIRATION_RFC3339" ) +var neofsAttributeHeaderPrefix = []byte("Neofs-") + // GetObjectAttributes forms object attributes from request headers. func GetObjectAttributes(ctx context.Context, pool *pool.Pool, attrs []*models.Attribute, prm PrmAttributes) ([]object.Attribute, error) { headers := make(map[string]string, len(attrs)) @@ -52,7 +58,8 @@ func GetObjectAttributes(ctx context.Context, pool *pool.Pool, attrs []*models.A if err != nil { return nil, fmt.Errorf("could not get epoch durations from network info: %w", err) } - if err = prepareExpirationHeader(headers, epochDuration); err != nil { + now := time.Now().UTC() + if err = prepareExpirationHeader(headers, epochDuration, now); err != nil { return nil, fmt.Errorf("could not prepare expiration header: %w", err) } } @@ -105,7 +112,7 @@ func needParseExpiration(headers map[string]string) bool { return ok1 || ok2 || ok3 } -func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations) error { +func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations, now time.Time) error { expirationInEpoch := headers[objectv2.SysAttributeExpEpoch] if timeRFC3339, ok := headers[ExpirationRFC3339Attr]; ok { @@ -114,7 +121,6 @@ func prepareExpirationHeader(headers map[string]string, epochDurations *epochDur return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, ExpirationRFC3339Attr) } - now := time.Now().UTC() if expTime.Before(now) { return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, ExpirationRFC3339Attr) } @@ -240,3 +246,67 @@ func decodeBasicACL(input string) (acl.Basic, error) { return res, nil } } + +func systemTranslator(key, prefix []byte) []byte { + // replace the specified prefix with `__NEOFS__` + key = bytes.Replace(key, prefix, []byte(container.SysAttributePrefix), 1) + + // replace `-` with `_` + key = bytes.ReplaceAll(key, []byte("-"), []byte("_")) + + // replace with uppercase + return bytes.ToUpper(key) +} + +func filterHeaders(l *zap.Logger, header http.Header) (map[string]string, error) { + result := make(map[string]string) + prefix := []byte(userAttributeHeaderPrefix) + + for key, values := range header { + // check if key gets duplicated + // return error containing full key name (with prefix) + if len(values) > 1 { + return nil, fmt.Errorf("key duplication error: %s", key) + } + + // checks that the value is not empty + if len(values) == 0 { + continue + } + + keyBytes := []byte(key) + valueBytes := []byte(values[0]) + + // checks that the key and the val not empty + if len(keyBytes) == 0 || len(valueBytes) == 0 { + continue + } + + // checks that the key has attribute prefix + if !bytes.HasPrefix(keyBytes, prefix) { + continue + } + + // removing attribute prefix + clearKey := bytes.TrimPrefix(keyBytes, prefix) + + // checks that it's a system NeoFS header + if bytes.HasPrefix(clearKey, neofsAttributeHeaderPrefix) { + clearKey = systemTranslator(clearKey, neofsAttributeHeaderPrefix) + } + + // checks that the attribute key is not empty + if len(clearKey) == 0 { + continue + } + + // make string representation of key / val + k, v := string(clearKey), string(valueBytes) + result[k] = v + + l.Debug("add attribute to result object", + zap.String("key", k), + zap.String("val", v)) + } + return result, nil +} diff --git a/handlers/util_test.go b/handlers/util_test.go index 42ce968..acd9560 100644 --- a/handlers/util_test.go +++ b/handlers/util_test.go @@ -2,12 +2,14 @@ package handlers import ( "math" + "net/http" "strconv" "testing" "time" objectv2 "github.com/nspcc-dev/neofs-api-go/v2/object" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func TestPrepareExpirationHeader(t *testing.T) { @@ -153,7 +155,8 @@ func TestPrepareExpirationHeader(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - err := prepareExpirationHeader(tc.headers, tc.durations) + now := time.Now().UTC() + err := prepareExpirationHeader(tc.headers, tc.durations, now) if tc.err { require.Error(t, err) } else { @@ -163,3 +166,46 @@ func TestPrepareExpirationHeader(t *testing.T) { }) } } + +func TestFilter(t *testing.T) { + log := zap.NewNop() + + t.Run("duplicate keys error", func(t *testing.T) { + req := http.Header{} + req.Add("X-Attribute-Dup-Key", "first-value") + req.Add("X-Attribute-Dup-Key", "second-value") + _, err := filterHeaders(log, req) + require.Error(t, err) + }) + + t.Run("duplicate system keys error", func(t *testing.T) { + req := http.Header{} + req.Add("X-Attribute-Neofs-Dup-Key", "first-value") + req.Add("X-Attribute-Neofs-Dup-Key", "second-value") + _, err := filterHeaders(log, req) + require.Error(t, err) + }) + + req := http.Header{} + + req.Set("X-Attribute-Neofs-Expiration-Epoch1", "101") + req.Set("X-Attribute-NEOFS-Expiration-Epoch2", "102") + req.Set("X-Attribute-neofs-Expiration-Epoch3", "103") + req.Set("X-Attribute-My-Attribute", "value") + req.Set("X-Attribute-MyAttribute", "value2") + req.Set("X-Attribute-Empty-Value", "") + req.Set("X-Attribute-", "prefix only") + req.Set("No-Prefix", "value") + + expected := map[string]string{ + "__NEOFS__EXPIRATION_EPOCH1": "101", + "__NEOFS__EXPIRATION_EPOCH2": "102", + "__NEOFS__EXPIRATION_EPOCH3": "103", + "My-Attribute": "value", + "Myattribute": "value2", + } + + result, err := filterHeaders(log, req) + require.NoError(t, err) + require.Equal(t, expected, result) +} diff --git a/spec/rest.yaml b/spec/rest.yaml index 976e425..bdcc6d6 100644 --- a/spec/rest.yaml +++ b/spec/rest.yaml @@ -664,6 +664,38 @@ paths: description: Not found schema: $ref: '#/definitions/ErrorResponse' + /upload/{containerId}: + parameters: + - $ref: '#/parameters/containerId' + post: + operationId: uploadContainerObject + summary: Upload object to NeoFS + security: + - { } + - BearerAuth: [ ] + - CookieAuth: [ ] + consumes: + - multipart/form-data + produces: + - application/json + parameters: + - name: payload + in: formData + required: false + type: file + description: The file to upload. If no file is present in this field, any other field name will be accepted, except for an empty one. + responses: + 200: + headers: + Access-Control-Allow-Origin: + type: string + description: Address of uploaded objects. + schema: + $ref: '#/definitions/AddressForUpload' + 400: + description: Bad request. + schema: + $ref: '#/definitions/ErrorResponse' definitions: BinaryBearer: @@ -1200,3 +1232,17 @@ definitions: - success example: success: true + AddressForUpload: + description: Address of the object in NeoFS. + type: object + properties: + container_id: + type: string + object_id: + type: string + required: + - object_id + - container_id + example: + object_id: 8N3o7Dtr6T1xteCt6eRwhpmJ7JhME58Hyu1dvaswuTDd + container_id: 5HZTn5qkRnmgSz9gSrw22CEdPPk6nQhkwf2Mgzyvkikv