From a6e541ae8a2180c6c0738bb26334d02f578f0caf Mon Sep 17 00:00:00 2001 From: AstaFrode Date: Wed, 14 Aug 2024 11:44:14 +0800 Subject: [PATCH] Sby10 (#175) * Added allowed request headers, decryption, and log optimization * update readme * update env name * update readme --- .env.local | 2 +- README.md | 273 +-------------------------------------------- cmd/cmd/run.go | 2 +- node/node.go | 2 + node/open_file.go | 6 +- node/put_chunks.go | 18 ++- node/retrieve.go | 47 ++++---- node/tasks.go | 2 + 8 files changed, 46 insertions(+), 306 deletions(-) diff --git a/.env.local b/.env.local index 8c97a23..3ba4b95 100644 --- a/.env.local +++ b/.env.local @@ -19,7 +19,7 @@ rpc="wss://testnet-rpc.cess.network/ws/" # communication ports in the storage network -sport="4001" +storage_port="4001" # bootstrap nodes in the storage network, multiple separated by spaces boot="_dnsaddr.boot-miner-testnet.cess.network" diff --git a/README.md b/README.md index e376135..40ba122 100644 --- a/README.md +++ b/README.md @@ -86,10 +86,7 @@ service iptables restart ## 🏗 Get the binary program ### Method one -Download the latest release of the binary application directly at： -``` -wget https://github.com/CESSProject/DeOSS/releases/download/v0.3.7/DeOSS0.3.7.linux-amd64.tar.gz -``` +Download the latest release code directly at： [Releases](https://github.com/CESSProject/DeOSS/releases) ### Method two Compile the binary program from the DeOSS source code and follow the process as follows: @@ -243,273 +240,7 @@ It is generally not recommended to use this command： # 📖 Usage for API -The public API endpoint URL of DeOSS is the server you deploy, All endpoints described in this document should be made relative to this root URL,The following example uses URL instead. - -**Before using DeOSS, you must authorize it as follows:** - -1. Create a wallet account and fund it, refer to [Configure Wallet](https://github.com/CESSProject/DeOSS#configure-wallet) - -2. [Purchase a territory](https://github.com/CESSProject/doc-v2/blob/main/products/deoss/picture/buy_territory.png) - -3. Authorize the use right to DeOSS:[Authorize](https://github.com/CESSProject/W3F-illustration/blob/4995c1584006823990806b9d30fa7d554630ec14/deoss/authorizeOss.png) - - -## Identity signature - -Calling some APIs requires authentication of your identity. In web3, your wallet is your identity. Generate your signature data in [the block browser](https://polkadot.js.org/apps/), and then add your signature information in the API request header to authenticate your identity. Please refer to [the signature method](https://github.com/CESSProject/doc-v2/blob/main/products/deoss/picture/sign.png). - -The authentication information you need to add in the header: - -| Key | Description | Example | -| --------- | -------------- | ------- | -| Account | wallet account | cX... | -| Message | signed message | ... | -| Signature | signature | 0x... | - - -## Create a bucket - -| **PUT** /bucket | -| ---------------- | - -The put bucket interface is used to create a bucket. When uploading files, the bucket must be specified for storage. - -- Request Header - -| key | value | -| ------------- | ------------------- | -| Bucket | created bucket name | - -_Identity signature required: yes_ - -- Request example - -```shell -# curl -X PUT URL/ -H "Bucket: bucket_name" -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..." -``` - -## Upload a file - -| **PUT** /file | -| -------------- | - -The put file interface is used to upload files to the cess system. You need to submit the file as form data and use provide the specific field. -If the upload is successful, you will get the fid of the file. If you want to encrypt your file, you can specify the `cipher` field in the header and enter your password (the length cannot exceed 32 characters), and the system will automatically encrypt it. - -- Request Header - -| key | description | -| ---------------- | ------------------ | -| Bucket | bucket name | -| Territory | territory name | -| Cipher(optional) | cipher | - -_Identity signature required: yes_ - -- Request Body - -| key | value | -| ---- | ------------ | -| file | file[binary] | - -- Request example - -```shell -# curl -X PUT URL/file -F 'file=@test.log;type=application/octet-stream' -H "Bucket: bucket_name" -H "Territory: territory_name" -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..." -``` - -## Upload an object - -| **PUT** /object | -| ---------------- | - -This interface is used to upload an object, you can write what you want to store directly in the body instead of specifying a file. -If the upload is successful, you will get the fid of the object. if you want to encrypt the object, you can specify the "Cipher" field in the header of the request and enter a password (the length can not be more than 32 characters), the system will encrypt it automatically. - -- Request Header - -| key | description | -| ---------------- | ------------------ | -| Bucket | bucket name | -| Territory | territory name | -| Cipher(optional) | cipher | - -_Identity signature required: yes_ - -- Request Body - -[content] - - -- Request example - -```shell -# curl -X PUT URL/object --data "content" -H "Bucket: bucket_name" -H "Territory: territory_name" -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..." -``` - -## Chunked upload - -| **PUT** /chunks | -| ---------------- | - -Compared with uploading the entire file directly, resumable upload has some more parameter requirements, but has the same return result. At the same time, the uploaded file can also be encrypted. - -- Request Header - -| key | description | -| ------------- | ------------------ | -| Bucket | stored bucket name | -| Territory | territory name | -| Cipher(optional) | your cipher | -| FileName | file name or alias | -| BlockNumber | The number of chunks the file is to be divided into | -| BlockIndex | index of chunk to be uploaded, [0,BlockNumber) | -| TotalSize | the byte size of the file, the sum of the sizes of all chunks | - -_Identity signature required: yes_ - -- Request Body - -| key | value | -| ---- | ------------ | -| file | file[binary] | - -- Request example - -```shell -# curl -X PUT URL/chunks -F 'file=@test-chunk0;type=application/octet-stream' -H "Bucket: bucket_name" -H "Territory: territory_name" -H "Account: cX..." -H "Message: ..." -H "Signature: 0x... -H FileName: test.log -H BlockNumber: 5 -H BlockIndex: 0 -H TotalSize: 1000" -``` - -## Download a file - -| **GET** /download/{fid} | -| ------------------------ | - -This interface is used to download a file with a specified fid. If you encrypted the file when you uploaded it, you also need to tell the gateway your cipher to decrypt your file. - -- Request Header - -| key | value | -| ---------------- | -------- | -| Cipher(optional) | cipher | - - -- Request example - -```shell -# curl -X GET -o URL/download/ -``` - -## Preview a file - -| **GET** /open/{fid} | -| -------------------- | - -This interface is used to preview a file, it has two prerequisites: one is that the file is not encrypted, and the other is that the file format supports preview. - -- Request example - -Open in browser: URL/open/ - - -## Delete a file - -The delete file interface is used for delete a file. - -| **DELETE** /file/{fid} | -| ----------------------- | - -- Request Header - -_Identity signature required: yes_ - -- Request example - -```shell -# curl -X DELETE URL/file/ -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..." -``` - -## Delete a bucket - -The delete bucket interface is used for delete a bucket, all files in the bucket will also be deleted together. - -| **DELETE** /bucket/{bucket_name} | -| -------------------------------- | - -- Request Header - -_Identity signature required: yes_ - -- Request example - -```shell -# curl -X DELETE URL/bucket/ -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..." -``` - -## View bucket info - -| **GET** /bucket | -| ---------------- | - -This interface is used to view bucket information, including the number of stored files and file IDs. - -- Request Header - -| key | value | -| ------- | ----------- | -| Account | cX... | -| Bucket | bucket_name | - -- Request example - -```shell -# curl -X GET URL/bucket -H "Account: cX..." -H "Bucket: bucket_name" -``` - -## View bucket list - -| **GET** /bucket | -| ---------------- | - -- Request Header - -| key | value | -| ------- | ------ | -| Account | cX... | - -This interface is used to view all buckets. - -- Request example - -```shell -# curl -X GET URL/bucket -H "Account: cX..." -``` - -## View file metadata - -| **GET** /metadata/{fid} | -| ------------------------ | - -This interface is used to view the basic information of a file. - -- Request example - -```shell -# curl -X GET URL/metadata/ -``` - -## View version - -| **GET** /version | -| ----------------- | - -This interface is used to view the version number of the gateway. - -- Request example - -```shell -# curl -X GET URL/version -``` +Please refer to [API Description](https://doc.cess.network/products/deoss/api_description) ## License diff --git a/cmd/cmd/run.go b/cmd/cmd/run.go index 2e850fb..a5240a4 100644 --- a/cmd/cmd/run.go +++ b/cmd/cmd/run.go @@ -234,7 +234,7 @@ func readEnv() (*confile.Config, error) { } // storage network port - sport, err := strconv.Atoi(os.Getenv("sport")) + sport, err := strconv.Atoi(os.Getenv("storage_port")) if err != nil { return nil, errors.Errorf("invalid storage network port: %v", err) } diff --git a/node/node.go b/node/node.go index d012e24..8120365 100644 --- a/node/node.go +++ b/node/node.go @@ -68,6 +68,8 @@ func (n *Node) Run() { n.Engine = gin.Default() config := cors.DefaultConfig() config.AllowAllOrigins = true + config.AddAllowHeaders("*") + config.AddExposeHeaders("*") n.Engine.MaxMultipartMemory = MaxMemUsed n.Engine.Use(cors.New(config)) diff --git a/node/open_file.go b/node/open_file.go index 175d655..47e8d8f 100644 --- a/node/open_file.go +++ b/node/open_file.go @@ -137,7 +137,11 @@ func (n *Node) Preview_file(c *gin.Context) { fpath, err = n.retrieve_file(fid, n.fileDir, "") if err != nil { n.Logopen("err", fmt.Sprintf("[%s] Download file [%s] : %v", clientIp, fid, err)) - c.JSON(http.StatusInternalServerError, "File download failed, please try again later.") + if strings.Contains(err.Error(), "being retrieved") { + c.JSON(http.StatusForbidden, err.Error()) + return + } + c.JSON(http.StatusInternalServerError, "File download failed, it is recommended to use another gateway.") return } n.Logopen("info", fmt.Sprintf("[%s] Download file [%s] suc", clientIp, fid)) diff --git a/node/put_chunks.go b/node/put_chunks.go index 556f97a..e630fdc 100644 --- a/node/put_chunks.go +++ b/node/put_chunks.go @@ -72,7 +72,12 @@ func init() { } func (n *Node) PutChunksHandle(c *gin.Context) { + clientIp := c.Request.Header.Get("X-Forwarded-For") account := c.Request.Header.Get(HTTPHeader_Account) + if clientIp == "" { + clientIp = c.ClientIP() + } + chunkReqLock.Lock() _, ok := chunkReq[account] chunkReqLock.Unlock() @@ -89,6 +94,7 @@ func (n *Node) PutChunksHandle(c *gin.Context) { chunkReqLock.Lock() chunkReq[account] = time.Now().Unix() chunkReqLock.Unlock() + n.Logchunk("info", clientIp+" chunk upload: "+account) } var ( @@ -97,7 +103,6 @@ func (n *Node) PutChunksHandle(c *gin.Context) { chunksInfo ChunksInfo ) - clientIp := c.Request.Header.Get("X-Forwarded-For") bucketName := c.Request.Header.Get(HTTPHeader_Bucket) territoryName := c.Request.Header.Get(HTTPHeader_Territory) cipher := c.Request.Header.Get(HTTPHeader_Cipher) @@ -115,20 +120,13 @@ func (n *Node) PutChunksHandle(c *gin.Context) { } contentLength := c.Request.ContentLength - if clientIp == "" { - clientIp = c.ClientIP() - } - shuntminers := c.Request.Header.Values(HTTPHeader_Miner) longitudes := c.Request.Header.Values(HTTPHeader_Longitude) latitudes := c.Request.Header.Values(HTTPHeader_Latitude) - shuntminerslength := len(shuntminers) - if shuntminerslength > 0 { - n.Logput("info", fmt.Sprintf("shuntminers: %d, %v", shuntminerslength, shuntminers)) - } + points, err := coordinate.ConvertToRange(longitudes, latitudes) if err != nil { - n.Logput("err", clientIp+" "+err.Error()) + n.Logchunk("err", clientIp+" "+err.Error()) c.JSON(http.StatusBadRequest, err.Error()) return } diff --git a/node/retrieve.go b/node/retrieve.go index 5c50a88..87691d6 100644 --- a/node/retrieve.go +++ b/node/retrieve.go @@ -9,6 +9,7 @@ import ( "time" sconfig "github.com/CESSProject/cess-go-sdk/config" + "github.com/CESSProject/cess-go-sdk/core/crypte" "github.com/CESSProject/cess-go-sdk/core/erasure" sutils "github.com/CESSProject/cess-go-sdk/utils" "github.com/CESSProject/p2p-go/core" @@ -30,25 +31,13 @@ func (n *Node) retrieve_file(fid, savedir, cipher string) (string, error) { userfile := filepath.Join(savedir, fid) ok := false retrieve_lock.Lock() - if _, ok = retrieve_files[fid]; !ok { - retrieve_files[fid] = struct{}{} - } - retrieve_lock.Unlock() + _, ok = retrieve_files[fid] if ok { - tick := time.NewTicker(time.Second * 3) - for { - select { - case <-tick.C: - retrieve_lock.Lock() - _, ok = retrieve_files[fid] - retrieve_lock.Unlock() - if !ok { - _, err := os.Stat(userfile) - return userfile, err - } - } - } + retrieve_lock.Unlock() + return "", errors.New("The file is being retrieved from the storage network, please try again later.") } + retrieve_files[fid] = struct{}{} + retrieve_lock.Unlock() defer func() { retrieve_lock.Lock() @@ -62,6 +51,7 @@ func (n *Node) retrieve_file(fid, savedir, cipher string) (string, error) { return userfile, nil } } + os.MkdirAll(savedir, 0755) f, err := os.Create(userfile) if err != nil { @@ -141,13 +131,26 @@ func (n *Node) retrieve_file(fid, savedir, cipher string) (string, error) { for i := 0; i < len(segmentspath); i++ { buf, err := os.ReadFile(segmentspath[i]) if err != nil { - fmt.Println("segmentspath not equal fmeta segmentspath") - os.Exit(0) + n.Logdown("err", fmt.Sprintf("ReadFile(%s): %v", segmentspath[i], err)) + return "", err } - if (writecount + 1) >= len(fmeta.SegmentList) { - f.Write(buf[:(fmeta.FileSize.Uint64() - uint64(writecount*sconfig.SegmentSize))]) + if cipher != "" { + buffer, err := crypte.AesCbcDecrypt(buf, []byte(cipher)) + if err != nil { + n.Logdown("err", fmt.Sprintf("AesCbcDecrypt(%s) with cipher: %s failed: %v", segmentspath[i], cipher, err)) + return "", err + } + if (writecount + 1) >= len(fmeta.SegmentList) { + f.Write(buffer[:(fmeta.FileSize.Uint64() - uint64(writecount*sconfig.SegmentSize))]) + } else { + f.Write(buffer) + } } else { - f.Write(buf) + if (writecount + 1) >= len(fmeta.SegmentList) { + f.Write(buf[:(fmeta.FileSize.Uint64() - uint64(writecount*sconfig.SegmentSize))]) + } else { + f.Write(buf) + } } writecount++ } diff --git a/node/tasks.go b/node/tasks.go index 612b15b..ca0185c 100644 --- a/node/tasks.go +++ b/node/tasks.go @@ -51,6 +51,8 @@ func (n *Node) TaskMgt() { if err != nil { n.Log("err", schain.ERR_RPC_CONNECTION.Error()) out.Err(schain.ERR_RPC_CONNECTION.Error()) + } else { + n.Log("info", "rpc reconnect suc: "+n.GetCurrentRpcAddr()) } } count++