diff --git a/sdk/bucket.go b/bucket.go similarity index 97% rename from sdk/bucket.go rename to bucket.go index 56dc338..6c7daab 100644 --- a/sdk/bucket.go +++ b/bucket.go @@ -1,13 +1,13 @@ -package chainstoragesdk +package sdk import ( "encoding/json" "errors" "fmt" "github.com/kataras/golog" - "github.com/paradeum-team/chainstorage-sdk/sdk/code" - "github.com/paradeum-team/chainstorage-sdk/sdk/consts" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" + "github.com/paradeum-team/chainstorage-sdk/code" + "github.com/paradeum-team/chainstorage-sdk/consts" + "github.com/paradeum-team/chainstorage-sdk/model" "github.com/ulule/deepcopier" "net/http" "net/url" diff --git a/sdk/car.go b/car.go similarity index 75% rename from sdk/car.go rename to car.go index 9e7904b..242bc33 100644 --- a/sdk/car.go +++ b/car.go @@ -1,4 +1,4 @@ -package chainstoragesdk +package sdk import ( "bytes" @@ -12,19 +12,24 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipldfmt "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-unixfsnode" + "github.com/ipfs/go-unixfsnode/data" "github.com/ipfs/go-unixfsnode/data/builder" + "github.com/ipfs/go-unixfsnode/file" "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" + carstorage "github.com/ipld/go-car/v2/storage" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/kataras/golog" "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" - "github.com/paradeum-team/chainstorage-sdk/sdk/code" - "github.com/paradeum-team/chainstorage-sdk/sdk/consts" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" - "github.com/paradeum-team/chainstorage-sdk/sdk/utils" + "github.com/paradeum-team/chainstorage-sdk/code" + "github.com/paradeum-team/chainstorage-sdk/consts" + "github.com/paradeum-team/chainstorage-sdk/model" + "github.com/paradeum-team/chainstorage-sdk/utils" "github.com/ulule/deepcopier" "io" "net/http" @@ -52,7 +57,7 @@ func (c *Car) CreateCarFile(dataPath string, fileDestination string) error { func (c *Car) SplitCarFile(carFilePath string, chunkedFileDestinations *[]string) error { // CAR file chunking setting // todo: - targetSize := c.Config.CarFileShardingThreshold * 10 //1024 * 1024 // 1MiB chunks + targetSize := c.Config.CarFileShardingThreshold // CAR文件分片阈值（固定44Mb） strategy := carbites.Treewalk return chunkCarFile(carFilePath, targetSize, strategy, chunkedFileDestinations) @@ -130,14 +135,16 @@ func (c *Car) UploadCarFile(req *model.CarFileUploadReq) (model.ObjectCreateResp fileDestination := req.FileDestination carFileCid := req.CarFileCid objectSize := strconv.FormatInt(req.ObjectSize, 10) + objectTypeCode := strconv.Itoa(req.ObjectTypeCode) params := map[string]string{ - "bucketId": strconv.Itoa(bucketId), - "rawSha256": rawSha256, - "objectCid": objectCid, - "carFileCid": carFileCid, - "objectName": objectName, - "objectSize": objectSize, + "bucketId": strconv.Itoa(bucketId), + "rawSha256": rawSha256, + "objectCid": objectCid, + "carFileCid": carFileCid, + "objectName": objectName, + "objectSize": objectSize, + "objectTypeCode": objectTypeCode, } //params := map[string]interface{}{ // "bucketId": bucketId, @@ -586,8 +593,8 @@ func (c *Car) SliceBigCarFile(carFilePath string) error { } defer bigCarFile.Close() - targetSize := c.Config.CarFileShardingThreshold * 10 //1024 * 1024 // 1MiB chunks - strategy := carbites.Treewalk // also carbites.Treewalk + targetSize := c.Config.CarFileShardingThreshold // CAR文件分片阈值（固定44Mb） + strategy := carbites.Treewalk // also carbites.Treewalk spltr, _ := carbites.Split(bigCarFile, targetSize, strategy) var i int @@ -621,7 +628,7 @@ func (c *Car) GenerateShardingCarFiles(req *model.CarFileUploadReq, shardingCarF defer bigCarFile.Close() // CAR文件分片设置 - targetSize := c.Config.CarFileShardingThreshold //1024 * 1024 // 1MiB chunks + targetSize := c.Config.CarFileShardingThreshold // CAR文件分片阈值（固定44Mb） strategy := carbites.Treewalk // also carbites.Treewalk spltr, _ := carbites.Split(bigCarFile, targetSize, strategy) @@ -875,14 +882,16 @@ func (c *Car) UploadCarFileExt(req *model.CarFileUploadReq, extReader io.Reader) fileDestination := req.FileDestination carFileCid := req.CarFileCid objectSize := strconv.FormatInt(req.ObjectSize, 10) + objectTypeCode := strconv.Itoa(req.ObjectTypeCode) params := map[string]string{ - "bucketId": strconv.Itoa(bucketId), - "rawSha256": rawSha256, - "objectCid": objectCid, - "carFileCid": carFileCid, - "objectName": objectName, - "objectSize": objectSize, + "bucketId": strconv.Itoa(bucketId), + "rawSha256": rawSha256, + "objectCid": objectCid, + "carFileCid": carFileCid, + "objectName": objectName, + "objectSize": objectSize, + "objectTypeCode": objectTypeCode, } //params := map[string]interface{}{ // "bucketId": bucketId, @@ -990,14 +999,16 @@ func (c *Car) ImportCarFileExt(req *model.CarFileUploadReq, extReader io.Reader) fileDestination := req.FileDestination carFileCid := req.CarFileCid objectSize := strconv.FormatInt(req.ObjectSize, 10) + objectTypeCode := strconv.Itoa(req.ObjectTypeCode) params := map[string]string{ - "bucketId": strconv.Itoa(bucketId), - "rawSha256": rawSha256, - "objectCid": objectCid, - "carFileCid": carFileCid, - "objectName": objectName, - "objectSize": objectSize, + "bucketId": strconv.Itoa(bucketId), + "rawSha256": rawSha256, + "objectCid": objectCid, + "carFileCid": carFileCid, + "objectName": objectName, + "objectSize": objectSize, + "objectTypeCode": objectTypeCode, } //params := map[string]interface{}{ // "bucketId": bucketId, @@ -1088,3 +1099,347 @@ func (c *Car) ImportShardingCarFileExt(req *model.CarFileUploadReq, extReader io return response, nil } + +// 提取CAR文件 +func (c *Car) ExtractCarFile(carFilePath string, dataDestination string) error { + carFile, err := os.Open(carFilePath) + if err != nil { + return err + } + + store, err := carstorage.OpenReadable(carFile) + if err != nil { + return err + } + + roots := store.(carstorage.ReadableCar).Roots() + + ls := cidlink.DefaultLinkSystem() + ls.TrustedStorage = true + ls.SetReadStorage(store) + + // The unixfs path to extract + unixfsPath := "" + path, err := pathSegments(unixfsPath) + if err != nil { + return err + } + + var extractedFiles int + for _, root := range roots { + count, err := c.extractRoot(&ls, root, dataDestination, path) + if err != nil { + return err + } + + extractedFiles += count + } + + return errors.New("no files extracted") + + //if extractedFiles == 0 { + // return cli.Exit("no files extracted", 1) + //} else { + // fmt.Fprintf(c.App.ErrWriter, "extracted %d file(s)\n", extractedFiles) + //} +} + +func (c *Car) extractRoot(ls *ipld.LinkSystem, root cid.Cid, outputDir string, path []string) (int, error) { + if root.Prefix().Codec == cid.Raw { + //if c.IsSet("verbose") { + // fmt.Fprintf(c.App.ErrWriter, "skipping raw root %s\n", root) + //} + return 0, nil + } + + pbn, err := ls.Load(ipld.LinkContext{}, cidlink.Link{Cid: root}, dagpb.Type.PBNode) + if err != nil { + return 0, err + } + pbnode := pbn.(dagpb.PBNode) + + ufn, err := unixfsnode.Reify(ipld.LinkContext{}, pbnode, ls) + if err != nil { + return 0, err + } + + var outputResolvedDir string + if outputDir != "-" { + outputResolvedDir, err = filepath.EvalSymlinks(outputDir) + if err != nil { + return 0, err + } + if _, err := os.Stat(outputResolvedDir); os.IsNotExist(err) { + if err := os.Mkdir(outputResolvedDir, 0755); err != nil { + return 0, err + } + } + } + + count, err := c.extractDir(ls, ufn, outputResolvedDir, "/", path) + if err != nil { + if !errors.Is(err, ErrNotDir) { + return 0, fmt.Errorf("%s: %w", root, err) + } + + // if it's not a directory, it's a file. + ufsData, err := pbnode.LookupByString("Data") + if err != nil { + return 0, err + } + + ufsBytes, err := ufsData.AsBytes() + if err != nil { + return 0, err + } + + ufsNode, err := data.DecodeUnixFSData(ufsBytes) + if err != nil { + return 0, err + } + + var outputName string + if outputDir != "-" { + outputName = filepath.Join(outputResolvedDir, "unknown") + } + + if ufsNode.DataType.Int() == data.Data_File || ufsNode.DataType.Int() == data.Data_Raw { + if err := c.extractFile(ls, pbnode, outputName); err != nil { + return 0, err + } + } + + return 1, nil + } + + return count, nil +} + +func (c *Car) extractDir(ls *ipld.LinkSystem, n ipld.Node, outputRoot, outputPath string, matchPath []string) (int, error) { + if outputRoot != "" { + dirPath, err := resolvePath(outputRoot, outputPath) + if err != nil { + return 0, err + } + // make the directory. + if err := os.MkdirAll(dirPath, 0755); err != nil { + return 0, err + } + } + + if n.Kind() != ipld.Kind_Map { + return 0, ErrNotDir + } + + subPath := matchPath + if len(matchPath) > 0 { + subPath = matchPath[1:] + } + + extractElement := func(name string, n ipld.Node) (int, error) { + var nextRes string + if outputRoot != "" { + var err error + nextRes, err = resolvePath(outputRoot, path.Join(outputPath, name)) + if err != nil { + return 0, err + } + //if c.IsSet("verbose") { + // fmt.Fprintf(c.App.ErrWriter, "%s\n", nextRes) + //} + } + + if n.Kind() != ipld.Kind_Link { + return 0, fmt.Errorf("unexpected map value for %s at %s", name, outputPath) + } + // a directory may be represented as a map of name: if unixADL is applied + vl, err := n.AsLink() + if err != nil { + return 0, err + } + + dest, err := ls.Load(ipld.LinkContext{}, vl, basicnode.Prototype.Any) + if err != nil { + if nf, ok := err.(interface{ NotFound() bool }); ok && nf.NotFound() { + c.logger.Infof("data for entry not found: %s (skipping...)\n", path.Join(outputPath, name)) + //fmt.Fprintf(c.App.ErrWriter, "data for entry not found: %s (skipping...)\n", path.Join(outputPath, name)) + return 0, nil + } + return 0, err + } + + // degenerate files are handled here. + if dest.Kind() == ipld.Kind_Bytes { + if err := c.extractFile(ls, dest, nextRes); err != nil { + return 0, err + } + + return 1, nil + } + + // dir / pbnode + pbb := dagpb.Type.PBNode.NewBuilder() + if err := pbb.AssignNode(dest); err != nil { + return 0, err + } + pbnode := pbb.Build().(dagpb.PBNode) + + // interpret dagpb 'data' as unixfs data and look at type. + ufsData, err := pbnode.LookupByString("Data") + if err != nil { + return 0, err + } + + ufsBytes, err := ufsData.AsBytes() + if err != nil { + return 0, err + } + + ufsNode, err := data.DecodeUnixFSData(ufsBytes) + if err != nil { + return 0, err + } + + switch ufsNode.DataType.Int() { + case data.Data_Directory, data.Data_HAMTShard: + ufn, err := unixfsnode.Reify(ipld.LinkContext{}, pbnode, ls) + if err != nil { + return 0, err + } + return c.extractDir(ls, ufn, outputRoot, path.Join(outputPath, name), subPath) + + case data.Data_File, data.Data_Raw: + if err := c.extractFile(ls, pbnode, nextRes); err != nil { + return 0, err + } + return 1, nil + + case data.Data_Symlink: + if nextRes == "" { + return 0, fmt.Errorf("cannot extract a symlink to stdout") + } + data := ufsNode.Data.Must().Bytes() + if err := os.Symlink(string(data), nextRes); err != nil { + return 0, err + } + return 1, nil + + default: + return 0, fmt.Errorf("unknown unixfs type: %d", ufsNode.DataType.Int()) + } + } + + // specific path segment + if len(matchPath) > 0 { + val, err := n.LookupByString(matchPath[0]) + if err != nil { + return 0, err + } + return extractElement(matchPath[0], val) + } + + if outputPath == "-" && len(matchPath) == 0 { + return 0, fmt.Errorf("cannot extract a directory to stdout, use a path to extract a specific file") + } + + // everything + var count int + var shardSkip int + mi := n.MapIterator() + for !mi.Done() { + key, val, err := mi.Next() + if err != nil { + if nf, ok := err.(interface{ NotFound() bool }); ok && nf.NotFound() { + shardSkip++ + continue + } + return 0, err + } + + ks, err := key.AsString() + if err != nil { + return 0, err + } + + ecount, err := extractElement(ks, val) + if err != nil { + return 0, err + } + + count += ecount + } + + if shardSkip > 0 { + c.logger.Infof("data for entry not found for %d unknown sharded entries (skipped...)\n", shardSkip) + //fmt.Fprintf(c.App.ErrWriter, "data for entry not found for %d unknown sharded entries (skipped...)\n", shardSkip) + } + + return count, nil +} + +func (c *Car) extractFile(ls *ipld.LinkSystem, n ipld.Node, outputName string) error { + ctx := context.Background() + node, err := file.NewUnixFSFile(ctx, n, ls) + if err != nil { + return err + } + + nlr, err := node.AsLargeBytes() + if err != nil { + return err + } + + var f *os.File + if outputName == "" { + f = os.Stdout + } else { + f, err = os.Create(outputName) + if err != nil { + return err + } + defer f.Close() + } + _, err = io.Copy(f, nlr) + + return err +} + +// TODO: dedupe this with lassie, probably into go-unixfsnode +func pathSegments(path string) ([]string, error) { + segments := strings.Split(path, "/") + filtered := make([]string, 0, len(segments)) + for i := 0; i < len(segments); i++ { + if segments[i] == "" { + // Allow one leading and one trailing '/' at most + if i == 0 || i == len(segments)-1 { + continue + } + return nil, fmt.Errorf("invalid empty path segment at position %d", i) + } + if segments[i] == "." || segments[i] == ".." { + return nil, fmt.Errorf("'%s' is unsupported in paths", segments[i]) + } + filtered = append(filtered, segments[i]) + } + return filtered, nil +} + +func resolvePath(root, pth string) (string, error) { + rp, err := filepath.Rel("/", pth) + if err != nil { + return "", fmt.Errorf("couldn't check relative-ness of %s: %w", pth, err) + } + joined := path.Join(root, rp) + + basename := path.Dir(joined) + final, err := filepath.EvalSymlinks(basename) + if err != nil { + return "", fmt.Errorf("couldn't eval symlinks in %s: %w", basename, err) + } + if final != path.Clean(basename) { + return "", fmt.Errorf("path attempts to redirect through symlinks") + } + return joined, nil +} + +var ErrNotDir = fmt.Errorf("not a directory") diff --git a/sdk/client.go b/client.go similarity index 54% rename from sdk/client.go rename to client.go index 50082f8..90cc05d 100644 --- a/sdk/client.go +++ b/client.go @@ -1,10 +1,10 @@ -package chainstoragesdk +package sdk import ( "encoding/json" "errors" "fmt" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" + "github.com/paradeum-team/chainstorage-sdk/model" "net/http" "sync" ) @@ -59,13 +59,13 @@ func (c *CssClient) GetIpfsVersion() (model.VersionResponse, error) { // API调用 httpStatus, body, err := c.httpClient.RestyGet(apiUrl) if err != nil { - c.Logger.logger.Errorf(fmt.Sprintf("API:GetObjectByName:HttpGet, apiUrl:%s, httpStatus:%d, err:%+v\n", apiUrl, httpStatus, err)) + c.Logger.logger.Errorf(fmt.Sprintf("API:GetIpfsVersion:HttpGet, apiUrl:%s, httpStatus:%d, err:%+v\n", apiUrl, httpStatus, err)) return response, err } if httpStatus != http.StatusOK { - c.Logger.logger.Errorf(fmt.Sprintf("API:GetObjectByName:HttpGet, apiUrl:%s, httpStatus:%d, body:%s\n", apiUrl, httpStatus, string(body))) + c.Logger.logger.Errorf(fmt.Sprintf("API:GetIpfsVersion:HttpGet, apiUrl:%s, httpStatus:%d, body:%s\n", apiUrl, httpStatus, string(body))) return response, errors.New(string(body)) } @@ -73,7 +73,40 @@ func (c *CssClient) GetIpfsVersion() (model.VersionResponse, error) { // 响应数据解析 err = json.Unmarshal(body, &response) if err != nil { - c.Logger.logger.Errorf(fmt.Sprintf("API:GetObjectByName:JsonUnmarshal, body:%s, err:%+v\n", string(body), err)) + c.Logger.logger.Errorf(fmt.Sprintf("API:GetIpfsVersion:JsonUnmarshal, body:%s, err:%+v\n", string(body), err)) + + return response, err + } + + return response, nil +} + +func (c *CssClient) GetApiVersion() (model.VersionResponse, error) { + response := model.VersionResponse{} + + // 请求Url + apiBaseAddress := c.Config.ChainStorageApiEndpoint + apiPath := "version" + apiUrl := fmt.Sprintf("%s%s", apiBaseAddress, apiPath) + + // API调用 + httpStatus, body, err := c.httpClient.RestyGet(apiUrl) + if err != nil { + c.Logger.logger.Errorf(fmt.Sprintf("API:GetApiVersion:HttpGet, apiUrl:%s, httpStatus:%d, err:%+v\n", apiUrl, httpStatus, err)) + + return response, err + } + + if httpStatus != http.StatusOK { + c.Logger.logger.Errorf(fmt.Sprintf("API:GetApiVersion:HttpGet, apiUrl:%s, httpStatus:%d, body:%s\n", apiUrl, httpStatus, string(body))) + + return response, errors.New(string(body)) + } + + // 响应数据解析 + err = json.Unmarshal(body, &response) + if err != nil { + c.Logger.logger.Errorf(fmt.Sprintf("API:GetApiVersion:JsonUnmarshal, body:%s, err:%+v\n", string(body), err)) return response, err } diff --git a/cmd/bucket.go b/cmd/bucket.go index 94cad55..93ef39f 100644 --- a/cmd/bucket.go +++ b/cmd/bucket.go @@ -1,10 +1,10 @@ package main import ( - chainstoragesdk "github.com/paradeum-team/chainstorage-sdk/sdk" - sdkcode "github.com/paradeum-team/chainstorage-sdk/sdk/code" - "github.com/paradeum-team/chainstorage-sdk/sdk/consts" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" + sdkcode "github.com/paradeum-team/chainstorage-sdk/code" + "github.com/paradeum-team/chainstorage-sdk/consts" + "github.com/paradeum-team/chainstorage-sdk/model" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/ulule/deepcopier" diff --git a/cmd/car.go b/cmd/car.go index 4b6ff24..dd5968f 100644 --- a/cmd/car.go +++ b/cmd/car.go @@ -2,11 +2,11 @@ package main import ( "fmt" - chainstoragesdk "github.com/paradeum-team/chainstorage-sdk/sdk" - "github.com/paradeum-team/chainstorage-sdk/sdk/code" - "github.com/paradeum-team/chainstorage-sdk/sdk/consts" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" - "github.com/paradeum-team/chainstorage-sdk/sdk/utils" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" + "github.com/paradeum-team/chainstorage-sdk/code" + "github.com/paradeum-team/chainstorage-sdk/consts" + "github.com/paradeum-team/chainstorage-sdk/model" + "github.com/paradeum-team/chainstorage-sdk/utils" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/ulule/deepcopier" diff --git a/cmd/chainstorage-sdk.yaml b/cmd/chainstorage-sdk.yaml index ae4f28b..73f7132 100755 --- a/cmd/chainstorage-sdk.yaml +++ b/cmd/chainstorage-sdk.yaml @@ -10,7 +10,7 @@ server: #HTTP request user agent (K2请求需要) httpRequestUserAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36" #HTTP request user agent (K2请求需要) - httpRequestOvertime: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36" + httpRequestOvertime: 30 logger: logPath: ./logs diff --git a/cmd/go.mod b/cmd/go.mod index 5fe4132..c01d492 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -27,7 +27,7 @@ require ( golang.org/x/term v0.5.0 // indirect ) -require github.com/paradeum-team/chainstorage-sdk/sdk v0.0.0-00010101000000-000000000000 +require github.com/paradeum-team/chainstorage-sdk v0.0.5 require ( github.com/alanshaw/go-carbites v0.6.0 // indirect @@ -102,5 +102,5 @@ require ( replace ( github.com/go-resty/resty => gopkg.in/resty.v1 v1.11.0 - github.com/paradeum-team/chainstorage-sdk/sdk => ./../sdk + github.com/paradeum-team/chainstorage-sdk => ./../ ) diff --git a/cmd/object.go b/cmd/object.go index 6dd7a95..30f5143 100644 --- a/cmd/object.go +++ b/cmd/object.go @@ -4,9 +4,9 @@ import ( "context" "fmt" "github.com/Code-Hex/pget" - chainstoragesdk "github.com/paradeum-team/chainstorage-sdk/sdk" - sdkcode "github.com/paradeum-team/chainstorage-sdk/sdk/code" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" + sdkcode "github.com/paradeum-team/chainstorage-sdk/code" + "github.com/paradeum-team/chainstorage-sdk/model" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/ulule/deepcopier" diff --git a/cmd/root.go b/cmd/root.go index 1355520..9248e74 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,7 +2,7 @@ package main import ( "fmt" - chainstoragesdk "github.com/paradeum-team/chainstorage-sdk/sdk" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" "github.com/spf13/cobra" "os" ) diff --git a/sdk/code/biz_error.go b/code/biz_error.go similarity index 95% rename from sdk/code/biz_error.go rename to code/biz_error.go index 8fd22fc..002ca8d 100644 --- a/sdk/code/biz_error.go +++ b/code/biz_error.go @@ -2,7 +2,7 @@ package code import ( "fmt" - "github.com/paradeum-team/chainstorage-sdk/sdk/utils" + "github.com/paradeum-team/chainstorage-sdk/utils" "strings" ) diff --git a/sdk/code/error.go b/code/error.go similarity index 100% rename from sdk/code/error.go rename to code/error.go diff --git a/sdk/config.go b/config.go similarity index 70% rename from sdk/config.go rename to config.go index 1b393e3..1bffda1 100644 --- a/sdk/config.go +++ b/config.go @@ -1,4 +1,4 @@ -package chainstoragesdk +package sdk import ( "fmt" @@ -32,7 +32,7 @@ type Configuration struct { CarFileWorkPath string `profile:"carFileWorkPath" profileDefault:"./tmp/carfile" json:"carFileWorkPath"` // CAR文件分片阈值 - CarFileShardingThreshold int `profile:"carFileShardingThreshold" profileDefault:"10485760" json:"carFileShardingThreshold"` + CarFileShardingThreshold int `profile:"carFileShardingThreshold" profileDefault:"46137344" json:"carFileShardingThreshold"` // 链存服务API token ChainStorageApiToken string `profile:"chainStorageApiToken" profileDefault:"" json:"chainStorageApiToken"` @@ -45,6 +45,8 @@ type Configuration struct { // CAR version CarVersion int `profile:"carVersion" profileDefault:"1" json:"carVersion"` + + UseHTTPSProtocol bool `profile:"useHttpsProtocol" profileDefault:"true" json:"useHttpsProtocol"` } type LoggerConf struct { @@ -68,9 +70,17 @@ func initConfig(config *ApplicationConfig) { //check chain-storage-api base address if len(cssConfig.ChainStorageApiEndpoint) > 0 { chainStorageAPIEndpoint := cssConfig.ChainStorageApiEndpoint - if !strings.HasPrefix(chainStorageAPIEndpoint, "http") { - fmt.Println("ERROR: invalid chain-storage-api endpoint in Configuration, chain-storage-api endpoint must be a valid http/https url, exiting") - os.Exit(1) + if !strings.HasPrefix(chainStorageAPIEndpoint, "http://") && + !strings.HasPrefix(chainStorageAPIEndpoint, "https://") { + + if cssConfig.UseHTTPSProtocol { + cssConfig.ChainStorageApiEndpoint = "https://" + chainStorageAPIEndpoint + } else { + cssConfig.ChainStorageApiEndpoint = "http://" + chainStorageAPIEndpoint + } + + //fmt.Println("ERROR: invalid chain-storage-api endpoint in Configuration, chain-storage-api endpoint must be a valid http/https url, exiting") + //os.Exit(1) } if !strings.HasSuffix(chainStorageAPIEndpoint, "/") { @@ -88,11 +98,13 @@ func initConfig(config *ApplicationConfig) { cssConfig.ChainStorageApiToken = "Bearer " + cssConfig.ChainStorageApiToken } - // CAR文件分片阈值，缺省10MB - carFileShardingThreshold := cssConfig.CarFileShardingThreshold - if carFileShardingThreshold <= 0 { - cssConfig.CarFileShardingThreshold = 10485760 - } + //// CAR文件分片阈值，缺省10MB + //carFileShardingThreshold := cssConfig.CarFileShardingThreshold + //if carFileShardingThreshold <= 0 { + // cssConfig.CarFileShardingThreshold = 10485760 + //} + // CAR文件分片阈值（固定44Mb） + cssConfig.CarFileShardingThreshold = 46137344 // CAR文件工作目录 carFileWorkPath := cssConfig.CarFileWorkPath @@ -146,16 +158,24 @@ func initConfigWithConfigFile(configFile string) { //check chain-storage-api base address if len(cssConfig.ChainStorageApiEndpoint) > 0 { chainStorageAPIEndpoint := cssConfig.ChainStorageApiEndpoint - if !strings.HasPrefix(chainStorageAPIEndpoint, "http") { - fmt.Println("ERROR: invalid chain-storage-api base address in Configuration file, chain-storage-api base address must be a valid http/https url, exiting") - os.Exit(1) + if !strings.HasPrefix(chainStorageAPIEndpoint, "http://") && + !strings.HasPrefix(chainStorageAPIEndpoint, "https://") { + + if cssConfig.UseHTTPSProtocol { + cssConfig.ChainStorageApiEndpoint = "https://" + chainStorageAPIEndpoint + } else { + cssConfig.ChainStorageApiEndpoint = "http://" + chainStorageAPIEndpoint + } + + //fmt.Println("ERROR: invalid chain-storage-api endpoint in Configuration, chain-storage-api endpoint must be a valid http/https url, exiting") + //os.Exit(1) } if !strings.HasSuffix(chainStorageAPIEndpoint, "/") { cssConfig.ChainStorageApiEndpoint += "/" } } else { - fmt.Println("ERROR: no chain-storage-api base address provided in Configuration file, at least 1 valid http/https chain-storage-api base address must be given, exiting") + fmt.Println("ERROR: no chain-storage-api endpoint provided in Configuration, at least 1 valid http/https chain-storage-api endpoint must be given, exiting") os.Exit(1) } @@ -199,16 +219,31 @@ func InitConfigWithDefault() { //check chain-storage-api base address if len(cssConfig.ChainStorageApiEndpoint) > 0 { chainStorageAPIEndpoint := cssConfig.ChainStorageApiEndpoint - if !strings.HasPrefix(chainStorageAPIEndpoint, "http") { - fmt.Println("ERROR: invalid chain-storage-api base address in Configuration file, chain-storage-api base address must be a valid http/https url, exiting") - os.Exit(1) + if !strings.HasPrefix(chainStorageAPIEndpoint, "http://") && + !strings.HasPrefix(chainStorageAPIEndpoint, "https://") { + + if cssConfig.UseHTTPSProtocol { + cssConfig.ChainStorageApiEndpoint = "https://" + chainStorageAPIEndpoint + } else { + cssConfig.ChainStorageApiEndpoint = "http://" + chainStorageAPIEndpoint + } + + //fmt.Println("ERROR: invalid chain-storage-api endpoint in Configuration, chain-storage-api endpoint must be a valid http/https url, exiting") + //os.Exit(1) } if !strings.HasSuffix(chainStorageAPIEndpoint, "/") { cssConfig.ChainStorageApiEndpoint += "/" } } else { - fmt.Println("ERROR: no chain-storage-api base address provided in Configuration file, at least 1 valid http/https chain-storage-api base address must be given, exiting") + fmt.Println("ERROR: no chain-storage-api endpoint provided in Configuration, at least 1 valid http/https chain-storage-api endpoint must be given, exiting") os.Exit(1) } + + if len(cssConfig.ChainStorageApiToken) == 0 { + fmt.Println("ERROR: invalid chain-storage-api token in Configuration file, chain-storage-api token must not be empty") + os.Exit(1) + } else if !strings.HasPrefix(cssConfig.ChainStorageApiToken, "Bearer ") { + cssConfig.ChainStorageApiToken = "Bearer " + cssConfig.ChainStorageApiToken + } } diff --git a/sdk/consts/global.go b/consts/global.go similarity index 99% rename from sdk/consts/global.go rename to consts/global.go index db4f61c..29c241f 100644 --- a/sdk/consts/global.go +++ b/consts/global.go @@ -608,6 +608,12 @@ var BucketPrincipleCodeMapping = map[int]string{ BucketPrincipleCodePublic: "公开", } +// 桶策略编码与名称映射关系（英文） +var BucketPrincipleCodeMappingEn = map[int]string{ + BucketPrincipleCodePrivate: "private", + BucketPrincipleCodePublic: "public", +} + //var FileExtensionLinkedStorageObjectMapping = map[string]models.ObjectDocumentMapping{ // FileExtensionEmpty: {Id: 10000, FileExtension: "", DocumentType: "", MimeType: "", LinkedStorageType: "", LinkedStorageTypeName: "其他"}, // FileExtensionDir: {Id: 20000, FileExtension: "", DocumentType: "", MimeType: "", LinkedStorageType: "Directory", LinkedStorageTypeName: "目录"}, diff --git a/sdk/go.mod b/go.mod similarity index 98% rename from sdk/go.mod rename to go.mod index 0f49e59..46539e6 100644 --- a/sdk/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/paradeum-team/chainstorage-sdk/sdk +module github.com/paradeum-team/chainstorage-sdk go 1.20 diff --git a/sdk/go.sum b/go.sum similarity index 100% rename from sdk/go.sum rename to go.sum diff --git a/sdk/model/api_key.go b/model/api_key.go similarity index 100% rename from sdk/model/api_key.go rename to model/api_key.go diff --git a/sdk/model/bucket.go b/model/bucket.go similarity index 100% rename from sdk/model/bucket.go rename to model/bucket.go diff --git a/sdk/model/car.go b/model/car.go similarity index 100% rename from sdk/model/car.go rename to model/car.go diff --git a/sdk/model/car_file_upload.go b/model/car_file_upload.go similarity index 100% rename from sdk/model/car_file_upload.go rename to model/car_file_upload.go diff --git a/sdk/model/common.go b/model/common.go similarity index 100% rename from sdk/model/common.go rename to model/common.go diff --git a/sdk/model/linked_storage_api_response.go b/model/linked_storage_api_response.go similarity index 100% rename from sdk/model/linked_storage_api_response.go rename to model/linked_storage_api_response.go diff --git a/sdk/model/object.go b/model/object.go similarity index 100% rename from sdk/model/object.go rename to model/object.go diff --git a/sdk/model/response.go b/model/response.go similarity index 100% rename from sdk/model/response.go rename to model/response.go diff --git a/sdk/object.go b/object.go similarity index 98% rename from sdk/object.go rename to object.go index edab3c1..f62f81a 100644 --- a/sdk/object.go +++ b/object.go @@ -1,4 +1,4 @@ -package chainstoragesdk +package sdk import ( "encoding/json" @@ -6,8 +6,8 @@ import ( "fmt" "github.com/ipfs/go-cid" "github.com/kataras/golog" - "github.com/paradeum-team/chainstorage-sdk/sdk/code" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" + "github.com/paradeum-team/chainstorage-sdk/code" + "github.com/paradeum-team/chainstorage-sdk/model" "net/http" "net/url" "regexp" diff --git a/sdk/pldlogger.go b/pldlogger.go similarity index 96% rename from sdk/pldlogger.go rename to pldlogger.go index a999769..18590c6 100755 --- a/sdk/pldlogger.go +++ b/pldlogger.go @@ -1,10 +1,10 @@ -package chainstoragesdk +package sdk import ( "fmt" "github.com/kataras/golog" - "github.com/lestrrat-go/file-rotatelogs" - "github.com/paradeum-team/chainstorage-sdk/sdk/utils" + rotatelogs "github.com/lestrrat-go/file-rotatelogs" + "github.com/paradeum-team/chainstorage-sdk/utils" "io" "os" "path" diff --git a/sdk/resty_client.go b/resty_client.go similarity index 99% rename from sdk/resty_client.go rename to resty_client.go index bcda71d..5795d8e 100644 --- a/sdk/resty_client.go +++ b/resty_client.go @@ -1,4 +1,4 @@ -package chainstoragesdk +package sdk import ( "bytes" diff --git a/sdk/.gitignore b/sdk/.gitignore deleted file mode 100644 index f29df68..0000000 --- a/sdk/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -../.idea/ -*.iml -*.yaml -*/*.iml -*.ipr -*/*.ipr -*.iws -*/*.iws -*/*.log -*/target/ -*.DS_Store -.DS_Store -/.DS_Store -*.project -*/data/ -*/logs/ -/bin -/temp -/tmp -*.log -VERSION -runtime \ No newline at end of file diff --git a/sdk/utils/sha256.go b/utils/sha256.go similarity index 100% rename from sdk/utils/sha256.go rename to utils/sha256.go diff --git a/sdk/utils/utils.go b/utils/utils.go similarity index 99% rename from sdk/utils/utils.go rename to utils/utils.go index e3f7561..89ddfdd 100644 --- a/sdk/utils/utils.go +++ b/utils/utils.go @@ -2,7 +2,7 @@ package utils import ( "fmt" - "github.com/paradeum-team/chainstorage-sdk/sdk/consts" + "github.com/paradeum-team/chainstorage-sdk/consts" "io" "log" "math/rand"