From 8953d7e1a9049c7556ff13cab53b0fa0c226a644 Mon Sep 17 00:00:00 2001 From: Mohammed Almaroof Date: Mon, 23 May 2022 12:53:11 -0700 Subject: [PATCH 1/2] Adding Meta and GetStats commands support --- memcache/memcache.go | 1058 +++++++++++++++++++++++++++++++++++++ memcache/memcache_test.go | 619 ++++++++++++++++++++++ 2 files changed, 1677 insertions(+) diff --git a/memcache/memcache.go b/memcache/memcache.go index 28eccf03..8177b9b6 100644 --- a/memcache/memcache.go +++ b/memcache/memcache.go @@ -70,10 +70,61 @@ const ( // DefaultMaxIdleConns is the default maximum number of idle connections // kept for any single address. DefaultMaxIdleConns = 2 + + // Meta commands flag constants + base64MetaFlag = "b" + casTokenResponseMetaFlag = "c" + itemValueMetaFlag = "v" + ttlResponseMetaFlag = "t" + updateTTLTokenMetaFlag = "T" + returnKeyAsTokenMetaFlag = "k" + clientFlagsResponseMetaFlag = "f" + itemHitResponseMetaFlag = "h" + lastAccessTimeResponseMetaFlag = "l" + itemSizeResponseMetaFlag = "s" + opaqueTokenMetaFlag = "O" + reCacheWonResponseMetaFlag = "W" + itemStaleResponseMetaFlag = "X" + reCacheWonAlreadySentResponseMetaFlag = "Z" + vivifyOnMissTokenMetaFlag = "N" + reCacheTTLTokenMetaFlag = "R" + dontBumpItemInLruMetaFlag = "u" + setClientFlagsToTokenMetaFlag = "F" + invalidateMetaFlag = "I" + noReplySemanticsMetaFlag = "q" + modeTokenMetaFlag = "M" + compareCasValueTokenMetaFlag = "C" + autoCreateItemOnMissTokenMetaFlag = "N" + initialValueTokenMetaFlag = "J" + deltaTokenMetaFlag = "D" + expirationTimeKey = "exp" + lastAccessTimeKey = "la" + casIdKey = "cas" + fetchKey = "fetch" + slabClassIdKey = "cls" + sizeKey = "size" ) const buffered = 8 // arbitrary buffered channel size, for readability +/* +Enum meant to be used with the mode flag in the meta set command- these are +currently the only supported modes by memcached and it forces the user to adhere +to these modes specifically when using the mode flag. + +Description of what the different modes do can be found in the SetModeToken attribute +of the MetaSetFlag struct. +*/ +type MetaSetMode string + +const ( + Add MetaSetMode = "E" + Append MetaSetMode = "A" + Prepend MetaSetMode = "P" + Replace MetaSetMode = "R" + Set MetaSetMode = "S" +) + // resumableError returns true if err is only a protocol-level cache error. // This is used to determine whether or not a server connection should // be re-used or not. If an error occurs, by default we don't reuse the @@ -112,7 +163,18 @@ var ( resultTouched = []byte("TOUCHED\r\n") resultClientErrorPrefix = []byte("CLIENT_ERROR ") + resultServerErrorPrefix = []byte("SERVER_ERROR ") + resultErrorPrefix = []byte("ERROR") versionPrefix = []byte("VERSION") + + metaValue = []byte("VA") + metaCacheMiss = []byte("EN") + metaResultStored = []byte("HD") + metaResultNotStored = []byte("NS") + metaResultExists = []byte("EX") + metaResultNotFound = []byte("NF") + metaResultDeleted = []byte("HD") + metaDebugSuccess = []byte("ME") ) // New returns a memcache client using the provided server(s) @@ -171,6 +233,187 @@ type Item struct { casid uint64 } +//MetaArithmeticMode flags mention the mode of meta arithmetic operation +//see ArithmeticModeToken in MetaArithmeticFlags for usage +type MetaArithmeticMode string + +const ( + MetaArithmeticIncrement MetaArithmeticMode = "I" + MetaArithmeticDecrement MetaArithmeticMode = "D" +) + +//Flags for MetaArithmetic command +type MetaArithmeticFlags struct { + + //Should be true if the key passed to MetaArithmetic is in base64Format. + //equivalent to the b flag + IsKeyBase64 bool + + // use if only want to store a value if the supplied token matches the current CAS token of the item + // equivalent to the C flag + CompareCasTokenToUpdateValue *uint64 + + // Expiration is the cache expiration time, in seconds: either a relative + // time from now (up to 1 month), or an absolute Unix epoch time. + // Zero means the Item has no expiration time. + //equivalent to the N flag + AutoCreateItemOnMissTTLToken *int32 + + //An unsigned 64-bit integer which will be seeded as the value on a miss. Must be + //combined with an AutoCreateItemOnMissTTLToken. If this value is not set then the default value is 0 + //equivalent to the J flag + AutoCreateInitialValueOnMissToken *uint64 + + //An integer to either add or subtract from the currently stored + //number. If this is not set then the default delta is 1 + //equivalent to the D flag + DeltaToken *uint64 + + //updates the remaining TTL of an item if hit. + //equivalent to the T flag + UpdateTTLToken *int32 + + // use if you wish to reduce the amount of data being sent back by memcached + // equivalent to the q flag + // note: this will always return an error for commands using this flag + UseNoReplySemanticsForResponse bool + + //Set this to true if remaining ttl of the item should be included in meta response metadata + //equivalent to the t flag + ReturnTTLRemainingSecondsInResponse bool + + //Set this to true if cas token should be included in meta response metadata + //equivalent to the c flag + ReturnCasTokenInResponse bool + + //Return new value + //equivalent to the v flag + ReturnItemValueInResponse bool + + //Use Increment to increment the value + //use Decrement to decrement the value + //if this is not set then the default mode is Increment + //equivalent to the M flag + ArithmeticModeToken *MetaArithmeticMode +} + +// MetaGetFlags are the flags supported by MetaGet function +type MetaGetFlags struct { + + // Should be true if the key passed to MetaGet is in base64Format. + // Same as Meta Get b option + IsKeyBase64 bool + + // Set this to true if cas token should be included in meta get response metadata + // Same as Meta Get c option + ReturnCasTokenInResponse bool + + // Set this to true if client flags should be included in meta get response metadata + // Same as Meta Get f option + ReturnClientFlagsInResponse bool + + // Set this to true if you want to know if an item is hit before + // Same as Meta Get h option + ReturnItemHitInResponse bool + + // Set this to true if key should be included in meta get response metadata + // Same as Meta Get k option + ReturnKeyInResponse bool + + // Set this to true if last time since item was accessed in Seconds + // be included in meta get response metadata + // Same as Meta Get l option + ReturnLastAccessedTimeSecondsInResponse bool + + // Set this to true if item size in bytes should be included in meta get response metadata + // Same as Meta Get s option + ReturnItemSizeBytesInResponse bool + + // Set this to true if remaining ttl of the item should be included in meta get response metadata + // Same as Meta Get t option + ReturnTTLRemainingSecondsInResponse bool + + //Set this to true to access an item without causing it to be "bumped" to the head + //of the LRU. This also avoids marking an item as being hit or updating its last + //access time. + //Same as Meta Get u option + PreventBumpInLRU bool + + // opaque value, consumes a token and copies back with response + // Same as Meta Get O option + OpaqueToken *string + + //If supplied, and meta get does not find the item in cache, it will + //create a stub item with the key and TTL as supplied. + //If such an item is created a IsReCacheWon in MetaResponseMetadata will be set to true + //to indicate to a client that they have "won" the right to re cache an item. + //The automatically created item has 0 bytes of data. + //Further, requests will see a IsReCacheWonFlagAlreadySent set to true in MetaResponseMetadata + //to indicate that another client has already received the win flag. + //Same as Meta Get N option + VivifyTTLToken *int32 + + //If the remaining TTL of an item is + //below the supplied token, IsReCacheWon in MetaResponseMetadata will be set to true + //to indicate the client has "won" the right to re cache an item. This allows refreshing an item before it leads to + //a miss. + //Same as Meta Get R option + ReCacheTTLToken *int32 + + // updates the remaining TTL of an item if hit. + // Same as Meta Get T option + UpdateTTLToken *int32 + + //Return value + //Same as Meta Get v flag + ReturnItemValueInResponse bool +} + +//MetaResponseMetadata contains response value and metadata from meta commands +//Some of these values can be nil if the corresponding meta flag is not set while calling +//meta functions +type MetaResponseMetadata struct { + + // Cache value. This will be always present for Meta get command + ReturnItemValue []byte + // Compare and set token + CasId *uint64 + // TTL remaining in seconds for the value. -1 means unlimited + TTLRemainingInSeconds *int32 + // Cache key + ItemKey *string + // Client flags + ClientFlag *uint32 + //Will be true if an item has been hit before + IsItemHitBefore *bool + //Cache value size in bytes + ItemSizeInBytes *uint64 + // Time since item was last accessed in seconds + TimeInSecondsSinceLastAccessed *uint32 + // Opaque token + OpaqueToken *string + //Will be true if client has won the re cache + //same as W flag from mem cache + IsReCacheWon bool + //if this is true then it indicates that a different client + //is responsible for re caching this item + //same as Z flag from memcache + IsReCacheWonFlagAlreadySent bool + //Will be true if the item is stale + //same as X flag from memcache + IsItemStale bool +} + +type MetaDebugResponse struct { + ItemKey *string + ExpirationTime *int32 + LastAccessTime *uint32 + CasId *uint64 + Fetched bool + SlabClassId *uint64 + Size *uint64 +} + // conn is a connection to a server. type conn struct { nc net.Conn @@ -485,6 +728,412 @@ func (c *Client) GetMulti(keys []string) (map[string]*Item, error) { return m, err } +//MetaArithmetic Function supports meta arithmetic operation +//Takes key and MetaArithmeticFlags as a param +//Returns MetaResponseMetadata on success +//Errors: +//Returns ErrMalformedKey error if the key is malformed +//Returns ErrCacheMiss error if the key is not found +//Returns ErrNotStored error to indicate that the item was not created as requested after a miss. +//Returns ErrCASConflict error to indicate that the supplied CAS token does not match the stored item. +func (c *Client) MetaArithmetic(key string, metaArithmeticFlags *MetaArithmeticFlags) (metaResponseMetadata *MetaResponseMetadata, err error) { + err = c.withKeyAddr(key, func(addr net.Addr) error { + return c.metaArithmeticFromAddr(addr, + key, + metaArithmeticFlags, + func(metaArithmeticRes *MetaResponseMetadata) { metaResponseMetadata = metaArithmeticRes }) + }) + return +} + +func (c *Client) metaArithmeticFromAddr(addr net.Addr, + key string, metaArithmeticFlags *MetaArithmeticFlags, + cb func(response *MetaResponseMetadata)) error { + + return c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { + metaArithmeticFlags := createMetaArithmeticFlagCommands(metaArithmeticFlags) + if _, err := fmt.Fprintf(rw, "ma %s %s\r\n", key, metaArithmeticFlags); err != nil { + return err + } + if err := rw.Flush(); err != nil { + return err + } + + if err := parseMetaArithmeticResponse(rw.Reader, cb); err != nil { + return err + } + return nil + + }) + +} + +func parseMetaArithmeticResponse(r *bufio.Reader, cb func(metadata *MetaResponseMetadata)) error { + response, err := r.ReadSlice('\n') + + if err != nil { + return err + } + + switch { + case bytes.HasPrefix(response, metaResultStored) || bytes.HasPrefix(response, metaValue): + + responseComponents := strings.Fields(string(response)) + metaRespMetadata := new(MetaResponseMetadata) + var responseMetadata []string + + if bytes.HasPrefix(response, metaValue) { + var size int + //read value size + if size, err = strconv.Atoi(responseComponents[1]); err != nil { + return err + } + + itemValue := make([]byte, size+2) + //populate value + _, err := io.ReadFull(r, itemValue) + + if err != nil { + return err + } + + //check if value has a suffix of clrf (i.e. \r\n) + if !bytes.HasSuffix(itemValue, crlf) { + return fmt.Errorf("memcache: corrupt meta arithmetic result read") + } + + metaRespMetadata.ReturnItemValue = itemValue[:size] + responseMetadata = responseComponents[2:] + } else { + responseMetadata = responseComponents[1:] + } + + if err = parseMetaResponseMetadata(responseMetadata, metaRespMetadata); err != nil { + return err + } + cb(metaRespMetadata) + + return nil + + case bytes.HasPrefix(response, metaResultNotFound): + return ErrCacheMiss + case bytes.HasPrefix(response, metaResultNotStored): + return ErrNotStored + case bytes.HasPrefix(response, metaResultExists): + return ErrCASConflict + + default: + return fmt.Errorf("memcache: unexpected response line from ms: %q", string(response)) + } +} + +//MetaGet function returns value for a cached item given a key. It also accepts few MetaGetFlags +//which supports additional functionality. +//ErrCacheMiss is returned if a key is not found +//The key must be at most 250 bytes in length. +//Will return error if metaGetFlags are nil or empty struct +func (c *Client) MetaGet(key string, metaGetFlags *MetaGetFlags) (metaResponseMetadata *MetaResponseMetadata, err error) { + err = c.withKeyAddr(key, func(addr net.Addr) error { + return c.metaGetFromAddr(addr, + key, + metaGetFlags, + func(metaGetRes *MetaResponseMetadata) { metaResponseMetadata = metaGetRes }) + }) + return +} + +func (c *Client) metaGetFromAddr(addr net.Addr, key string, metaGetFlags *MetaGetFlags, cb func(response *MetaResponseMetadata)) error { + + return c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { + metaGetCommandFlags := createMetaGetFlagCommands(metaGetFlags) + if _, err := fmt.Fprintf(rw, "mg %s %s\r\n", key, metaGetCommandFlags); err != nil { + return err + } + if err := rw.Flush(); err != nil { + return err + } + + if err := parseMetaGetResponse(rw.Reader, cb); err != nil { + return err + } + return nil + + }) + +} + +// function parses meta get response +func parseMetaGetResponse(r *bufio.Reader, cb func(metadata *MetaResponseMetadata)) error { + + response, err := r.ReadSlice('\n') + if err != nil { + return err + } + + switch { + //Check if cache miss has occurred + case bytes.HasPrefix(response, metaCacheMiss): + return ErrCacheMiss + + //metaGet command response will be in the format if value is requested in response + //VA *\r\n + //\r\n + // if value is not requested then it will be like below + //HD *\r\n + case bytes.HasPrefix(response, metaValue) || bytes.HasPrefix(response, metaResultStored): + responseComponents := strings.Fields(string(response)) + metaRespMetadata := new(MetaResponseMetadata) + var responseMetadata []string + + if bytes.HasPrefix(response, metaValue) { + var size int + //read value size + if size, err = strconv.Atoi(responseComponents[1]); err != nil { + return err + } + + itemValue := make([]byte, size+2) + //populate value + _, err := io.ReadFull(r, itemValue) + + if err != nil { + return err + } + + //check if value has a suffix of clrf (i.e. \r\n) + if !bytes.HasSuffix(itemValue, crlf) { + return fmt.Errorf("memcache: corrupt meta arithmetic result read") + } + + metaRespMetadata.ReturnItemValue = itemValue[:size] + responseMetadata = responseComponents[2:] + } else { + responseMetadata = responseComponents[1:] + } + if err = parseMetaResponseMetadata(responseMetadata, metaRespMetadata); err != nil { + return err + } + cb(metaRespMetadata) + + return nil + default: + return fmt.Errorf("memcache: unexpected response line from ms: %q", string(response)) + } +} + +//populates the MetaResponseMetadata based on the response flags +func populateMetaResponseMetadata(metadata string, respMetadata *MetaResponseMetadata) error { + var err error + respFlagKey := metadata[0:1] + respValue := metadata[1:] + switch respFlagKey { + + case casTokenResponseMetaFlag: + var casId uint64 + if casId, err = strconv.ParseUint(respValue, 10, 64); err != nil { + return err + } + respMetadata.CasId = &casId + + case ttlResponseMetaFlag: + var ttl int64 + + if ttl, err = strconv.ParseInt(respValue, 10, 32); err != nil { + return err + } + ttl32 := int32(ttl) + respMetadata.TTLRemainingInSeconds = &ttl32 + + case returnKeyAsTokenMetaFlag: + respMetadata.ItemKey = &respValue + + case clientFlagsResponseMetaFlag: + var flag *uint32 + if flag, err = convertToUInt32(respValue); err != nil { + return err + } + respMetadata.ClientFlag = flag + + case itemHitResponseMetaFlag: + hitValue := respValue != "0" + respMetadata.IsItemHitBefore = &hitValue + + case itemSizeResponseMetaFlag: + var size uint64 + if size, err = strconv.ParseUint(respValue, 10, 64); err != nil { + return err + } + respMetadata.ItemSizeInBytes = &size + + case lastAccessTimeResponseMetaFlag: + var lastAccessTime *uint32 + if lastAccessTime, err = convertToUInt32(respValue); err != nil { + return err + } + respMetadata.TimeInSecondsSinceLastAccessed = lastAccessTime + + case opaqueTokenMetaFlag: + respMetadata.OpaqueToken = &respValue + + case reCacheWonResponseMetaFlag: + respMetadata.IsReCacheWon = true + + case itemStaleResponseMetaFlag: + respMetadata.IsItemStale = true + + case reCacheWonAlreadySentResponseMetaFlag: + respMetadata.IsReCacheWonFlagAlreadySent = true + } + + return nil +} + +func convertToUInt32(num string) (*uint32, error) { + + var err error + var uInt64Num uint64 + var uInt32Num uint32 + //parse uint always returns uint64 even though bit size is 32. + if uInt64Num, err = strconv.ParseUint(num, 10, 32); err != nil { + return nil, err + } + //have to convert uint64 to uint32 + uInt32Num = uint32(uInt64Num) + return &uInt32Num, nil +} + +func createMetaArithmeticFlagCommands(flags *MetaArithmeticFlags) string { + + var metaFlagCommands strings.Builder + + if flags != nil { + if flags.UpdateTTLToken != nil { + updateTTLTokenFlag := updateTTLTokenMetaFlag + strconv.FormatInt(int64(*flags.UpdateTTLToken), 10) + metaFlagCommands.WriteString(updateTTLTokenFlag + " ") + } + + if flags.IsKeyBase64 { + metaFlagCommands.WriteString(base64MetaFlag + " ") + } + + if flags.ReturnCasTokenInResponse { + metaFlagCommands.WriteString(casTokenResponseMetaFlag + " ") + } + + if flags.CompareCasTokenToUpdateValue != nil { + casTokenFlag := compareCasValueTokenMetaFlag + strconv.FormatUint(*flags.CompareCasTokenToUpdateValue, 10) + metaFlagCommands.WriteString(casTokenFlag + " ") + } + + if flags.AutoCreateItemOnMissTTLToken != nil { + autoCreateOnMissTokenFlag := autoCreateItemOnMissTokenMetaFlag + + strconv.FormatInt(int64(*flags.AutoCreateItemOnMissTTLToken), 10) + metaFlagCommands.WriteString(autoCreateOnMissTokenFlag + " ") + } + + if flags.AutoCreateInitialValueOnMissToken != nil { + initialValueTokenFlag := initialValueTokenMetaFlag + + strconv.FormatUint(*flags.AutoCreateInitialValueOnMissToken, 10) + metaFlagCommands.WriteString(initialValueTokenFlag + " ") + } + + if flags.DeltaToken != nil { + deltaTokenFlag := deltaTokenMetaFlag + + strconv.FormatUint(*flags.DeltaToken, 10) + metaFlagCommands.WriteString(deltaTokenFlag + " ") + } + + if flags.ArithmeticModeToken != nil { + modeTokenFlags := modeTokenMetaFlag + string(*flags.ArithmeticModeToken) + metaFlagCommands.WriteString(modeTokenFlags + " ") + } + + if flags.UseNoReplySemanticsForResponse { + metaFlagCommands.WriteString(noReplySemanticsMetaFlag + " ") + } + + if flags.ReturnTTLRemainingSecondsInResponse { + metaFlagCommands.WriteString(ttlResponseMetaFlag + " ") + } + + if flags.ReturnItemValueInResponse { + metaFlagCommands.WriteString(itemValueMetaFlag + " ") + } + + } + + return strings.TrimSuffix(metaFlagCommands.String(), " ") +} + +//creates the necessary meta get flag commands from the MetaGetFlags struct +func createMetaGetFlagCommands(metaFlags *MetaGetFlags) string { + + var metaFlagCommands strings.Builder + if metaFlags != nil { + if metaFlags.ReturnItemValueInResponse { + metaFlagCommands.WriteString(itemValueMetaFlag + " ") + } + + if metaFlags.UpdateTTLToken != nil { + updateTTLTokenFlag := updateTTLTokenMetaFlag + strconv.FormatInt(int64(*metaFlags.UpdateTTLToken), 10) + metaFlagCommands.WriteString(updateTTLTokenFlag + " ") + } + + if metaFlags.IsKeyBase64 { + metaFlagCommands.WriteString(base64MetaFlag + " ") + } + + if metaFlags.ReturnCasTokenInResponse { + metaFlagCommands.WriteString(casTokenResponseMetaFlag + " ") + } + + if metaFlags.ReturnTTLRemainingSecondsInResponse { + metaFlagCommands.WriteString(ttlResponseMetaFlag + " ") + } + + if metaFlags.ReturnKeyInResponse { + metaFlagCommands.WriteString(returnKeyAsTokenMetaFlag + " ") + } + + if metaFlags.ReturnClientFlagsInResponse { + metaFlagCommands.WriteString(clientFlagsResponseMetaFlag + " ") + } + + if metaFlags.ReturnItemHitInResponse { + metaFlagCommands.WriteString(itemHitResponseMetaFlag + " ") + } + + if metaFlags.ReturnItemSizeBytesInResponse { + metaFlagCommands.WriteString(itemSizeResponseMetaFlag + " ") + } + + if metaFlags.ReturnLastAccessedTimeSecondsInResponse { + metaFlagCommands.WriteString(lastAccessTimeResponseMetaFlag + " ") + } + + if metaFlags.OpaqueToken != nil { + opaqueToken := opaqueTokenMetaFlag + *metaFlags.OpaqueToken + metaFlagCommands.WriteString(opaqueToken + " ") + } + + if metaFlags.VivifyTTLToken != nil { + vivifyToken := vivifyOnMissTokenMetaFlag + strconv.FormatInt(int64(*metaFlags.VivifyTTLToken), 10) + metaFlagCommands.WriteString(vivifyToken + " ") + } + + if metaFlags.ReCacheTTLToken != nil { + reCacheTTLToken := reCacheTTLTokenMetaFlag + strconv.FormatInt(int64(*metaFlags.ReCacheTTLToken), 10) + metaFlagCommands.WriteString(reCacheTTLToken + " ") + } + + if metaFlags.PreventBumpInLRU { + metaFlagCommands.WriteString(dontBumpItemInLruMetaFlag + " ") + } + } + + return strings.TrimSuffix(metaFlagCommands.String(), " ") +} + // parseGetResponse reads a GET response from r and calls cb for each // read and allocated Item func parseGetResponse(r *bufio.Reader, cb func(*Item)) error { @@ -711,3 +1360,412 @@ func (c *Client) incrDecr(verb, key string, delta uint64) (uint64, error) { }) return val, err } + +type MetaDebugItem struct { + Key string + IsKeyBase64 bool +} + +type MetaSetItem struct { + Key string + Flags MetaSetFlags + Value []byte +} +type MetaDeleteItem struct { + Key string + Flags MetaDeleteFlags +} + +type MetaSetFlags struct { + // use if the key provided is base 64 encoded + // equivalent to the b flag + IsKeyBase64 bool + + // use if you wish to see the cas token as a part of the response + // equivalent to the c flag + ReturnCasTokenInResponse bool + + // use if you provide a cas token with CompareCasTokenToUpdateValue attribute and it is older than the item's CAS + // note: only functional when combined with the CompareCasTokenToUpdateValue attribute + // equivalent to the I flag + Invalidate bool + + // use if you want to get the key as a part of the response + // equivalent to the k flag + ReturnKeyInResponse bool + + // use if you wish to reduce the amount of data being sent back by memcached + // note: this will always return an error for commands using this flag + // equivalent to the q flag + UseNoReplySemanticsForResponse bool + + // use if you want to switch modes + // E: "add" command. LRU bump and return NS if item exists, else add + // A: "append" command. If item exists, append the new value to its data + // P: "prepend" command. If item exists, prepend the new value to its data + // R: "replace" command. Set only if item exists, replace its value + // S: "set" command. The default mode, added for completeness + // equivalent to the M flag + SetModeToken *MetaSetMode + + // use if only want to store a value if the supplied token matches the current CAS token of the item + // equivalent to the C flag + CompareCasTokenToUpdateValue *uint64 + + // use if you want to set client flags to a token + // equivalent to the F flag + ClientFlagToken *uint32 + + // use if you want to consume a token and copy back with a response + // equivalent to the O flag + OpaqueToken *string + + // use if you want to set the TTL for the item + // equivalent to the T flag + UpdateTTLToken *int32 +} + +type MetaDeleteFlags struct { + // use if the key provided is base 64 encoded + // equivalent to the b flag + IsKeyBase64 bool + + // instead of removing an item, it will give the item a new CAS value and mark it as stale so the next metaget + // will be supplied an 'X' flag to show that the data is stale and needs to be recached + // equivalent to the I flag + Invalidate bool + + // use if you want to get the key as a part of the response + // equivalent to the k flag + ReturnKeyInResponse bool + + // use if you wish to reduce the amount of data being sent back by memcached + // note: this will always return an error for commands using this flag + // equivalent to the q flag + UseNoReplySemanticsForResponse bool + + // use if only want to store a value if the supplied token matches the current CAS token of the item + // equivalent to the C flag + CompareCasTokenToUpdateValue *uint64 + + // use if you want to consume a token and copy back with a response + // equivalent to the O flag + OpaqueToken *string + + // use if you want to set the TTL for the item + // note: only works when paired with the Invalidate attribute + // equivalent to the T flag + UpdateTTLToken *int32 +} + +/* +This function provides functionality in Golang for the meta set command. The meta set command is a more flexible +approach to setting values in memcached- instead of having multiple methods each do different things, you can consolidate +everything you wish to do in one line with the meta command and its array of flags. + +Arguments +@metaItem: encapsulates the key of the item, its new value, and all flags to apply to the meta set command + +Return values +@metaDataResponse: encapsulates all values returned as a result of the flags which return a value applied to the meta set command +@err: error that can be raised as a result of the operation- errors include: error because value wasn't stored, error because of cache miss, +error because the item alerady exists (occurs with certain flags), I/O errors, malformed key error, and generic error for an unknown response +from memcached +*/ +func (c *Client) MetaSet(metaItem *MetaSetItem) (metaDataResponse *MetaResponseMetadata, err error) { + err = c.onMetaItem(metaItem, (*Client).processMetaSet, func(metaData *MetaResponseMetadata) { metaDataResponse = metaData }) + return +} + +/* +This function provides functionality in Golang for the meta delete command. The meta delete command is a more flexible +approach to deleting values in memcached- instead of having multiple methods each do different things, you can consolidate +everything you wish to do in one line with the meta command and its array of flags. + +Arguments +@metaItem: encapsulates the key of the item and all flags to apply to the meta delete command + +Return values +@metaDataResponse: encapsulates all values returned as a result of the flags which return a value applied to the meta delete command +@err: error that can be raised as a result of the operation- errors include: malformed key, error establishing server connection, cache miss, +CAS token conflict, and potential client/server errors +*/ +func (c *Client) MetaDelete(metaItem *MetaDeleteItem) (metaDataResponse *MetaResponseMetadata, err error) { + err = c.withKeyRw(metaItem.Key, func(rw *bufio.ReadWriter) error { + return c.processMetaDelete(rw, c.parseFlagsForMetaDelete(metaItem), func(metaData *MetaResponseMetadata) { metaDataResponse = metaData }) + }) + return +} + +/* +This function provides functionality in Golang for the meta debug command. The meta debug command is a human readable dump of all +available internal metadata of an item, minus its value. + +Arguments +@metaItem: encapsulates the key of the item and support for a base-64 key encoded flag + +Return values +@metaDebugResponse: encapsulates all values returned as a result of the meta debug command +@err: error that can be raised as a result of the operation- errors include: malformed key, error establishing server connection, cache miss, +CAS token conflict, and potential client/server errors +*/ +func (c *Client) MetaDebug(metaItem *MetaDebugItem) (metaDebugResponse *MetaDebugResponse, err error) { + err = c.withKeyRw(metaItem.Key, func(rw *bufio.ReadWriter) error { + return c.processMetaDebug(rw, c.parseMetaDebugItem(metaItem), func(response *MetaDebugResponse) { metaDebugResponse = response }) + }) + return +} + +func (c *Client) parseMetaDebugItem(metaItem *MetaDebugItem) string { + var commandBuilder strings.Builder + commandBuilder.WriteString(fmt.Sprintf("me %s", metaItem.Key)) + if metaItem.IsKeyBase64 { + commandBuilder.WriteString(" b") + } + commandBuilder.WriteString(string(crlf)) + return commandBuilder.String() +} + +func (c *Client) processMetaDebug(rw *bufio.ReadWriter, command string, cb func(*MetaDebugResponse)) error { + response, err := writeReadLine(rw, command) + if err != nil { + return err + } + if bytes.HasPrefix(response, metaDebugSuccess) { + metaResponseMetadataComponents := strings.Fields(string(response))[2:] + metaDebugResponse := new(MetaDebugResponse) + metaDebugResponse.ItemKey = &metaResponseMetadataComponents[1] + if err = c.parseMetaDebugResponseMetadata(metaResponseMetadataComponents, metaDebugResponse); err != nil { + return err + } + cb(metaDebugResponse) + return nil + } else if bytes.HasPrefix(response, metaCacheMiss) { + return ErrCacheMiss + } + return fmt.Errorf("memcache: unexpected response line from me: %q", string(response)) +} + +func (c *Client) parseMetaDebugResponseMetadata(metadata []string, response *MetaDebugResponse) error { + for _, component := range metadata { + if err := c.populateMetaDebugResponse(component, response); err != nil { + return err + } + } + return nil +} + +func (c *Client) populateMetaDebugResponse(component string, response *MetaDebugResponse) error { + // response after me is in format of k=v + keyValueComponents := strings.Split(component, "=") + key := keyValueComponents[0] + value := keyValueComponents[1] + var err error + switch key { + case expirationTimeKey: + var expirationTime64 int64 + if expirationTime64, err = strconv.ParseInt(value, 10, 32); err != nil { + return err + } + timeToSave := int32(expirationTime64) + response.ExpirationTime = &timeToSave + case lastAccessTimeKey: + var lastAccessedTime64 int64 + if lastAccessedTime64, err = strconv.ParseInt(value, 10, 32); err != nil { + return err + } + timeToSave := uint32(lastAccessedTime64) + response.LastAccessTime = &timeToSave + case casIdKey: + var casId uint64 + if casId, err = strconv.ParseUint(value, 10, 64); err != nil { + return err + } + response.CasId = &casId + case fetchKey: + response.Fetched = value == "yes" + case slabClassIdKey: + var slabId uint64 + if slabId, err = strconv.ParseUint(value, 10, 64); err != nil { + return err + } + response.SlabClassId = &slabId + case sizeKey: + var size uint64 + if size, err = strconv.ParseUint(value, 10, 64); err != nil { + return err + } + response.Size = &size + } + return nil +} + +func (c *Client) onMetaItem(item *MetaSetItem, fn func(*Client, *bufio.ReadWriter, *MetaSetItem, func(metaData *MetaResponseMetadata)) error, cb func(metaData *MetaResponseMetadata)) error { + addr, err := c.selector.PickServer(item.Key) + if err != nil { + return err + } + cn, err := c.getConn(addr) + if err != nil { + return err + } + defer cn.condRelease(&err) + if err = fn(c, cn.rw, item, cb); err != nil { + return err + } + return nil +} + +func (c *Client) processMetaSet(rw *bufio.ReadWriter, item *MetaSetItem, cb func(*MetaResponseMetadata)) error { + if !legalKey(item.Key) { + return ErrMalformedKey + } + + command := c.parseFlagsForMetaSet(item) + + var err error + _, err = fmt.Fprintf(rw, command) + + if err != nil { + return nil + } + if _, err = rw.Write(item.Value); err != nil { + return err + } + if _, err := rw.Write(crlf); err != nil { + return err + } + if err := rw.Flush(); err != nil { + return err + } + response, err := rw.ReadSlice('\n') + if err != nil { + return err + } + switch { + case bytes.HasPrefix(response, metaResultStored): + // the first two characters are being processed in this switch-case block, the other cases are all errors + // so we don't need to save them into the meta data object + responseMetadataComponents := strings.Fields(string(response))[1:] + metaResponseMetadata := new(MetaResponseMetadata) + if err = parseMetaResponseMetadata(responseMetadataComponents, metaResponseMetadata); err != nil { + return err + } + cb(metaResponseMetadata) + return nil + case bytes.HasPrefix(response, metaResultNotStored): + return ErrNotStored + case bytes.HasPrefix(response, metaResultExists): + return ErrCASConflict + case bytes.HasPrefix(response, metaResultNotFound): + return ErrCacheMiss + } + return fmt.Errorf("memcache: unexpected response line from ms: %q", string(response)) +} + +func (c *Client) processMetaDelete(rw *bufio.ReadWriter, command string, cb func(*MetaResponseMetadata)) error { + response, err := writeReadLine(rw, command) + if err != nil { + return err + } + switch { + case bytes.HasPrefix(response, metaResultDeleted): + metaResponseMetadataComponents := strings.Fields(string(response))[1:] + metaResponseMetadata := new(MetaResponseMetadata) + if err = parseMetaResponseMetadata(metaResponseMetadataComponents, metaResponseMetadata); err != nil { + return err + } + cb(metaResponseMetadata) + return nil + case bytes.HasPrefix(response, metaResultNotFound): + return ErrCacheMiss + case bytes.HasPrefix(response, metaResultExists): + return ErrCASConflict + } + return fmt.Errorf("memcache: unexpected response line from md: %q", string(response)) +} + +//function reads the response from meta commands and returns a struct MetaResponseMetadata +//response of meta commands contain flags which are single characters. +// For ex if MetaGetFlags.ReturnTTLRemainingSecondsInResponse is set to true then response +//will contain t300. Here 300 is the amount of TTL remaining in Seconds +func parseMetaResponseMetadata(metaResponseMetadata []string, respMetadata *MetaResponseMetadata) error { + for _, metadata := range metaResponseMetadata { + if err := populateMetaResponseMetadata(metadata, respMetadata); err != nil { + return err + } + } + + return nil +} + +func (c *Client) parseFlagsForMetaSet(metaItem *MetaSetItem) string { + var commandBuilder strings.Builder + commandBuilder.WriteString(fmt.Sprintf("ms %s %d", metaItem.Key, len(metaItem.Value))) + + itemFlags := metaItem.Flags + if itemFlags.IsKeyBase64 { + commandBuilder.WriteString(fmt.Sprintf(" %s", base64MetaFlag)) + } + if itemFlags.ReturnCasTokenInResponse { + commandBuilder.WriteString(fmt.Sprintf(" %s", casTokenResponseMetaFlag)) + } + if itemFlags.Invalidate { + commandBuilder.WriteString(fmt.Sprintf(" %s", invalidateMetaFlag)) + } + if itemFlags.ReturnKeyInResponse { + commandBuilder.WriteString(fmt.Sprintf(" %s", returnKeyAsTokenMetaFlag)) + } + if itemFlags.UseNoReplySemanticsForResponse { + commandBuilder.WriteString(fmt.Sprintf(" %s", noReplySemanticsMetaFlag)) + } + if itemFlags.CompareCasTokenToUpdateValue != nil { + commandBuilder.WriteString(fmt.Sprintf(" %s%d", compareCasValueTokenMetaFlag, *itemFlags.CompareCasTokenToUpdateValue)) + } + if itemFlags.ClientFlagToken != nil { + commandBuilder.WriteString(fmt.Sprintf(" %s%d", setClientFlagsToTokenMetaFlag, *itemFlags.ClientFlagToken)) + } + if itemFlags.OpaqueToken != nil { + commandBuilder.WriteString(fmt.Sprintf(" %s%s", opaqueTokenMetaFlag, *itemFlags.OpaqueToken)) + } + if itemFlags.UpdateTTLToken != nil { + commandBuilder.WriteString(fmt.Sprintf(" %s%d", updateTTLTokenMetaFlag, *itemFlags.UpdateTTLToken)) + } + if itemFlags.SetModeToken != nil { + commandBuilder.WriteString(fmt.Sprintf(" %s%s", modeTokenMetaFlag, *itemFlags.SetModeToken)) + } + commandBuilder.WriteString(string(crlf)) + + return commandBuilder.String() +} + +func (c *Client) parseFlagsForMetaDelete(metaItem *MetaDeleteItem) string { + var commandBuilder strings.Builder + commandBuilder.WriteString(fmt.Sprintf("md %s", metaItem.Key)) + + itemFlags := metaItem.Flags + if itemFlags.IsKeyBase64 { + commandBuilder.WriteString(fmt.Sprintf(" %s", base64MetaFlag)) + } + if itemFlags.Invalidate { + commandBuilder.WriteString(fmt.Sprintf(" %s", invalidateMetaFlag)) + } + if itemFlags.ReturnKeyInResponse { + commandBuilder.WriteString(fmt.Sprintf(" %s", returnKeyAsTokenMetaFlag)) + } + if itemFlags.UseNoReplySemanticsForResponse { + commandBuilder.WriteString(fmt.Sprintf(" %s", noReplySemanticsMetaFlag)) + } + if itemFlags.CompareCasTokenToUpdateValue != nil { + commandBuilder.WriteString(fmt.Sprintf(" %s%d", compareCasValueTokenMetaFlag, *itemFlags.CompareCasTokenToUpdateValue)) + } + if itemFlags.OpaqueToken != nil { + commandBuilder.WriteString(fmt.Sprintf(" %s%s", opaqueTokenMetaFlag, *itemFlags.OpaqueToken)) + } + if itemFlags.UpdateTTLToken != nil { + commandBuilder.WriteString(fmt.Sprintf(" %s%d", updateTTLTokenMetaFlag, *itemFlags.UpdateTTLToken)) + } + commandBuilder.WriteString(string(crlf)) + + return commandBuilder.String() +} diff --git a/memcache/memcache_test.go b/memcache/memcache_test.go index b94a30ca..6476a47d 100644 --- a/memcache/memcache_test.go +++ b/memcache/memcache_test.go @@ -19,6 +19,7 @@ package memcache import ( "bufio" + "bytes" "fmt" "io" "io/ioutil" @@ -212,6 +213,13 @@ func testWithClient(t *testing.T, c *Client) { // Test Ping err = c.Ping() checkErr(err, "error ping: %s", err) + + // test meta commands + testMetaGetCommandsWithClient(t, c, checkErr) + testMetaSetCommandsWithClient(t, c, checkErr) + testMetaDeleteCommandsWithClient(t, c, checkErr) + testMetaArithmeticCommandsWithClient(t, c, checkErr) + testMetaDebugCommandsWithClient(t, c, checkErr) } func testTouchWithClient(t *testing.T, c *Client) { @@ -261,6 +269,617 @@ func testTouchWithClient(t *testing.T, c *Client) { } } +func testMetaArithmeticCommandsWithClient(t *testing.T, c *Client, + checkErr func(err error, format string, args ...interface{})) { + + c.DeleteAll() + defer c.DeleteAll() + + var response *MetaResponseMetadata + + //meta arithmetic test when the key is not found + _, err := c.MetaArithmetic("k1", &MetaArithmeticFlags{}) + if err != ErrCacheMiss { + t.Errorf("metaArithmetic(k1) expected error ErrCacheMiss instead of %v", err) + } + + //meta arithmetic on a non numeric value + stringItem := &Item{Key: "k0", Value: []byte("fooval"), Flags: 123} + err = c.Set(stringItem) + checkErr(err, "first set(stringItem): %v", err) + response, err = c.MetaArithmetic("k0", &MetaArithmeticFlags{}) + if err == nil { + t.Errorf("metaArithmetic(k0) error should not be nil") + } + + //meta arithmetic with non-existing key and AutoCreateItemOnMissTTLToken + var itemTTL int32 = 300 + response, err = c.MetaArithmetic("k1", &MetaArithmeticFlags{AutoCreateItemOnMissTTLToken: &itemTTL}) + checkErr(err, "metaArithmetic(k1): %v", err) + if response == nil { + t.Errorf("metaArithmetic(k1) response should not be nil") + } + + //meta arithmetic with non-existing key , create and return Cas , ttl and value. + response, err = c.MetaArithmetic("k2", &MetaArithmeticFlags{ReturnItemValueInResponse: true, + AutoCreateItemOnMissTTLToken: &itemTTL, ReturnCasTokenInResponse: true, ReturnTTLRemainingSecondsInResponse: true}) + checkErr(err, "metaArithmetic(k2) return value: %v", err) + if response == nil || string(response.ReturnItemValue) != "0" { + t.Errorf("metaArithmetic(k2) Actual Value=%q, Expected Value=0", string(response.ReturnItemValue)) + } + if response.TTLRemainingInSeconds == nil || *response.TTLRemainingInSeconds == 0 { + t.Errorf("metaArithmetic(k2) TTL remaining should not be nil or zero") + } + if response.CasId == nil { + t.Errorf("metaArithmetic(k2) CasId should not be nil") + } + casIdk2 := *response.CasId + + //meta arithmetic with existing key and default increment mode with delta 1 + response, err = c.MetaArithmetic("k2", &MetaArithmeticFlags{CompareCasTokenToUpdateValue: &casIdk2, + ReturnItemValueInResponse: true, ReturnCasTokenInResponse: true}) + checkErr(err, "metaArithmetic(k2) with cas token return value: %v", err) + if response == nil || string(response.ReturnItemValue) != "1" { + t.Errorf("metaArithmetic(k2) with cas token Actual Value=%q, Expected Value=1", string(response.ReturnItemValue)) + } + + //meta arithmetic with invalid cas token + invalidCasToken := (*response.CasId) + 1 + response, err = c.MetaArithmetic("k2", &MetaArithmeticFlags{CompareCasTokenToUpdateValue: &invalidCasToken, + ReturnItemValueInResponse: true}) + if err != ErrCASConflict { + t.Errorf("metaArithmetic(k2) expected error ErrCASConflict instead of %v", err) + } + + //meta arithmetic with increment mode and delta + incrModeToken := MetaArithmeticIncrement + var deltaToken uint64 = 5 + response, err = c.MetaArithmetic("k2", &MetaArithmeticFlags{ArithmeticModeToken: &incrModeToken, + DeltaToken: &deltaToken, ReturnItemValueInResponse: true}) + checkErr(err, "metaArithmetic(k2) with increment and delta value: %v", err) + if response == nil || string(response.ReturnItemValue) != "6" { + t.Errorf("metaArithmetic(k2) with increment and delta Actual Value=%q, Expected Value=6", string(response.ReturnItemValue)) + } + + //meta arithmetic with decrement mode and delta + decrModeToken := MetaArithmeticDecrement + response, err = c.MetaArithmetic("k2", &MetaArithmeticFlags{ArithmeticModeToken: &decrModeToken, + DeltaToken: &deltaToken, ReturnItemValueInResponse: true}) + checkErr(err, "metaArithmetic(k2) with decrement and delta value: %v", err) + if response == nil || string(response.ReturnItemValue) != "1" { + t.Errorf("metaArithmetic(k2) with decrement and delta Actual Value=%q, Expected Value=1", string(response.ReturnItemValue)) + } + + //meta arithmetic mode update TTL token and return ttl + var updateTTLToken int32 = 600 + response, err = c.MetaArithmetic("k2", &MetaArithmeticFlags{UpdateTTLToken: &updateTTLToken, + ReturnTTLRemainingSecondsInResponse: true}) + checkErr(err, "metaArithmetic(k2) with update ttl and fetch ttl: %v", err) + if response == nil || response.TTLRemainingInSeconds == nil || *response.TTLRemainingInSeconds == 0 { + t.Errorf("metaArithmetic(k2) with update ttl and fetch ttl, response should not be nil and " + + "TTLRemainingInSeconds should not be nil or zero") + } + + //meta arithmetic with no reply semantics. Error is always thrown in this case + response, err = c.MetaArithmetic("k2", &MetaArithmeticFlags{UseNoReplySemanticsForResponse: true}) + if err == nil { + t.Errorf("metaArithmetic(k2) with no reply semantics err should not be nil and its expected") + } + + //meta arithmetic with initial value set and base encoded key + //azM=base64Encode(k3) + var initialValue uint64 = 5 + response, err = c.MetaArithmetic("azM=", &MetaArithmeticFlags{AutoCreateItemOnMissTTLToken: &itemTTL, + AutoCreateInitialValueOnMissToken: &initialValue, ReturnItemValueInResponse: true, IsKeyBase64: true}) + checkErr(err, "metaArithmetic(k3) with initial value set and base encoded key: %v", err) + if response == nil || string(response.ReturnItemValue) != "5" { + t.Errorf("metaArithmetic(k3) with initial value set and base encoded key"+ + " Actual Value=%q, Expected Value=5", string(response.ReturnItemValue)) + } + //fetch the item with decoded key. This ensures that IsKeyBase64 option is working correct + var item *Item + item, err = c.Get("k3") + checkErr(err, "get(k3): %v", err) + if item == nil || item.Key != "k3" { + t.Errorf("get(k3) item should not be nil and the key should be k3.Actual key:%q ", item.Key) + } + + //meta arithmetic with malformed key + response, err = c.MetaArithmetic("malformed key", &MetaArithmeticFlags{}) + if err != ErrMalformedKey { + t.Errorf("metaArithmetic(malformed key) expected error ErrMalformedKey instead of %v", err) + } + + //meta arithmetic with AutoCreateInitialValueOnMissToken and no TTL. Cache miss is expected here. + response, err = c.MetaArithmetic("k4", &MetaArithmeticFlags{AutoCreateInitialValueOnMissToken: &initialValue}) + if err != ErrCacheMiss { + t.Errorf("metaArithmetic(k4) expected ErrCacheMiss instead of %v", err) + } + +} + +func testMetaGetCommandsWithClient(t *testing.T, c *Client, + checkErr func(err error, format string, args ...interface{})) { + c.DeleteAll() + defer c.DeleteAll() + + //preparing some test data for test cases + key := &Item{Key: "key", Value: []byte("value")} + err := c.Set(key) + checkErr(err, "first set(key): %v", err) + + key2 := &Item{Key: "key2", Value: []byte("value\r\n"), Flags: 345} + err = c.Set(key2) + checkErr(err, "second set(key): %v", err) + + key3 := &Item{Key: "key3", Value: []byte("value 3")} + err = c.Set(key3) + checkErr(err, "third set(key3): %v", err) + + //simple meta get with nil flags. Should expect error + respMetadata, err := c.MetaGet("key", nil) + if err == nil { + t.Errorf("metaGet(key) error expected as no options are enabled") + } + + respMetadata, err = c.MetaGet("key", &MetaGetFlags{ReturnItemValueInResponse: true}) + checkErr(err, "second metaGet(key): %v", err) + if string(respMetadata.ReturnItemValue) != "value" { + t.Errorf("metaGet(key) Actual Value=%q, Expected Value=value", string(respMetadata.ReturnItemValue)) + } + + //meta get with base64 key , returnItemHitInResponse, LastAccessedTime, ItemSize and flags + respMetadata, err = c.MetaGet("a2V5Mg==", &MetaGetFlags{IsKeyBase64: true, + ReturnItemHitInResponse: true, ReturnLastAccessedTimeSecondsInResponse: true, + ReturnItemSizeBytesInResponse: true, ReturnClientFlagsInResponse: true, + ReturnItemValueInResponse: true, + ReturnKeyInResponse: true}) + checkErr(err, "third metaGet(key2): %v", err) + if string(respMetadata.ReturnItemValue) != "value\r\n" { + t.Errorf("metaGet(key2) Value=%q, Expected Value=value", string(respMetadata.ReturnItemValue)) + } + if respMetadata.IsItemHitBefore == nil || *respMetadata.IsItemHitBefore == true { + t.Errorf("metaGet(key2) IsItemHitBefore should not be nil but should be false") + } + if respMetadata.TimeInSecondsSinceLastAccessed == nil || *respMetadata.TimeInSecondsSinceLastAccessed != 0 { + t.Errorf("metaGet(key2) TimeInSecondsSinceLastAccessed should not be nil but should be 0") + } + if respMetadata.ItemSizeInBytes == nil || *respMetadata.ItemSizeInBytes != 7 { + t.Errorf("metaGet(key2) ItemSizeInBytes should not be nil but should be 7") + } + if respMetadata.ClientFlag == nil || *respMetadata.ClientFlag != 345 { + t.Errorf("metaGet(key2) ClientFlag should not be nil but should be 345") + } + if respMetadata.ItemKey == nil || *respMetadata.ItemKey != "key2" { + t.Errorf("metaGet(key2) ItemKey should not be nil but should be key2") + } + + //sleep so that we can test ReturnLastAccessedTimeInSeconds + time.Sleep(2 * time.Second) + + respMetadata, err = c.MetaGet("key", &MetaGetFlags{ + ReturnItemHitInResponse: true, ReturnLastAccessedTimeSecondsInResponse: true}) + checkErr(err, "metaGet(key3): %v", err) + if respMetadata.IsItemHitBefore == nil || *respMetadata.IsItemHitBefore == false { + t.Errorf("metaGet(key3) IsItemHitBefore should not be nil but should be true") + } + if respMetadata.TimeInSecondsSinceLastAccessed == nil || *respMetadata.TimeInSecondsSinceLastAccessed == 0 { + t.Errorf("metaGet(key2) TimeInSecondsSinceLastAccessed should not be nil but should be non zero") + } + if respMetadata.ReturnItemValue != nil { + t.Errorf("metaGet(key2) ReturnItemValue should be nil") + } + + //meta get cache miss + respMetadata, err = c.MetaGet("key53", &MetaGetFlags{ReturnItemValueInResponse: true}) + if err != ErrCacheMiss { + t.Errorf("metaGet(key53) expected error ErrCacheMiss instead of %v", err) + } + + //meta get with malformed key + respMetadata, err = c.MetaGet("key val", nil) + if err != ErrMalformedKey { + t.Errorf("metaGet(key val) should return ErrMalformedKey instead of %v", err) + } + + //meta get with cas response flag , ttl response flag , key response flag , Opaque token + opaqueToken := "Opaque" + respMetadata, err = c.MetaGet("key", &MetaGetFlags{ReturnCasTokenInResponse: true, + ReturnTTLRemainingSecondsInResponse: true, ReturnKeyInResponse: true, + OpaqueToken: &opaqueToken}) + checkErr(err, "cas,ttl metaGet(key): %v", err) + if respMetadata.CasId == nil { + t.Errorf("metaGet(key) casid should not be nil") + } + if respMetadata.TTLRemainingInSeconds == nil || *respMetadata.TTLRemainingInSeconds != -1 { + t.Errorf("metaGet(key) TTLRemainingInSeconds should not be nil or should be -1 ") + } + if respMetadata.ItemKey == nil || *respMetadata.ItemKey != "key" { + t.Errorf("metaGet(key) ItemKey should not be nil. Should be key") + } + if respMetadata.OpaqueToken == nil || *respMetadata.OpaqueToken != "Opaque" { + t.Errorf("metaGet(key) OpaqueToken should not be nil. Should be Opaque") + } + + //meta get update ttl and fetch ttl + var updateTTlToken int32 = 5000 + respMetadata, err = c.MetaGet("key", &MetaGetFlags{UpdateTTLToken: &updateTTlToken, + ReturnTTLRemainingSecondsInResponse: true}) + checkErr(err, "ttl,update ttl metaGet(key): %v", err) + if respMetadata.TTLRemainingInSeconds == nil || *respMetadata.TTLRemainingInSeconds != 5000 { + t.Errorf("metaGet(key) TTLRemainingInSeconds should not be nil. should be 5000") + } + + //test PreventBumpInLRU flag + key4 := &Item{Key: "key4", Value: []byte("value")} + err = c.Set(key4) + checkErr(err, "set(key4): %v", err) + respMetadata, err = c.MetaGet("key4", &MetaGetFlags{PreventBumpInLRU: true}) + checkErr(err, "metaGet(key4): %v", err) + + respMetadata, err = c.MetaGet("key4", &MetaGetFlags{ReturnLastAccessedTimeSecondsInResponse: true, + ReturnItemHitInResponse: true}) + checkErr(err, "metaGet(key4): %v", err) + if respMetadata.IsItemHitBefore == nil || *respMetadata.IsItemHitBefore == true { + t.Errorf("metaGet(key4) IsItemHitBefore should not be nil but should be false") + } + + //testVivify TTL token + var vivifyTTLToken int32 = 300 + respMetadata, err = c.MetaGet("key5", &MetaGetFlags{VivifyTTLToken: &vivifyTTLToken}) + checkErr(err, "metaGet(key5): %v", err) + if !respMetadata.IsReCacheWon { + t.Errorf("metaGet(key5) IsReCacheWon should be true instead of false") + } + if string(respMetadata.ReturnItemValue) != "" { + t.Errorf("metaGet(key5) value should be empty") + } + + respMetadata, err = c.MetaGet("key5", &MetaGetFlags{VivifyTTLToken: &vivifyTTLToken}) + if respMetadata.IsReCacheWon || !respMetadata.IsReCacheWonFlagAlreadySent { + t.Errorf("metaGet(key5) IsReCacheWonFlagAlreadySent should be true. IsReCacheWon should be false") + } + + //testRecacheTTLToken + key6 := &Item{Key: "key6", Value: []byte("value"), Expiration: 300} + err = c.Set(key6) + checkErr(err, "set(key6): %v", err) + time.Sleep(2 * time.Second) + + var reCacheTTLToken int32 = 300 + respMetadata, err = c.MetaGet("key6", &MetaGetFlags{ReCacheTTLToken: &reCacheTTLToken, + ReturnTTLRemainingSecondsInResponse: true}) + checkErr(err, "metaGet(key6): %v", err) + if !respMetadata.IsReCacheWon { + t.Errorf("metaGet(key6) IsReCacheWon should be true") + } + respMetadata, err = c.MetaGet("key6", &MetaGetFlags{ReturnItemValueInResponse: true}) + checkErr(err, "metaGet(key6): %v", err) + if respMetadata.IsReCacheWon || !respMetadata.IsReCacheWonFlagAlreadySent { + t.Errorf("metaGet(key6) IsReCacheWon should be false and IsReCacheWonFlagAlreadySent should be true") + } +} + +func testMetaSetCommandsWithClient(t *testing.T, c *Client, checkErr func(err error, format string, args ...interface{})) { + key := "bah" + value := []byte("bahval") + opaqueToken := "A123" + metaFoo := &MetaSetItem{Key: key, Value: value, Flags: MetaSetFlags{ReturnKeyInResponse: true, ReturnCasTokenInResponse: true, OpaqueToken: &opaqueToken}} + response, err := c.MetaSet(metaFoo) + if response.ItemKey == nil || *response.ItemKey != key { + t.Errorf("meta set(%s) key should not be nil and should be %s", key, key) + } + if response.OpaqueToken == nil || *response.OpaqueToken != opaqueToken { + t.Errorf("meta set(%s) Opaque token should not be nil and should be %s", key, opaqueToken) + } + casToken := response.CasId + if casToken == nil { + t.Errorf("meta set(%s) error, no CAS token returned", key) + } + checkErr(err, "normal meta set(%s): %v", key, err) + testMetaSetSavedValue(t, c, checkErr, key, value) + + // set using the same cas token as what was last set + value = []byte("new_bah_val") + var newTTL int32 = 900000 + var clientFlagToken uint32 = 90 + metaFoo = &MetaSetItem{Key: key, Value: value, Flags: MetaSetFlags{CompareCasTokenToUpdateValue: casToken, ClientFlagToken: &clientFlagToken, UpdateTTLToken: &newTTL}} + _, err = c.MetaSet(metaFoo) + checkErr(err, "Same CAS token meta set(%s): %v", key, err) + it, err := c.Get(key) + if it.Flags != clientFlagToken { + t.Errorf("Same CAS token meta set(%s) expected client flag %d but got %d", key, clientFlagToken, it.Flags) + } + testMetaSetSavedValue(t, c, checkErr, key, value) + + // set using a different cas token + value = []byte("byte_val_invalid") + var newCasToken uint64 = 123456789 + metaFoo = &MetaSetItem{Key: key, Value: value, Flags: MetaSetFlags{CompareCasTokenToUpdateValue: &newCasToken}} + _, err = c.MetaSet(metaFoo) + if err != ErrCASConflict { + t.Errorf("Different CAS token meta set(%s) expected an CAS conflict error but got %e", key, err) + } + + // set with no reply semantics turned on + // note that the documentation says that this flag will always return an error (even if the command runs successfully) + value = []byte("with_base64_key") + metaFoo = &MetaSetItem{Key: key, Value: value, Flags: MetaSetFlags{UseNoReplySemanticsForResponse: true}} + _, err = c.MetaSet(metaFoo) + // the error raised is an internal error, so we can't change for explicit type + if err == nil { + t.Errorf("no reply meta set(%s) expected an error to be returned but got none", key) + } + + // set using the append mode with existing key + valueToAppend := []byte("append_value_to_existing") + value = append(value, valueToAppend...) + mode := Append + metaFoo = &MetaSetItem{Key: key, Value: valueToAppend, Flags: MetaSetFlags{SetModeToken: &mode}} + _, err = c.MetaSet(metaFoo) + checkErr(err, "successful append meta set(%s): %v", key, err) + testMetaSetSavedValue(t, c, checkErr, key, value) + + // set using the prepend mode + valueToPrepend := []byte("prepend_value_to_existing") + value = append(valueToPrepend, value...) + mode = Prepend + metaFoo = &MetaSetItem{Key: key, Value: valueToPrepend, Flags: MetaSetFlags{SetModeToken: &mode}} + _, err = c.MetaSet(metaFoo) + checkErr(err, "successful prepend meta set(%s): %v", key, err) + testMetaSetSavedValue(t, c, checkErr, key, value) + + // set using the add mode and existing key + // will fail to store and return ErrNotStored error because key is in use + mode = Add + metaFoo = &MetaSetItem{Key: key, Value: []byte("add_value_to_existing"), Flags: MetaSetFlags{SetModeToken: &mode}} + _, err = c.MetaSet(metaFoo) + if err != ErrNotStored { + t.Errorf("add mode with existing key meta set(%s) expected not stored error but got %e", key, err) + } + + // set using the replace mode + value = []byte("replacement_value") + mode = Replace + metaFoo = &MetaSetItem{Key: key, Value: value, Flags: MetaSetFlags{SetModeToken: &mode}} + _, err = c.MetaSet(metaFoo) + checkErr(err, "successful replace mode meta set(%s): %v", key, err) + testMetaSetSavedValue(t, c, checkErr, key, value) + + // set using the add mode + // will store because key is not in use + key = "new_key" + value = []byte("add_value_to_new_key") + mode = Add + metaFoo = &MetaSetItem{Key: key, Value: value, Flags: MetaSetFlags{SetModeToken: &mode}} + _, err = c.MetaSet(metaFoo) + checkErr(err, "add mode without existing key meta set(%s): %v", key, err) + testMetaSetSavedValue(t, c, checkErr, key, value) + + // set using base64 encoded string + key = "bmV3QmFzZUtleQ==" + decodedKey := "newBaseKey" + value = []byte("with_base64_key") + metaFoo = &MetaSetItem{Key: key, Value: value, Flags: MetaSetFlags{IsKeyBase64: true}} + _, err = c.MetaSet(metaFoo) + checkErr(err, "base64 encoded key meta set(%s): %v", key, err) + testMetaSetSavedValue(t, c, checkErr, decodedKey, value) + + // set using the append mode with non-existent key + valueToAppend = []byte("new_append_value") + key = "non_existing_for_append" + mode = Append + metaFoo = &MetaSetItem{Key: key, Value: valueToAppend, Flags: MetaSetFlags{SetModeToken: &mode}} + _, err = c.MetaSet(metaFoo) + if err != ErrNotStored { + t.Errorf("Append with non-existent key meta set(%s) expected a not stored error but got %e", key, err) + } + + // set using the prepend mode with non-existent key + valueToPrepend = []byte("new_prepend_value") + key = "non_existing_for_prepend" + mode = Prepend + metaFoo = &MetaSetItem{Key: key, Value: valueToPrepend, Flags: MetaSetFlags{SetModeToken: &mode}} + _, err = c.MetaSet(metaFoo) + if err != ErrNotStored { + t.Errorf("Prepend with non-existent key meta set(%s) expected a not stored error but got %e", key, err) + } + + // set using the replace mode with non-existent key + value = []byte("new_replace_value") + key = "non_existing_for_replace" + mode = Replace + metaFoo = &MetaSetItem{Key: key, Value: value, Flags: MetaSetFlags{SetModeToken: &mode}} + _, err = c.MetaSet(metaFoo) + if err != ErrNotStored { + t.Errorf("Replace with non-existent key meta set(%s) expected a not stored error but got %e", key, err) + } +} + +func testMetaDeleteCommandsWithClient(t *testing.T, c *Client, checkErr func(err error, format string, args ...interface{})) { + setForDelete := func(key string, value []byte, flags MetaSetFlags) *MetaResponseMetadata { + metaSetItem := &MetaSetItem{Key: key, Value: value, Flags: flags} + metaDataResponse, err := c.MetaSet(metaSetItem) + checkErr(err, "meta set(%s): %v", key, err) + return metaDataResponse + } + + key := "foo_key" + value := []byte("foo_val") + setResponse := setForDelete(key, value, MetaSetFlags{ReturnCasTokenInResponse: true}) + casValue := setResponse.CasId + + // normal delete with key return and opaque token provided on matching CAS token + opaqueToken := "opaque_token" + metaDeleteItem := &MetaDeleteItem{Key: key, Flags: MetaDeleteFlags{ReturnKeyInResponse: true, OpaqueToken: &opaqueToken, CompareCasTokenToUpdateValue: casValue}} + response, err := c.MetaDelete(metaDeleteItem) + if *response.ItemKey != key { + t.Errorf("meta delete(%s) Key = %q, want %s", key, *response.ItemKey, key) + } + if *response.OpaqueToken != opaqueToken { + t.Errorf("meta delete(%s) Opaque token = %s, want %s", key, *response.OpaqueToken, opaqueToken) + } + checkErr(err, "meta delete(%s): %v", key, err) + + // failed delete due to mismatched CAS token + var mismatchCasToken uint64 = 123456 + setResponse = setForDelete(key, value, MetaSetFlags{}) + metaDeleteItem = &MetaDeleteItem{Key: key, Flags: MetaDeleteFlags{ReturnKeyInResponse: true, OpaqueToken: &opaqueToken, CompareCasTokenToUpdateValue: &mismatchCasToken}} + _, err = c.MetaDelete(metaDeleteItem) + if err != ErrCASConflict { + t.Errorf("Different CAS token meta delete(%s) expected an CAS conflict error but got %e", key, err) + } + + // delete with non-existent key + key = "does_not_exist" + metaDeleteItem = &MetaDeleteItem{Key: key, Flags: MetaDeleteFlags{}} + _, err = c.MetaDelete(metaDeleteItem) + if err != ErrCacheMiss { + t.Errorf("Non-existent key meta delete(%s) expected a cache miss error but got %e", key, err) + } + + // delete with invalidation and TTL update- expect to see a new CAS token and TTL set after not being initially set + setResponse = setForDelete(key, value, MetaSetFlags{ReturnCasTokenInResponse: true}) + getResponse, err := c.MetaGet(key, &MetaGetFlags{ReturnTTLRemainingSecondsInResponse: true}) + if getResponse.TTLRemainingInSeconds == nil || *getResponse.TTLRemainingInSeconds == 0 { + t.Errorf("invalidation meta delete(%s) expected TTL to be non-nil and equal to 0", key) + } + checkErr(err, "meta get(%s): %v", key, err) + casValue = setResponse.CasId + var newTTL int32 = 5000 + metaDeleteItem = &MetaDeleteItem{Key: key, Flags: MetaDeleteFlags{Invalidate: true, UpdateTTLToken: &newTTL}} + response, err = c.MetaDelete(metaDeleteItem) + checkErr(err, "meta delete(%s): %v", key, err) + // after invalidation, the cas token should differ, so the CompareAndSwap method should raise an error when using the pre-delete CAS ID + item := &Item{Key: key, Value: []byte("new_val_for_invalidation"), casid: *casValue} + err = c.CompareAndSwap(item) + if err != ErrCASConflict { + t.Errorf("invalidation meta delete(%s) expected CAS error to be returned but got %e", key, err) + } + getResponse, err = c.MetaGet(key, &MetaGetFlags{ReturnTTLRemainingSecondsInResponse: true}) + if getResponse.TTLRemainingInSeconds == nil || *getResponse.TTLRemainingInSeconds == 0 { + t.Errorf("invalidation meta delete(%s) expected TTL to be non-nil and greater than 0", key) + } + if !getResponse.IsItemStale { + t.Errorf("invalidation meta delete(%s) expected item stale status to be true", key) + } + checkErr(err, "meta get(%s): %v", key, err) + + // delete with no-reply semantics + // note that the documentation says that this flag will always return an error (even if the command runs successfully) + setResponse = setForDelete(key, value, MetaSetFlags{}) + metaDeleteItem = &MetaDeleteItem{Key: key, Flags: MetaDeleteFlags{ReturnKeyInResponse: true, UseNoReplySemanticsForResponse: true}} + _, err = c.MetaDelete(metaDeleteItem) + // the error raised is an internal error, so we can't check for explicit type + if err == nil { + t.Errorf("no reply meta delete(%s) expected an error to be returned but got none", key) + } + + // delete with base-64 encoded key where decodedKey = newBaseKey + key = "bmV3QmFzZUtleQ==" + decodedKey := "newBaseKey" + setResponse = setForDelete(key, value, MetaSetFlags{IsKeyBase64: true, ReturnKeyInResponse: true}) + metaDeleteItem = &MetaDeleteItem{Key: key, Flags: MetaDeleteFlags{IsKeyBase64: true, ReturnKeyInResponse: true}} + // check that the item was set with the properly decoded key before deleting + it, err := c.Get(decodedKey) + if it == nil { + t.Errorf("base-64 encoded key meta set(%s) expected did not set", key) + } + checkErr(err, "get(%s): %v", key, err) + // delete item + response, err = c.MetaDelete(metaDeleteItem) + if response.ItemKey == nil || *response.ItemKey != key { + t.Errorf("base-64 encoded key meta delete(%s) expected key to be non-nil and %s", key, key) + } + checkErr(err, "base-64 encoded key meta delete(%s): %v", key, err) + // check that the item is no longer able to be received with the decoded key post delete + it, err = c.Get(decodedKey) + if it != nil { + t.Errorf("base-64 encoded key meta delete(%s) did not delete item", key) + } + if err != ErrCacheMiss { + t.Errorf("base-64 encoded key meta delete(%s) expected cached miss error but got %v", key, err) + } +} + +func testMetaDebugCommandsWithClient(t *testing.T, c *Client, checkErr func(err error, format string, args ...interface{})) { + key := "test_key" + value := []byte("debug_value") + var timeToLive int32 = 5000 + flags := MetaSetFlags{ReturnCasTokenInResponse: true, UpdateTTLToken: &timeToLive} + setResponse := metaSetForSetup(t, c, key, value, flags, checkErr) + + // debug call prior to fetch + var size uint64 + metaDebugItem := &MetaDebugItem{Key: key} + metaDebugResponse, err := c.MetaDebug(metaDebugItem) + if metaDebugResponse.ExpirationTime == nil || *metaDebugResponse.ExpirationTime == -1 { + t.Errorf("meta debug(%s) expected to have non-nil TTL and in the range [-5000, -1)", key) + } + if metaDebugResponse.LastAccessTime == nil { + t.Errorf("meta debug(%s) expected to have non-nil time since last access", key) + } + if metaDebugResponse.CasId == nil || *metaDebugResponse.CasId != *setResponse.CasId { + t.Errorf("meta debug(%s) expected to have non-nil CAS ID and have it equal to what it was when it was set", key) + } + if metaDebugResponse.Fetched { + t.Errorf("meta debug(%s) expected to not have been fetched yet", key) + } + if metaDebugResponse.SlabClassId == nil || *metaDebugResponse.SlabClassId != 1 { + t.Errorf("meta debug(%s) expected to have non-nil Slab Class ID and have it equal 1", key) + } + if metaDebugResponse.Size == nil || *metaDebugResponse.Size == 0 { + t.Errorf("meta debug(%s) expected to have non-nil size and have it be greater than 0", key) + } else { + size = *metaDebugResponse.Size + } + checkErr(err, "meta debug(%s) immediately after set", key) + + // check that Fetched attribute is set to true after get + c.Get(key) + metaDebugResponse, _ = c.MetaDebug(metaDebugItem) + if !metaDebugResponse.Fetched { + t.Errorf("meta debug(%s) expected to have been fetched", key) + } + if metaDebugResponse.Size == nil || *metaDebugResponse.Size != size { + t.Errorf("meta debug(%s) expected to have non-nil size and have it stay the same across calls", key) + } + + // meta debug when value was not set with TTL + metaSetForSetup(t, c, key, value, MetaSetFlags{}, checkErr) + metaDebugItem = &MetaDebugItem{Key: key} + metaDebugResponse, err = c.MetaDebug(metaDebugItem) + if metaDebugResponse.ExpirationTime == nil || *metaDebugResponse.ExpirationTime != -1 { + t.Errorf("meta debug(%s) expected to have non-nil TTL and to be -1", key) + } + + // non-existent key meta debug call + key = "non_existent_key" + metaDebugItem = &MetaDebugItem{Key: key} + _, err = c.MetaDebug(metaDebugItem) + if err != ErrCacheMiss { + t.Errorf("meta debug(%s) with non-existent key expected to have cache miss error but got %v", key, err) + } +} + +func metaSetForSetup(t *testing.T, c *Client, key string, value []byte, flags MetaSetFlags, checkErr func(err error, format string, args ...interface{})) *MetaResponseMetadata { + metaSetItem := &MetaSetItem{Key: key, Value: value, Flags: flags} + response, err := c.MetaSet(metaSetItem) + checkErr(err, "meta set(%s): %v", key, err) + return response +} + +func testMetaSetSavedValue(t *testing.T, c *Client, checkErr func(err error, format string, args ...interface{}), key string, value []byte) { + it, err := c.Get(key) + checkErr(err, "get(%s): %v", key, err) + if it.Key != key { + t.Errorf("get(%s) Key = %q, want %s", key, it.Key, key) + } + if bytes.Compare(it.Value, value) != 0 { + t.Errorf("get(%s) Value = %q, want %q", key, string(it.Value), string(value)) + } +} + func BenchmarkOnItem(b *testing.B) { fakeServer, err := net.Listen("tcp", "localhost:0") if err != nil { From d9486adf45ea2cebd642f8cc86f6e305d236346a Mon Sep 17 00:00:00 2001 From: Mohammed Almaroof Date: Mon, 23 May 2022 13:12:14 -0700 Subject: [PATCH 2/2] Adding get stats support --- memcache/memcache.go | 74 +++++++++++++++++++++++++++++++++++++++ memcache/memcache_test.go | 59 +++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/memcache/memcache.go b/memcache/memcache.go index 8177b9b6..62d628dc 100644 --- a/memcache/memcache.go +++ b/memcache/memcache.go @@ -1769,3 +1769,77 @@ func (c *Client) parseFlagsForMetaDelete(metaItem *MetaDeleteItem) string { return commandBuilder.String() } + +// This function retrieves stats for each connection. The stats command prints statistical information +// for a given stat type + +// Arguments +// @args: the stat command to get statistical information of each connection, this will be passed as-is +// to memcache. Some examples of valid stats commands: "stats", "stats slabs", "stats settings", and "stats items" + +// GetStats Return values +// @addrToStats: contains stats for each connection, +// which for each connection is encapsulated in a string-to-string hashmap +// @err: error that can be raised as a result of the operation +func (c *Client) GetStats(args string) (addrToStats map[string]map[string]string, err error) { + if len(args) <= 0 { + return nil, errors.New("arg string parameter must not be empty.") + } + addrToStats = make(map[string]map[string]string) + err = c.selector.Each(func(addr net.Addr) error { + return c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { + if _, err := fmt.Fprintf(rw, "%s%s", args, string(crlf)); err != nil { + return err + } + if err := rw.Flush(); err != nil { + return err + } + stats, err := parseStatsResponse(rw.Reader) + if err != nil { + return err + } + addrToStats[addr.String()] = stats + return nil + }) + }) + + return addrToStats, err +} + +// parseStatsResponse reads a GetStats response and maps each string property to its string value +func parseStatsResponse(r *bufio.Reader) (stats map[string]string, err error) { + stats = make(map[string]string) + var firstToken string + for { + line, err := r.ReadSlice('\n') + if err != nil { + return nil, err + } + + if len(line) <= 0 { + // nothing to read but no error too, should not throw an error + continue + } + + lineSlices := bytes.Split(line, space) + if len(lineSlices) <= 0 { + continue + } + + firstToken = strings.TrimSpace(string(lineSlices[0])) + if firstToken == "END" { + break + } + + // stat type is not supported, should throw + if firstToken == "ERROR" { + return nil, errors.New("stat type is not supported") + } + + // if this line has a stat name and value + if len(lineSlices) > 2 { + stats[strings.TrimSpace(string(lineSlices[1]))] = strings.TrimSpace(string(lineSlices[2])) + } + } + return stats, nil +} diff --git a/memcache/memcache_test.go b/memcache/memcache_test.go index 6476a47d..0064946f 100644 --- a/memcache/memcache_test.go +++ b/memcache/memcache_test.go @@ -220,6 +220,28 @@ func testWithClient(t *testing.T, c *Client) { testMetaDeleteCommandsWithClient(t, c, checkErr) testMetaArithmeticCommandsWithClient(t, c, checkErr) testMetaDebugCommandsWithClient(t, c, checkErr) + + // test successful GetStats command + testGetStatsCommand(t, c, "stats", false, []string{"pid", "uptime", "version"}) + testGetStatsCommand(t, c, "stats conns", false, []string{"addr", "listen_addr", "state"}) + testGetStatsCommand(t, c, "stats slabs", false, []string{"chunk_size", "total_pages", "used_chunks"}) + testGetStatsCommand(t, c, "stats settings", false, []string{"maxbytes", "maxconns", "tcpport"}) + + // items stats won't return properties if no items are set in memcached + itemsStatsTestVal := &Item{Key: "itemsStatsTest", Value: []byte("itemsStatsTestVal"), Flags: 123} + err = c.Set(itemsStatsTestVal) + checkErr(err, "Setting test value for items stats fetch: %v", err) + testGetStatsCommand(t, c, "stats items", false, []string{"number", "age", "age_warm"}) + err = c.Delete(itemsStatsTestVal.Key) + checkErr(err, "Deleting items state fetch test item: %v", err) + + // stats sizes is disabled by default + testGetStatsCommand(t, c, "stats sizes", false, []string{}) + + // test failed GetStats command + testGetStatsCommand(t, c, "stat", true, []string{}) + testGetStatsCommand(t, c, "stats typo", true, []string{}) + testGetStatsCommand(t, c, "STATS FAKE", true, []string{}) } func testTouchWithClient(t *testing.T, c *Client) { @@ -909,3 +931,40 @@ func BenchmarkOnItem(b *testing.B) { c.onItem(&item, dummyFn) } } + +func testGetStatsCommand(t *testing.T, c *Client, args string, expectedFailure bool, expectedProperties []string) { + stats, err := c.GetStats(args) + if err != nil || len(stats) < 1 { + if expectedFailure { + return + } + t.Errorf("Could not call GetStats for %s successfully", args) + } + c.selector.Each(func(addr net.Addr) error { + if stats[addr.String()] == nil || len(stats[addr.String()]) < 1 { + t.Errorf("Empty or nil stats for %s address and %s stat", addr.String(), args) + } + c.selector.Each(func(addr net.Addr) error { + currentUnmatchedProperties := expectedProperties + for statKey, _ := range stats[addr.String()] { + // separate stat key by : delim + statKeySlices := strings.Split(statKey, ":") + // the stat key name should always be after the last ":" + parsedStatKey := statKeySlices[len(statKeySlices)-1] + for i, property := range currentUnmatchedProperties { + if parsedStatKey == property { + currentUnmatchedProperties = append(currentUnmatchedProperties[:i], currentUnmatchedProperties[i+1:]...) + } + } + if len(currentUnmatchedProperties) == 0 { + break + } + } + if len(currentUnmatchedProperties) > 0 { + t.Errorf("Stats map does not contain %s property in %s address and %s stat, this may be caused by a stats fetch failure, or an update to the memcached stats schema.", currentUnmatchedProperties, addr, args) + } + return nil + }) + return nil + }) +}