From edf9f63a1726abb9ff01b620f6b27cd950c9f09d Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 22 Apr 2017 22:40:31 +0800 Subject: [PATCH] add project of libfc and a simple test for libfc libfc is non-thread-safe lib --- configure.ac | 1 + src/Makefile.am | 20 ++++- src/fc.h | 43 ++++++++++ src/fc_settings.h | 4 +- src/fc_test.c | 205 ++++++++++++++++++++++++++++++++++++++++++++++ src/libfc.c | 105 ++++++++++++++++++++++++ 6 files changed, 374 insertions(+), 4 deletions(-) create mode 100644 src/fc.h create mode 100644 src/fc_test.c create mode 100644 src/libfc.c diff --git a/configure.ac b/configure.ac index 065c6f8..5bfff54 100644 --- a/configure.ac +++ b/configure.ac @@ -30,6 +30,7 @@ AC_PROG_CC AC_PROG_INSTALL AC_PROG_MAKE_SET AM_PROG_CC_C_O +AC_PROG_LIBTOOL # Checks for typedefs, structures, and compiler characteristics AC_C_INLINE diff --git a/src/Makefile.am b/src/Makefile.am index f041deb..9320631 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,5 +1,5 @@ -bin_PROGRAMS = fatcache stg_ins_test - +bin_PROGRAMS = fatcache libfc_test +noinst_LIBRARIES = libfc.a AM_CPPFLAGS = -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 CFLAGS = -g AM_CFLAGS = -Wall @@ -44,3 +44,19 @@ stg_ins_test_SOURCES = \ fc_util.c fc_util.h \ fc_queue.h \ stg_ins_test.c + +libfc_test_LDFLAGS= -lpthread -lrt +libfc_test_LDADD=libfc.a +libfc_test_SOURCES = \ + fc_test.c + +libfc_a_SOURCES = \ + fc_slab.c fc_slab.h \ + fc_item.c fc_item.h \ + fc_itemx.c fc_itemx.h \ + fc_time.c fc_time.h \ + fc_sha1.c fc_sha1.h \ + fc_log.c fc_log.h \ + fc_util.c fc_util.h \ + fc_queue.h \ + fc.h libfc.c diff --git a/src/fc.h b/src/fc.h new file mode 100644 index 0000000..8b561c1 --- /dev/null +++ b/src/fc.h @@ -0,0 +1,43 @@ +#ifndef _FC_H_ +#define _FC_H_ + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#define FC_CHUNK_SIZE ITEM_CHUNK_SIZE +#define FC_SLAB_SIZE SLAB_SIZE + +#define FC_DAEMONIZE true + +#define FC_LOG_FILE NULL +#define FC_LOG_DEFAULT LOG_INFO +#define FC_LOG_MIN LOG_EMERG +#define FC_LOG_MAX LOG_PVERB + +#define FC_PORT 11211 +#define FC_ADDR "0.0.0.0" + +#define FC_HASH_POWER ITEMX_HASH_POWER + +#define FC_FACTOR 1.25 + +#define FC_INDEX_MEMORY (64 * MB) +#define FC_SLAB_MEMORY (64 * MB) + +#define FC_SERVER_ID 0 +#define FC_SERVER_N 1 + +void set_options(settings_t* s); +rstatus_t fc_generate_profile(void); +rstatus_t fc_init(); + +#endif diff --git a/src/fc_settings.h b/src/fc_settings.h index 89b7505..cfe15f0 100644 --- a/src/fc_settings.h +++ b/src/fc_settings.h @@ -1,6 +1,6 @@ #ifndef _FC_SETTINGS_H_ #define _FC_SETTINGS_H_ -struct settings { +typedef struct settings { bool daemonize; /* daemonize? */ char *log_filename; /* log filename */ @@ -25,5 +25,5 @@ struct settings { uint32_t server_id; /* server id */ uint32_t server_n; /* # server */ -}; +}settings_t; #endif //_FC_SETTINGS_H_ diff --git a/src/fc_test.c b/src/fc_test.c new file mode 100644 index 0000000..ef76582 --- /dev/null +++ b/src/fc_test.c @@ -0,0 +1,205 @@ +#include +#include +#include +int put(char* key, int nkey, char* value, int vlen, int expiry, int flags); +int get(char* key, int nkey, char* value); +int delete(char* key, int nkey); +int num(char *src_key, int src_key_len, int num, int expiry, int flags); +int put(char* key, int nkey, char* value, int vlen, int expiry, int flags){ + uint8_t md[20]; + uint32_t hash; + + uint8_t * tmp_key = (uint8_t*)key; + sha1(tmp_key, nkey, md); + hash = sha1_hash(md); + uint8_t cid; + struct item *it; + + cid = item_slabcid(nkey, vlen); + if (cid == SLABCLASS_INVALID_ID) { + return -1; + } + + itemx_removex(hash, md); + it = item_get(tmp_key, nkey, cid, vlen, time_reltime(expiry), + flags, md, hash); + if (it == NULL) { + return -2; + } + memcpy(item_data(it), value, (size_t)(vlen)); + return 0; +} + +int get(char* key, int nkey, char* value){ + uint8_t md[20]; + uint32_t hash; + + struct itemx *itx; + struct item *it; + + uint8_t * tmp_key = (uint8_t*)key; + sha1(tmp_key, nkey, md); + hash = sha1_hash(md); + + itx = itemx_getx(hash,md); + if (itx == NULL) { + return 1; + } + if (itemx_expired(itx)) { + return 2; + } + it = slab_read_item(itx->sid, itx->offset); + if (it == NULL) { + // rsp_send_error(ctx, conn, msg, MSG_RSP_SERVER_ERROR, errno); + return -1; + } + memcpy(value, item_data(it), it->ndata); + return 0; +} +int delete(char* key, int nkey){ + uint8_t md[20]; + uint32_t hash; + uint8_t * tmp_key = (uint8_t*)key; + sha1(tmp_key, nkey, md); + hash = sha1_hash(md); + + //uint8_t cid; + struct itemx *itx; + + itx = itemx_getx(hash, md); + if (itx == NULL) { + // rsp_send_status(ctx, conn, msg, MSG_RSP_NOT_FOUND); + return 0; + } + //cid = slab_get_cid(itx->sid); + itemx_removex(hash, md); + return 0; +} + +int num(char *src_key, int src_key_len, int num, int expiry, int flags){ + rstatus_t status; + uint8_t *pkey, nkey, cid; + struct item *it; + struct itemx *itx; + uint64_t cnum; + int64_t nnum; + char numstr[FC_UINT64_MAXLEN]; + int n; + + pkey = (uint8_t *)src_key; + nkey = (uint8_t)(src_key_len); + + uint8_t md[20]; + uint32_t hash; + sha1(pkey, nkey, md); + hash = sha1_hash(md); + + /* 1). look up existing itemx */ + itx = itemx_getx(hash, md); + if (itx == NULL || itemx_expired(itx)) { + /* 2a). miss -> return NOT_FOUND */ + //rsp_send_status(ctx, conn, msg, MSG_RSP_NOT_FOUND); + return -1; + } + + /* 2b). hit -> read existing item into it */ + it = slab_read_item(itx->sid, itx->offset); + if (it == NULL) { + //rsp_send_error(ctx, conn, msg, MSG_RSP_SERVER_ERROR, errno); + return -2; + } + + /* 3). sanity check item data to be a number */ + status = fc_atou64(item_data(it), it->ndata, &cnum); + if (status != FC_OK) { + //rsp_send_error(ctx, conn, msg, MSG_RSP_CLIENT_ERROR, EINVAL); + return -3; + } + + /* 4). remove existing itemx of it */ + itemx_removex(hash, md); + + /* 5). compute the new incr/decr number nnum and numstr */ + nnum = cnum + num; + if (nnum<0) + nnum = 0; + n = _scnprintf(numstr, sizeof(numstr), "%"PRIu64"", (uint64_t)nnum); + + /* 6). alloc new item that can hold n worth of bytes */ + cid = item_slabcid(nkey, n); + ASSERT(cid != SLABCLASS_INVALID_ID); + + it = item_get(pkey, nkey, cid, n, time_reltime(expiry), flags, + md, hash); + if (it == NULL) { + //rsp_send_error(ctx, conn, msg, MSG_RSP_SERVER_ERROR, ENOMEM); + return -2; + } + + /* 7). copy numstr to it */ + memcpy(item_data(it), numstr, n); + return 0; +} +int main(int argc, char** argv){ + char key[5]= "testk"; + char *value=NULL; + char *ret=NULL; + if (argc < 2){ + printf("%s [test value]\n", argv[0]); + return 0; + } + + settings_t t_s; + t_s.log_filename = "fclog"; + t_s.verbose = 11; + t_s.ssd_device = argv[1]; + t_s.max_index_memory = 64*1024*1024; + t_s.max_slab_memory = 64*1024*1024; + t_s.factor = 1.1; + + set_options(&t_s); + fc_generate_profile(); + fc_init(); + if (argc > 2){ + printf("test value: %s\n", argv[2]); + value = malloc((strlen(argv[2])+1) * sizeof(char)); + ret = malloc((strlen(argv[2])+1) * sizeof(char)); + memset(value, 0, (strlen(argv[2])+1) * sizeof(char)); + memcpy(value, argv[2], strlen(argv[2])); + }else{ + ret = malloc(10 * sizeof(char)); + value = malloc(10 * sizeof(char)); + memcpy(value, "testv", 5); + + } + + if (get(key, 5, ret) == 1){ + printf("no data\n"); + } + if (put(key, 5, value, strlen(value), 0, 1) == 0){ + printf("set data ok\n"); + } + if (get(key, 5, ret) == 0){ + printf("get data %s\n", ret); + } + delete(key, 5); + if (get(key, 5, ret) == 1){ + printf("after delete, no data\n"); + } + printf("start check add count\n"); + if (put(key, 5, value, strlen(value), 0, 1) == 0){ + printf("set data ok\n"); + } + if (get(key, 5, ret) == 1){ + printf("error, no data\n"); + }else{ + printf("get data: %s\n", ret); + } + if (num(key, 5, 12, 0, 1) == 0){ + printf("num data ok\n"); + } + if (get(key, 5, ret) == 0){ + printf("get data: %s\n", ret); + } + return 0; +} diff --git a/src/libfc.c b/src/libfc.c new file mode 100644 index 0000000..e44b6f2 --- /dev/null +++ b/src/libfc.c @@ -0,0 +1,105 @@ +#include +#include +#include +struct settings settings; /* fatcache settings */ + +void set_options(settings_t* s){ + settings.daemonize = FC_DAEMONIZE; + + settings.log_filename = FC_LOG_FILE; + settings.verbose = FC_LOG_DEFAULT; + + settings.port = FC_PORT; + settings.addr = FC_ADDR; + settings.hash_power = FC_HASH_POWER; + + settings.factor = FC_FACTOR; + settings.max_index_memory = FC_INDEX_MEMORY; + settings.max_slab_memory = FC_SLAB_MEMORY; + settings.chunk_size = FC_CHUNK_SIZE; + settings.slab_size = FC_SLAB_SIZE; + + memset(settings.profile, 0, sizeof(settings.profile)); + settings.profile_last_id = SLABCLASS_MAX_ID; + + settings.ssd_device = NULL; + + settings.server_id = FC_SERVER_ID; + settings.server_n = FC_SERVER_N; + // use config + if (s == NULL) + return; + settings.log_filename = s->log_filename; + settings.verbose = s->verbose; + settings.ssd_device = s->ssd_device; + settings.max_index_memory = s->max_index_memory; + settings.max_slab_memory = s->max_slab_memory; + settings.factor = s->factor; + return; +} + +rstatus_t +fc_generate_profile(void) +{ + size_t *profile = settings.profile; /* slab profile */ + uint8_t id; /* slab class id */ + size_t item_sz, last_item_sz; /* current and last item chunk size */ + size_t min_item_sz, max_item_sz; /* min and max item chunk size */ + + ASSERT(settings.chunk_size % FC_ALIGNMENT == 0); + ASSERT(settings.chunk_size <= slab_data_size()); + + min_item_sz = settings.chunk_size; + max_item_sz = slab_data_size(); + id = SLABCLASS_MIN_ID; + item_sz = min_item_sz; + + while (id < SLABCLASS_MAX_ID && item_sz < max_item_sz) { + /* save the cur item chunk size */ + last_item_sz = item_sz; + profile[id] = item_sz; + id++; + + /* get the next item chunk size */ + item_sz *= settings.factor; + if (item_sz == last_item_sz) { + item_sz++; + } + item_sz = FC_ALIGN(item_sz, FC_ALIGNMENT); + } + + /* last profile entry always has a 1 item/slab of maximum size */ + profile[id] = max_item_sz; + settings.profile_last_id = id; + settings.max_chunk_size = max_item_sz; + + return FC_OK; +} + +rstatus_t fc_init(){ + rstatus_t status; + + status = log_init(settings.verbose, settings.log_filename); + if (status != FC_OK) { + return status; + } + + status = time_init(); + if (status != FC_OK) { + return status; + } + + status = itemx_init(); + if (status != FC_OK) { + return status; + } + + item_init(); + + status = slab_init(); + if (status != FC_OK) { + return status; + } + + return FC_OK; +}