From 9f04415ff81f9f63955c0994f6aa1bd1c8e60377 Mon Sep 17 00:00:00 2001 From: eljeko Date: Thu, 27 Jul 2023 19:09:40 +0200 Subject: [PATCH 1/2] Closes #75 --- pkg/cmd/templateRun.go | 9 ++- pkg/ctx/context.go | 5 ++ pkg/emitter/emitter.go | 4 ++ pkg/emitter/loop.go | 10 ++- pkg/functions/functions.go | 60 ++++++++++++++++ pkg/functions/utilities.go | 13 ++++ pkg/functions_test/functions_test.go | 99 ++++++++++++++++++++++++++ templates/csv_product.tpl | 6 ++ templates/csv_user.tpl | 8 +++ templates/usercsv.tpl | 8 +++ testfiles/products.csv | 101 +++++++++++++++++++++++++++ testfiles/test2.csv | 3 + testfiles/test3.csv | 4 ++ 13 files changed, 326 insertions(+), 4 deletions(-) create mode 100644 templates/csv_product.tpl create mode 100644 templates/csv_user.tpl create mode 100644 templates/usercsv.tpl create mode 100644 testfiles/products.csv create mode 100644 testfiles/test2.csv create mode 100644 testfiles/test3.csv diff --git a/pkg/cmd/templateRun.go b/pkg/cmd/templateRun.go index 9eb372f5..cec4efd4 100644 --- a/pkg/cmd/templateRun.go +++ b/pkg/cmd/templateRun.go @@ -58,7 +58,9 @@ jr template run --template "{{name}}" seed, _ := cmd.Flags().GetInt64("seed") topic, _ := cmd.Flags().GetString("topic") preload, _ := cmd.Flags().GetInt("preload") - + + csv, _ := cmd.Flags().GetString("csv") + if kcat { oneline = true output = "stdout" @@ -112,7 +114,7 @@ jr template run --template "{{name}}" e := emitter.Emitter{ Name: name, - Locale: locale, + Locale: locale, Num: num, Frequency: frequency, Duration: duration, @@ -125,6 +127,7 @@ jr template run --template "{{name}}" Topic: topic, Kcat: kcat, Oneline: oneline, + Csv: csv, } functions.SetSeed(seed) @@ -144,6 +147,8 @@ func init() { templateRunCmd.Flags().Int64("seed", time.Now().UTC().UnixNano(), "Seed to init pseudorandom generator") + templateRunCmd.Flags().String("csv", "", "Path to csv file to use") + templateRunCmd.Flags().StringP("kafkaConfig", "F", "", "Kafka configuration") templateRunCmd.Flags().String("registryConfig", "", "Kafka configuration") templateRunCmd.Flags().Bool("embedded", false, "If enabled, [template] must be a string containing a template, to be embedded directly in the script") diff --git a/pkg/ctx/context.go b/pkg/ctx/context.go index 6d3082de..823c0521 100644 --- a/pkg/ctx/context.go +++ b/pkg/ctx/context.go @@ -39,9 +39,12 @@ type Context struct { CtxLock sync.RWMutex CtxList map[string][]string CtxListLock sync.RWMutex + CtxCSV map[int]map[string]string + CtxCSVLock sync.RWMutex LastIndex int CountryIndex int CityIndex int + CurrentIterationLoopIndex int } func init() { @@ -57,6 +60,8 @@ func init() { CtxLock: sync.RWMutex{}, CtxList: make(map[string][]string), CtxListLock: sync.RWMutex{}, + CtxCSV: make(map[int]map[string]string), + CtxCSVLock: sync.RWMutex{}, LastIndex: -1, CountryIndex: 232, CityIndex: -1, diff --git a/pkg/emitter/emitter.go b/pkg/emitter/emitter.go index e1049a36..0dd1091e 100644 --- a/pkg/emitter/emitter.go +++ b/pkg/emitter/emitter.go @@ -54,6 +54,7 @@ type Emitter struct { Topic string `mapstructure:"topic"` Kcat bool `mapstructure:"kcat"` Oneline bool `mapstructure:"oneline"` + Csv string `mapstructure:"csv"` Producer Producer KTpl tpl.Tpl VTpl tpl.Tpl @@ -61,6 +62,8 @@ type Emitter struct { func (e *Emitter) Initialize(conf configuration.GlobalConfiguration) { + functions.InitCSV(e.Csv) + templateName := e.ValueTemplate if e.EmbeddedTemplate == "" { path := os.ExpandEnv(fmt.Sprintf("%s/%s", constants.JRhome, "templates")) @@ -71,6 +74,7 @@ func (e *Emitter) Initialize(conf configuration.GlobalConfiguration) { log.Printf("Template '%s' not found in %s\n", templateName, path) } } + keyTpl, err := tpl.NewTpl("key", e.KeyTemplate, functions.FunctionsMap(), &ctx.JrContext) if err != nil { diff --git a/pkg/emitter/loop.go b/pkg/emitter/loop.go index d602fc12..efc97a49 100644 --- a/pkg/emitter/loop.go +++ b/pkg/emitter/loop.go @@ -73,13 +73,15 @@ func DoLoop(es []Emitter) { defer stop() for i := 0; i < numTimers; i++ { + + index := i stopChannels[i] = make(chan struct{}) go func(timerIndex int) { defer wg.Done() - + frequency := es[timerIndex].Frequency if frequency > 0 { ticker := time.NewTicker(es[timerIndex].Frequency) @@ -113,8 +115,12 @@ func doTemplate(emitter Emitter) { ctx.JrContext.Locale = emitter.Locale ctx.JrContext.CountryIndex = functions.IndexOf(strings.ToUpper(emitter.Locale), "country") - for i := 0; i < emitter.Num; i++ { + + + for i := 0; i < emitter.Num; i++ { + ctx.JrContext.CurrentIterationLoopIndex++ + k := emitter.KTpl.Execute() v := emitter.VTpl.Execute() if emitter.Oneline { diff --git a/pkg/functions/functions.go b/pkg/functions/functions.go index fbea0e28..b4c170fa 100644 --- a/pkg/functions/functions.go +++ b/pkg/functions/functions.go @@ -22,6 +22,7 @@ package functions import ( "bufio" + "encoding/csv" "fmt" "github.com/google/uuid" "github.com/ugol/jr/pkg/constants" @@ -192,6 +193,7 @@ var fmap = map[string]interface{}{ "random_n_v_from_list": RandomNValuesFromList, "get_v": GetV, "set_v": SetV, + "fromcsv": fromcsv, } // Seed sets seeds and can be used in a template @@ -448,3 +450,61 @@ func initialize(filename string) []string { return words } + +func InitCSV(csvpath string) { + //Loads the csv file in the context + if len(csvpath) > 0 { + + var csvHeaders = make(map[int]string) + csvValues := make(map[int]map[string]string) + + if _, err := os.Stat(csvpath); err != nil { + println("File does not exist: ", csvpath) + os.Exit(1) + } + + file, err := os.Open(csvpath) + + if err != nil { + println("Error opening file:", csvpath, "error:", err) + os.Exit(1) + } else { + reader := csv.NewReader(file) + + lines, err := reader.ReadAll() + if err != nil { + println("Error reading CSV file:", csvpath, "error:", err) + os.Exit(1) + } else { + + for row := 0; row < len(lines); row++ { + var aRow = lines[row] + for col := 0; col < len(aRow); col++ { + var value = aRow[col] + //print(" ROW -> ",row) + if row == 0 { + //print(" H: ", value) + csvHeaders[col] = strings.Trim(value, " ") + } else { + + val, exists := csvValues[row-1] + if exists { + val[csvHeaders[col]] = strings.Trim(value, " ") + csvValues[row-1] = val + } else { + var localmap = make(map[string]string) + localmap[csvHeaders[col]] = strings.Trim(value, " ") + csvValues[row-1] = localmap + } + // print(" V: ", value) + } + } + //println() + } + ctx.JrContext.CtxCSV = csvValues + } + } + defer file.Close() + } + +} \ No newline at end of file diff --git a/pkg/functions/utilities.go b/pkg/functions/utilities.go index f6ecb09f..3972c828 100644 --- a/pkg/functions/utilities.go +++ b/pkg/functions/utilities.go @@ -89,3 +89,16 @@ func Contains(s []string, str string) bool { } return false } + +// get the label value from csv file +func fromcsv(c string) string { + ctx.JrContext.CtxCSVLock.Lock() + defer ctx.JrContext.CtxCSVLock.Unlock() + + if len(ctx.JrContext.CtxCSV) > 0 { + return ctx.JrContext.CtxCSV[(ctx.JrContext.CurrentIterationLoopIndex-1)%len(ctx.JrContext.CtxCSV)][c] + } else { + return "" + } + +} \ No newline at end of file diff --git a/pkg/functions_test/functions_test.go b/pkg/functions_test/functions_test.go index f1d17f3a..e59febda 100644 --- a/pkg/functions_test/functions_test.go +++ b/pkg/functions_test/functions_test.go @@ -23,6 +23,7 @@ package functions_test import ( "bytes" "fmt" + "github.com/ugol/jr/pkg/ctx" "github.com/ugol/jr/pkg/functions" "testing" "text/template" @@ -160,3 +161,101 @@ func runtv(tpl, expect string, vars interface{}) error { } return nil } + + +func TestParamFromCSV_odd(t *testing.T) { + functions.InitCSV("../../testfiles/test3.csv") + + tpl := `{{fromcsv "NAME"}} {{fromcsv "SURNAME"}}` + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Jhon Brown"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Mary White"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Anna Green"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Jhon Brown"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Mary White"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Anna Green"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Jhon Brown"); err != nil { + t.Error(err) + } +} + +func TestParamFromCSV_not_initialized(t *testing.T) { + + ctx.JrContext.CtxCSV = make(map[int]map[string]string) + + tpl := `{{fromcsv "NAME"}} {{fromcsv "SURNAME"}}` + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, " "); err != nil { + t.Error(err) + } +} + +func TestParamFromCSV_even(t *testing.T) { + functions.InitCSV("../../testfiles/test2.csv") + + tpl := `{{fromcsv "NAME"}} {{fromcsv "SURNAME"}}` + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Jhon Brown"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Mary White"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Jhon Brown"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Mary White"); err != nil { + t.Error(err) + } + + ctx.JrContext.CurrentIterationLoopIndex++ + + if err := runt(tpl, "Jhon Brown"); err != nil { + t.Error(err) + } +} diff --git a/templates/csv_product.tpl b/templates/csv_product.tpl new file mode 100644 index 00000000..56763224 --- /dev/null +++ b/templates/csv_product.tpl @@ -0,0 +1,6 @@ +{ + "product_id": "{{fromcsv "id"}}", + "name": "{{fromcsv "name"}}", + "brand": "{{fromcsv "brand"}}", + "page_url": "https://www.acme.com/product/{{random_string 4 5}}" +} \ No newline at end of file diff --git a/templates/csv_user.tpl b/templates/csv_user.tpl new file mode 100644 index 00000000..692e8ccf --- /dev/null +++ b/templates/csv_user.tpl @@ -0,0 +1,8 @@ +{ + "age": {{integer 20 60}}, + "eyeColor": "{{randoms "blue|brown|green"}}", + "name": "{{fromcsv "NAME"}} {{fromcsv "SURNAME"}}", + "gender": "{{gender}}", + "company": "{{company}}", + "email": "{{fromcsv "NAME"}}.{{fromcsv "SURNAME"}}@thelandoof.oz" +} diff --git a/templates/usercsv.tpl b/templates/usercsv.tpl new file mode 100644 index 00000000..692e8ccf --- /dev/null +++ b/templates/usercsv.tpl @@ -0,0 +1,8 @@ +{ + "age": {{integer 20 60}}, + "eyeColor": "{{randoms "blue|brown|green"}}", + "name": "{{fromcsv "NAME"}} {{fromcsv "SURNAME"}}", + "gender": "{{gender}}", + "company": "{{company}}", + "email": "{{fromcsv "NAME"}}.{{fromcsv "SURNAME"}}@thelandoof.oz" +} diff --git a/testfiles/products.csv b/testfiles/products.csv new file mode 100644 index 00000000..df404ed8 --- /dev/null +++ b/testfiles/products.csv @@ -0,0 +1,101 @@ +id,name,brand +1,"Beef - Ox Tongue, Pickled",Gabspot +2,Squid - Breaded,Browsezoom +3,"Crab - Dungeness, Whole",Vipe +4,"Chicken - Thigh, Bone In",Vipe +5,Bread - Pita,Linktype +6,"Cheese - Boursin, Garlic / Herbs",Jaloo +7,"Cookies - Oreo, 4 Pack",Tagopia +8,"Water, Tap",Omba +9,Zucchini - Green,Jaxnation +10,Appetizer - Sausage Rolls,Tagtune +11,Table Cloth 54x54 Colour,Voonyx +12,Veal - Sweetbread,Yoveo +13,Wine - Redchard Merritt,Browsebug +14,Crush - Cream Soda,Voonyx +15,Mustard - Individual Pkg,Dynabox +16,Pimento - Canned,Flipopia +17,"Crab - Dungeness, Whole, live",Yakidoo +18,Coffee - Flavoured,JumpXS +19,Magnotta Bel Paese Red,Zoomzone +20,"Shrimp - Baby, Warm Water",Gabtype +21,Soup - Base Broth Beef,Linkbridge +22,Fudge - Cream Fudge,Zooveo +23,Flavouring - Raspberry,Bubbletube +24,Wine - Lou Black Shiraz,Quatz +25,Bread Cranberry Foccacia,Tazzy +26,Napkin White - Starched,BlogXS +27,Wine - Sauvignon Blanc Oyster,Thoughtworks +28,"Crab - Back Fin Meat, Canned",Rhycero +29,Guinea Fowl,Brightbean +30,"Veal - Inside Round / Top, Lean",Tanoodle +31,"Chilli Paste, Ginger Garlic",Kaymbo +32,Beans - Soya Bean,Twitterbridge +33,Wine - Zinfandel California 2002,Lajo +34,Appetiser - Bought,Lazz +35,"Bread - Crumbs, Bulk",Roodel +36,"Turnip - White, Organic",Trudeo +37,Tea - Grapefruit Green Tea,Photobug +38,Scallops - Live In Shell,Bluejam +39,Lid Tray - 12in Dome,Kazu +40,Potatoes - Pei 10 Oz,Youspan +41,Chocolate - Dark Callets,Skalith +42,Soup Campbells Turkey Veg.,Edgeblab +43,Cake - Mini Potato Pancake,Yotz +44,Plastic Wrap,Divavu +45,Ginsing - Fresh,Thoughtstorm +46,"Beans - Long, Chinese",Edgeify +47,Dasheen,Browsedrive +48,Snapple Raspberry Tea,Brainlounge +49,Chinese Foods - Pepper Beef,Rhyzio +50,Onions - Spanish,Centidel +51,"Soup - Clam Chowder, Dry Mix",Eimbee +52,French Pastry - Mini Chocolate,Devcast +53,Icecream Cone - Areo Chocolate,Twitterworks +54,Pepper - Pablano,Oyondu +55,Pike - Frozen Fillet,Fiveclub +56,"Nut - Almond, Blanched, Whole",Gigashots +57,Pants Custom Dry Clean,Divape +58,Cookie Double Choco,Zava +59,Appetizer - Tarragon Chicken,Mybuzz +60,Glove - Cutting,Skivee +61,"Basil - Dry, Rubbed",Quinu +62,Beer - True North Lager,Eabox +63,Bread - Olive,Fivespan +64,Appetizer - Mushroom Tart,Oyonder +65,Buffalo - Striploin,Avamba +66,Peas Snow,Latz +67,Gelatine Powder,Camimbo +68,"Cake Circle, Paprus",Kwideo +69,Pastry - Cheese Baked Scones,Edgepulse +70,"Noodles - Cellophane, Thin",Brainbox +71,Brandy - Bar,Dabshots +72,"Soup - Knorr, Veg / Beef",Skyba +73,"Potatoes - Yukon Gold, 80 Ct",Riffpath +74,Cookies Almond Hazelnut,Trilia +75,Flavouring - Orange,Twitternation +76,Wine - Segura Viudas Aria Brut,Shuffledrive +77,Yeast Dry - Fermipan,Leenti +78,Wine - Cava Aria Estate Brut,Yodoo +79,Cheese Cheddar Processed,Gabtype +80,Bread - Pain Au Liat X12,Youopia +81,Tea - Grapefruit Green Tea,Aibox +82,Chicken Breast Wing On,Youspan +83,Wine La Vielle Ferme Cote Du,Innotype +84,Rambutan,Jaxnation +85,Onions - Red Pearl,Jabberbean +86,Ice Cream Bar - Hageen Daz To,Fivespan +87,"Sprite, Diet - 355ml",Jetpulse +88,"Beans - Black Bean, Canned",Flipopia +89,Sugar - Cubes,Flipstorm +90,Wine - Sauvignon Blanc,Avavee +91,Tart - Butter Plain Squares,Photofeed +92,Veal - Loin,Trunyx +93,Soup - Campbells Pasta Fagioli,Quinu +94,Dasheen,Kwilith +95,Beets - Golden,Edgetag +96,Beer - Mill St Organic,Babbleopia +97,Tomato - Tricolor Cherry,Youopia +98,Radish,Tazz +99,"Beans - Black Bean, Canned",Youspan +100,Carbonated Water - White Grape,Skivee \ No newline at end of file diff --git a/testfiles/test2.csv b/testfiles/test2.csv new file mode 100644 index 00000000..32d65694 --- /dev/null +++ b/testfiles/test2.csv @@ -0,0 +1,3 @@ +NAME, SURNAME +Jhon, Brown +Mary, White \ No newline at end of file diff --git a/testfiles/test3.csv b/testfiles/test3.csv new file mode 100644 index 00000000..60f94e6c --- /dev/null +++ b/testfiles/test3.csv @@ -0,0 +1,4 @@ +NAME, SURNAME +Jhon, Brown +Mary, White +Anna, Green \ No newline at end of file From a10aa5dc92de1e1d3f35690820bbb473bfc9acb5 Mon Sep 17 00:00:00 2001 From: eljeko Date: Thu, 27 Jul 2023 19:14:26 +0200 Subject: [PATCH 2/2] clean --- templates/usercsv.tpl | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 templates/usercsv.tpl diff --git a/templates/usercsv.tpl b/templates/usercsv.tpl deleted file mode 100644 index 692e8ccf..00000000 --- a/templates/usercsv.tpl +++ /dev/null @@ -1,8 +0,0 @@ -{ - "age": {{integer 20 60}}, - "eyeColor": "{{randoms "blue|brown|green"}}", - "name": "{{fromcsv "NAME"}} {{fromcsv "SURNAME"}}", - "gender": "{{gender}}", - "company": "{{company}}", - "email": "{{fromcsv "NAME"}}.{{fromcsv "SURNAME"}}@thelandoof.oz" -}