diff --git a/assets/database/db.bin b/assets/database/db.bin index 40cd49df20..c23712efbf 100644 Binary files a/assets/database/db.bin and b/assets/database/db.bin differ diff --git a/assets/database/db.json b/assets/database/db.json index ddc82a1c69..6697a87e24 100644 --- a/assets/database/db.json +++ b/assets/database/db.json @@ -12958,6 +12958,7 @@ {"id":67383,"name":"Unholy Force","icon":"spell_shadow_unholystrength"}, {"id":67839,"name":"Mind Amplification Dish","icon":"trade_engineering"}, {"id":68051,"name":"Overpower Ready!","icon":"ability_rogue_hungerforblood"}, +{"id":68055,"name":"Judgements of the Just","icon":"spell_holy_righteousfury"}, {"id":70164,"name":"Rune of the Nerubian Carapace","icon":"inv_sword_61"}, {"id":70550,"name":"Leggings of Woven Death","icon":"inv_pants_cloth_34"}, {"id":70551,"name":"Deathfrost Boots","icon":"inv_boots_cloth_25"}, diff --git a/tools/database/atlasloot.go b/tools/database/atlasloot.go index 31e4a5898d..b32bdf184d 100644 --- a/tools/database/atlasloot.go +++ b/tools/database/atlasloot.go @@ -129,7 +129,7 @@ func readAtlasLootDungeonData(db *WowDatabase, expansion proto.Expansion, srcUrl curLocation := 0 for _, itemMatch := range itemsPattern.FindAllStringSubmatch(difficultyMatch[0], -1) { - itemParams := core.MapSlice(strings.Split(itemMatch[1], ","), strings.TrimSpace) + itemParams := core.MapSlice(strings.Split(itemMatch[1], ","), func(s string) string { return strings.TrimSpace(s) }) location, _ := strconv.Atoi(itemParams[0]) // Location within AtlasLoot's menu. idStr := itemParams[1] @@ -178,8 +178,8 @@ func readAtlasLootDungeonData(db *WowDatabase, expansion proto.Expansion, srcUrl } func readZoneData(db *WowDatabase) { - zoneIDs := make([]int32, 0, len(db.Zones)) - for zoneID := range db.Zones { + var zoneIDs []int32 + for zoneID, _ := range db.Zones { zoneIDs = append(zoneIDs, zoneID) } zoneIDStrs := core.MapSlice(zoneIDs, func(zoneID int32) string { return strconv.Itoa(int(zoneID)) }) diff --git a/tools/database/blizzard_api.go b/tools/database/blizzard_api.go index 6753ca8c81..72e7c63342 100644 --- a/tools/database/blizzard_api.go +++ b/tools/database/blizzard_api.go @@ -123,7 +123,7 @@ func getItemData(itemId int, accessToken string) BlizzardItemResponse { log.Fatal(err) } - fmt.Print(string(resultBody)) + fmt.Printf(string(resultBody)) itemResponse := BlizzardItemResponse{} err = json.Unmarshal(resultBody, &itemResponse) if err != nil { diff --git a/tools/database/database.go b/tools/database/database.go index 5bcd4be61e..02c24b7d16 100644 --- a/tools/database/database.go +++ b/tools/database/database.go @@ -1,16 +1,16 @@ package database import ( - "bytes" "fmt" + "log" + "os" + "strings" + "github.com/wowsims/wotlk/sim/core/proto" "github.com/wowsims/wotlk/tools" - "golang.org/x/exp/maps" "golang.org/x/exp/slices" "google.golang.org/protobuf/encoding/protojson" googleProto "google.golang.org/protobuf/proto" - "log" - "os" ) type EnchantDBKey struct { @@ -56,17 +56,31 @@ func NewWowDatabase() *WowDatabase { } func (db *WowDatabase) Clone() *WowDatabase { - return &WowDatabase{ - Items: maps.Clone(db.Items), - Enchants: maps.Clone(db.Enchants), - Gems: maps.Clone(db.Gems), - Zones: maps.Clone(db.Zones), - Npcs: maps.Clone(db.Npcs), - ItemIcons: maps.Clone(db.ItemIcons), - SpellIcons: maps.Clone(db.SpellIcons), - Encounters: slices.Clone(db.Encounters), - GlyphIDs: slices.Clone(db.GlyphIDs), + other := NewWowDatabase() + + for k, v := range db.Items { + other.Items[k] = v + } + for k, v := range db.Enchants { + other.Enchants[k] = v + } + for k, v := range db.Gems { + other.Gems[k] = v + } + for k, v := range db.Zones { + other.Zones[k] = v + } + for k, v := range db.Npcs { + other.Npcs[k] = v + } + for k, v := range db.ItemIcons { + other.ItemIcons[k] = v } + for k, v := range db.SpellIcons { + other.SpellIcons[k] = v + } + + return other } func (db *WowDatabase) MergeItems(arr []*proto.UIItem) { @@ -76,7 +90,7 @@ func (db *WowDatabase) MergeItems(arr []*proto.UIItem) { } func (db *WowDatabase) MergeItem(src *proto.UIItem) { if dst, ok := db.Items[src.Id]; ok { - // googleproto.Merge concatenates lists, but we want replacement, so do them manually. + // googleproto.Merge concatenates lists but we want replacement, so do them manually. if src.Stats != nil { dst.Stats = src.Stats src.Stats = nil @@ -99,7 +113,7 @@ func (db *WowDatabase) MergeEnchants(arr []*proto.UIEnchant) { func (db *WowDatabase) MergeEnchant(src *proto.UIEnchant) { key := EnchantToDBKey(src) if dst, ok := db.Enchants[key]; ok { - // googleproto.Merge concatenates lists, but we want replacement, so do them manually. + // googleproto.Merge concatenates lists but we want replacement, so do them manually. if src.Stats != nil { dst.Stats = src.Stats src.Stats = nil @@ -117,7 +131,7 @@ func (db *WowDatabase) MergeGems(arr []*proto.UIGem) { } func (db *WowDatabase) MergeGem(src *proto.UIGem) { if dst, ok := db.Gems[src.Id]; ok { - // googleproto.Merge concatenates lists, but we want replacement, so do them manually. + // googleproto.Merge concatenates lists but we want replacement, so do them manually. if src.Stats != nil { dst.Stats = src.Stats src.Stats = nil @@ -176,49 +190,71 @@ func (db *WowDatabase) AddSpellIcon(id int32, tooltips map[int32]WowheadItemResp } } -type idKeyed interface { - GetId() int32 -} - -func mapToSlice[T idKeyed](m map[int32]T) []T { - vs := make([]T, 0, len(m)) - for _, v := range m { - vs = append(vs, v) +func (db *WowDatabase) MergeUIProto(dbProto *proto.UIDatabase) { + db.MergeItems(dbProto.Items) + db.MergeEnchants(dbProto.Enchants) + db.MergeGems(dbProto.Gems) + db.MergeZones(dbProto.Zones) + db.MergeNpcs(dbProto.Npcs) + for _, icon := range dbProto.ItemIcons { + db.ItemIcons[icon.Id] = icon + } + for _, icon := range dbProto.SpellIcons { + db.SpellIcons[icon.Id] = icon } - slices.SortFunc(vs, func(a, b T) bool { - return a.GetId() < b.GetId() - }) - return vs } func (db *WowDatabase) ToUIProto() *proto.UIDatabase { - enchants := make([]*proto.UIEnchant, 0, len(db.Enchants)) - for _, v := range db.Enchants { - enchants = append(enchants, v) - } - slices.SortFunc(enchants, func(v1, v2 *proto.UIEnchant) bool { - return v1.EffectId < v2.EffectId || v1.EffectId == v2.EffectId && v1.Type < v2.Type - }) - - return &proto.UIDatabase{ - Items: mapToSlice(db.Items), - Enchants: enchants, - Gems: mapToSlice(db.Gems), + uidb := &proto.UIDatabase{ Encounters: db.Encounters, - Zones: mapToSlice(db.Zones), - Npcs: mapToSlice(db.Npcs), - ItemIcons: mapToSlice(db.ItemIcons), - SpellIcons: mapToSlice(db.SpellIcons), GlyphIds: db.GlyphIDs, } -} -func sliceToMap[T idKeyed](vs []T) map[int32]T { - m := make(map[int32]T, len(vs)) - for _, v := range vs { - m[v.GetId()] = v + for _, v := range db.Items { + uidb.Items = append(uidb.Items, v) + } + for _, v := range db.Enchants { + uidb.Enchants = append(uidb.Enchants, v) + } + for _, v := range db.Gems { + uidb.Gems = append(uidb.Gems, v) + } + for _, v := range db.Zones { + uidb.Zones = append(uidb.Zones, v) + } + for _, v := range db.Npcs { + uidb.Npcs = append(uidb.Npcs, v) } - return m + for _, v := range db.ItemIcons { + uidb.ItemIcons = append(uidb.ItemIcons, v) + } + for _, v := range db.SpellIcons { + uidb.SpellIcons = append(uidb.SpellIcons, v) + } + + slices.SortStableFunc(uidb.Items, func(v1, v2 *proto.UIItem) bool { + return v1.Id < v2.Id + }) + slices.SortStableFunc(uidb.Enchants, func(v1, v2 *proto.UIEnchant) bool { + return v1.EffectId < v2.EffectId || v1.EffectId == v2.EffectId && v1.Type < v2.Type + }) + slices.SortStableFunc(uidb.Gems, func(v1, v2 *proto.UIGem) bool { + return v1.Id < v2.Id + }) + slices.SortStableFunc(uidb.Zones, func(v1, v2 *proto.UIZone) bool { + return v1.Id < v2.Id + }) + slices.SortStableFunc(uidb.Npcs, func(v1, v2 *proto.UINPC) bool { + return v1.Id < v2.Id + }) + slices.SortStableFunc(uidb.ItemIcons, func(v1, v2 *proto.IconData) bool { + return v1.Id < v2.Id + }) + slices.SortStableFunc(uidb.SpellIcons, func(v1, v2 *proto.IconData) bool { + return v1.Id < v2.Id + }) + + return uidb } func ReadDatabaseFromJson(jsonStr string) *WowDatabase { @@ -227,22 +263,9 @@ func ReadDatabaseFromJson(jsonStr string) *WowDatabase { panic(err) } - enchants := make(map[EnchantDBKey]*proto.UIEnchant, len(dbProto.Enchants)) - for _, v := range dbProto.Enchants { - enchants[EnchantToDBKey(v)] = v - } - - return &WowDatabase{ - Items: sliceToMap(dbProto.Items), - Enchants: enchants, - Gems: sliceToMap(dbProto.Gems), - Zones: sliceToMap(dbProto.Zones), - Npcs: sliceToMap(dbProto.Npcs), - ItemIcons: sliceToMap(dbProto.ItemIcons), - SpellIcons: sliceToMap(dbProto.SpellIcons), - Encounters: dbProto.Encounters, - GlyphIDs: dbProto.GlyphIds, - } + db := NewWowDatabase() + db.MergeUIProto(dbProto) + return db } func (db *WowDatabase) WriteBinaryAndJson(binFilePath, jsonFilePath string) { @@ -265,31 +288,30 @@ func (db *WowDatabase) WriteJson(jsonFilePath string) { // Also write in JSON format so we can manually inspect the contents. // Write it out line-by-line so we can have 1 line / item, making it more human-readable. uidb := db.ToUIProto() - - var buffer bytes.Buffer - buffer.WriteString("{\n") - - tools.WriteProtoArrayToBuffer(uidb.Items, buffer, "items") - buffer.WriteString(",\n") - tools.WriteProtoArrayToBuffer(uidb.Enchants, buffer, "enchants") - buffer.WriteString(",\n") - tools.WriteProtoArrayToBuffer(uidb.Gems, buffer, "gems") - buffer.WriteString(",\n") - tools.WriteProtoArrayToBuffer(uidb.Zones, buffer, "zones") - buffer.WriteString(",\n") - tools.WriteProtoArrayToBuffer(uidb.Npcs, buffer, "npcs") - buffer.WriteString(",\n") - tools.WriteProtoArrayToBuffer(uidb.ItemIcons, buffer, "itemIcons") - buffer.WriteString(",\n") - tools.WriteProtoArrayToBuffer(uidb.SpellIcons, buffer, "spellIcons") - buffer.WriteString(",\n") - tools.WriteProtoArrayToBuffer(uidb.Encounters, buffer, "encounters") - buffer.WriteString(",\n") - tools.WriteProtoArrayToBuffer(uidb.GlyphIds, buffer, "glyphIds") - buffer.WriteString("\n") - - buffer.WriteString("}") - os.WriteFile(jsonFilePath, buffer.Bytes(), 0666) + builder := &strings.Builder{} + builder.WriteString("{\n") + + tools.WriteProtoArrayToBuilder(uidb.Items, builder, "items") + builder.WriteString(",\n") + tools.WriteProtoArrayToBuilder(uidb.Enchants, builder, "enchants") + builder.WriteString(",\n") + tools.WriteProtoArrayToBuilder(uidb.Gems, builder, "gems") + builder.WriteString(",\n") + tools.WriteProtoArrayToBuilder(uidb.Zones, builder, "zones") + builder.WriteString(",\n") + tools.WriteProtoArrayToBuilder(uidb.Npcs, builder, "npcs") + builder.WriteString(",\n") + tools.WriteProtoArrayToBuilder(uidb.ItemIcons, builder, "itemIcons") + builder.WriteString(",\n") + tools.WriteProtoArrayToBuilder(uidb.SpellIcons, builder, "spellIcons") + builder.WriteString(",\n") + tools.WriteProtoArrayToBuilder(uidb.Encounters, builder, "encounters") + builder.WriteString(",\n") + tools.WriteProtoArrayToBuilder(uidb.GlyphIds, builder, "glyphIds") + builder.WriteString("\n") + + builder.WriteString("}") + os.WriteFile(jsonFilePath, []byte(builder.String()), 0666) } func toSlice(stats Stats) []float64 { diff --git a/tools/database/gen_db/main.go b/tools/database/gen_db/main.go index 61ea8dd210..75bffc5b2e 100644 --- a/tools/database/gen_db/main.go +++ b/tools/database/gen_db/main.go @@ -77,7 +77,7 @@ func main() { for _, response := range itemTooltips { if response.IsEquippable() { // Only included items that are in wowheads gearplanner db - // Wowhead doesn't seem to have a field/flag to signify 'not available / in game' but their gearplanner db has them filtered + // Wowhead doesnt seem to have a field/flag to signify 'not available / in game' but their gearplanner db has them filtered item := response.ToItemProto() if _, ok := wowheadDB.Items[strconv.Itoa(int(item.Id))]; ok { db.MergeItem(item) @@ -289,7 +289,10 @@ func simmableItemFilter(_ int32, item *proto.UIItem) bool { return true } func simmableGemFilter(_ int32, gem *proto.UIGem) bool { - return gem.Quality >= proto.ItemQuality_ItemQualityUncommon + if gem.Quality < proto.ItemQuality_ItemQualityUncommon { + return false + } + return true } type TalentConfig struct { diff --git a/tools/database/wowhead_tooltips.go b/tools/database/wowhead_tooltips.go index 480f82cabc..73df5405bb 100644 --- a/tools/database/wowhead_tooltips.go +++ b/tools/database/wowhead_tooltips.go @@ -161,7 +161,6 @@ var staminaRegex = regexp.MustCompile(`\+([0-9]+) Stamina`) var spellPowerRegex = regexp.MustCompile(`Increases spell power by ([0-9]+)\.`) var spellPowerRegex2 = regexp.MustCompile(`Increases spell power by ([0-9]+)\.`) -/* // Not sure these exist anymore? var arcaneSpellPowerRegex = regexp.MustCompile(`Increases Arcane power by ([0-9]+)\.`) var fireSpellPowerRegex = regexp.MustCompile(`Increases Fire power by ([0-9]+)\.`) @@ -169,7 +168,6 @@ var frostSpellPowerRegex = regexp.MustCompile(`Increases Frost power by ([0-9]+) var holySpellPowerRegex = regexp.MustCompile(`Increases Holy power by ([0-9]+)\.`) var natureSpellPowerRegex = regexp.MustCompile(`Increases Nature power by ([0-9]+)\.`) var shadowSpellPowerRegex = regexp.MustCompile(`Increases Shadow power by ([0-9]+)\.`) -*/ var hitRegex = regexp.MustCompile(`Improves hit rating by ([0-9]+)\.`) var critRegex = regexp.MustCompile(`Improves critical strike rating by ([0-9]+)\.`) diff --git a/tools/io_utils.go b/tools/io_utils.go index f24aff5113..6b142c400b 100644 --- a/tools/io_utils.go +++ b/tools/io_utils.go @@ -4,6 +4,7 @@ package tools import ( "bufio" "bytes" + "encoding/csv" "encoding/json" "flag" "fmt" @@ -11,6 +12,8 @@ import ( "log" "net/http" "os" + "reflect" + "sort" "strconv" "strings" "sync" @@ -25,11 +28,17 @@ import ( var readWebThreads = flag.Int("readWebThreads", 8, "number of parallel workers to fetch web pages") func ReadFile(filePath string) string { - b, err := os.ReadFile(filePath) + bytes, err := os.ReadFile(filePath) if err != nil { log.Fatalf("Failed to open %s: %s", filePath, err) } - return string(b) + return string(bytes) +} +func ReadFileLines(filePath string) []string { + return readFileLinesInternal(filePath, true) +} +func ReadFileLinesOrNil(filePath string) []string { + return readFileLinesInternal(filePath, false) } func readFileLinesInternal(filePath string, throwIfMissing bool) []string { file, err := os.Open(filePath) @@ -51,6 +60,9 @@ func readFileLinesInternal(filePath string, throwIfMissing bool) []string { return lines } +func ReadMap(filePath string) map[string]string { + return readMapInternal(filePath, true) +} func ReadMapOrNil(filePath string) map[string]string { return readMapInternal(filePath, false) } @@ -86,6 +98,19 @@ func WriteFileLines(filePath string, lines []string) { } } +func WriteMap(filePath string, contents map[string]string) { + lines := make([]string, len(contents)) + i := 0 + for k, v := range contents { + lines[i] = fmt.Sprintf("%s,%s", k, v) + i++ + } + + // Sort so the output is stable. + sort.Strings(lines) + + WriteFileLines(filePath, lines) +} func WriteMapSortByIntKey(filePath string, contents map[string]string) { WriteMapCustomSort(filePath, contents, func(a, b string) bool { intA, err1 := strconv.Atoi(a) @@ -124,26 +149,67 @@ func WriteMapCustomSort(filePath string, contents map[string]string, sortFunc fu WriteFileLines(filePath, lines) } -func WriteProtoArrayToBuffer[T googleProto.Message](arr []T, buffer bytes.Buffer, name string) { - buffer.WriteString("\"") - buffer.WriteString(name) - buffer.WriteString("\":[\n") +func ReadCsvFile(filePath string) [][]string { + f, err := os.Open(filePath) + if err != nil { + log.Fatal("Unable to read input file "+filePath, err) + } + defer f.Close() + + csvReader := csv.NewReader(f) + records, err := csvReader.ReadAll() + if err != nil { + log.Fatal("Unable to parse file as CSV for "+filePath, err) + } + + return records +} + +func WriteProtoArrayToBuilder(arrInterface interface{}, builder *strings.Builder, name string) { + arr := InterfaceSlice(arrInterface) + builder.WriteString("\"") + builder.WriteString(name) + builder.WriteString("\":[\n") for i, elem := range arr { - jsonBytes, err := protojson.MarshalOptions{UseEnumNumbers: true}.Marshal(elem) + jsonBytes, err := protojson.MarshalOptions{UseEnumNumbers: true}.Marshal(elem.(googleProto.Message)) if err != nil { log.Printf("[ERROR] Failed to marshal: %s", err.Error()) } // Format using Compact() so we get a stable output (no random diffs for version control). - json.Compact(&buffer, jsonBytes) + var formatted bytes.Buffer + json.Compact(&formatted, jsonBytes) + builder.WriteString(string(formatted.Bytes())) if i != len(arr)-1 { - buffer.WriteString(",") + builder.WriteString(",") } - buffer.WriteString("\n") + builder.WriteString("\n") + } + builder.WriteString("]") +} + +// Needed because Go won't let us cast from []FooProto --> []googleProto.Message +// https://stackoverflow.com/questions/12753805/type-converting-slices-of-interfaces +func InterfaceSlice(slice interface{}) []interface{} { + s := reflect.ValueOf(slice) + if s.Kind() != reflect.Slice { + panic("InterfaceSlice() given a non-slice type") + } + + // Keep the distinction between nil and empty slice input + if s.IsNil() { + return nil } - buffer.WriteString("]") + + ret := make([]interface{}, s.Len()) + + for i := 0; i < s.Len(); i++ { + ret[i] = s.Index(i).Interface() + } + + return ret } // Fetches web results a single url, and returns the page contents as a string.