Merge pull request #3602 from NerdEgghead/master

Revert changes to tools folder that broke DB generation

assets/database/db.json

@@ -12958,6 +12958,7 @@
 {"id":67383,"name":"Unholy Force","icon":"spell_shadow_unholystrength"},
 {"id":67839,"name":"Mind Amplification Dish","icon":"trade_engineering"},
 {"id":68051,"name":"Overpower Ready!","icon":"ability_rogue_hungerforblood"},
+{"id":68055,"name":"Judgements of the Just","icon":"spell_holy_righteousfury"},
 {"id":70164,"name":"Rune of the Nerubian Carapace","icon":"inv_sword_61"},
 {"id":70550,"name":"Leggings of Woven Death","icon":"inv_pants_cloth_34"},
 {"id":70551,"name":"Deathfrost Boots","icon":"inv_boots_cloth_25"},

tools/database/atlasloot.go

@@ -129,7 +129,7 @@ func readAtlasLootDungeonData(db *WowDatabase, expansion proto.Expansion, srcUrl
 				curLocation := 0
 
 				for _, itemMatch := range itemsPattern.FindAllStringSubmatch(difficultyMatch[0], -1) {
-					itemParams := core.MapSlice(strings.Split(itemMatch[1], ","), strings.TrimSpace)
+					itemParams := core.MapSlice(strings.Split(itemMatch[1], ","), func(s string) string { return strings.TrimSpace(s) })
 					location, _ := strconv.Atoi(itemParams[0]) // Location within AtlasLoot's menu.
 
 					idStr := itemParams[1]
@@ -178,8 +178,8 @@ func readAtlasLootDungeonData(db *WowDatabase, expansion proto.Expansion, srcUrl
 }
 
 func readZoneData(db *WowDatabase) {
-	zoneIDs := make([]int32, 0, len(db.Zones))
-	for zoneID := range db.Zones {
+	var zoneIDs []int32
+	for zoneID, _ := range db.Zones {
 		zoneIDs = append(zoneIDs, zoneID)
 	}
 	zoneIDStrs := core.MapSlice(zoneIDs, func(zoneID int32) string { return strconv.Itoa(int(zoneID)) })

tools/database/blizzard_api.go

@@ -123,7 +123,7 @@ func getItemData(itemId int, accessToken string) BlizzardItemResponse {
 		log.Fatal(err)
 	}
 
-	fmt.Print(string(resultBody))
+	fmt.Printf(string(resultBody))
 	itemResponse := BlizzardItemResponse{}
 	err = json.Unmarshal(resultBody, &itemResponse)
 	if err != nil {

tools/database/database.go

@@ -1,16 +1,16 @@
 package database
 
 import (
-	"bytes"
 	"fmt"
+	"log"
+	"os"
+	"strings"
+
 	"github.com/wowsims/wotlk/sim/core/proto"
 	"github.com/wowsims/wotlk/tools"
-	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
 	"google.golang.org/protobuf/encoding/protojson"
 	googleProto "google.golang.org/protobuf/proto"
-	"log"
-	"os"
 )
 
 type EnchantDBKey struct {
@@ -56,17 +56,31 @@ func NewWowDatabase() *WowDatabase {
 }
 
 func (db *WowDatabase) Clone() *WowDatabase {
-	return &WowDatabase{
-		Items:      maps.Clone(db.Items),
-		Enchants:   maps.Clone(db.Enchants),
-		Gems:       maps.Clone(db.Gems),
-		Zones:      maps.Clone(db.Zones),
-		Npcs:       maps.Clone(db.Npcs),
-		ItemIcons:  maps.Clone(db.ItemIcons),
-		SpellIcons: maps.Clone(db.SpellIcons),
-		Encounters: slices.Clone(db.Encounters),
-		GlyphIDs:   slices.Clone(db.GlyphIDs),
+	other := NewWowDatabase()
+
+	for k, v := range db.Items {
+		other.Items[k] = v
+	}
+	for k, v := range db.Enchants {
+		other.Enchants[k] = v
+	}
+	for k, v := range db.Gems {
+		other.Gems[k] = v
+	}
+	for k, v := range db.Zones {
+		other.Zones[k] = v
+	}
+	for k, v := range db.Npcs {
+		other.Npcs[k] = v
+	}
+	for k, v := range db.ItemIcons {
+		other.ItemIcons[k] = v
 	}
+	for k, v := range db.SpellIcons {
+		other.SpellIcons[k] = v
+	}
+
+	return other
 }
 
 func (db *WowDatabase) MergeItems(arr []*proto.UIItem) {
@@ -76,7 +90,7 @@ func (db *WowDatabase) MergeItems(arr []*proto.UIItem) {
 }
 func (db *WowDatabase) MergeItem(src *proto.UIItem) {
 	if dst, ok := db.Items[src.Id]; ok {
-		// googleproto.Merge concatenates lists, but we want replacement, so do them manually.
+		// googleproto.Merge concatenates lists but we want replacement, so do them manually.
 		if src.Stats != nil {
 			dst.Stats = src.Stats
 			src.Stats = nil
@@ -99,7 +113,7 @@ func (db *WowDatabase) MergeEnchants(arr []*proto.UIEnchant) {
 func (db *WowDatabase) MergeEnchant(src *proto.UIEnchant) {
 	key := EnchantToDBKey(src)
 	if dst, ok := db.Enchants[key]; ok {
-		// googleproto.Merge concatenates lists, but we want replacement, so do them manually.
+		// googleproto.Merge concatenates lists but we want replacement, so do them manually.
 		if src.Stats != nil {
 			dst.Stats = src.Stats
 			src.Stats = nil
@@ -117,7 +131,7 @@ func (db *WowDatabase) MergeGems(arr []*proto.UIGem) {
 }
 func (db *WowDatabase) MergeGem(src *proto.UIGem) {
 	if dst, ok := db.Gems[src.Id]; ok {
-		// googleproto.Merge concatenates lists, but we want replacement, so do them manually.
+		// googleproto.Merge concatenates lists but we want replacement, so do them manually.
 		if src.Stats != nil {
 			dst.Stats = src.Stats
 			src.Stats = nil
@@ -176,49 +190,71 @@ func (db *WowDatabase) AddSpellIcon(id int32, tooltips map[int32]WowheadItemResp
 	}
 }
 
-type idKeyed interface {
-	GetId() int32
-}
-
-func mapToSlice[T idKeyed](m map[int32]T) []T {
-	vs := make([]T, 0, len(m))
-	for _, v := range m {
-		vs = append(vs, v)
+func (db *WowDatabase) MergeUIProto(dbProto *proto.UIDatabase) {
+	db.MergeItems(dbProto.Items)
+	db.MergeEnchants(dbProto.Enchants)
+	db.MergeGems(dbProto.Gems)
+	db.MergeZones(dbProto.Zones)
+	db.MergeNpcs(dbProto.Npcs)
+	for _, icon := range dbProto.ItemIcons {
+		db.ItemIcons[icon.Id] = icon
+	}
+	for _, icon := range dbProto.SpellIcons {
+		db.SpellIcons[icon.Id] = icon
 	}
-	slices.SortFunc(vs, func(a, b T) bool {
-		return a.GetId() < b.GetId()
-	})
-	return vs
 }
 
 func (db *WowDatabase) ToUIProto() *proto.UIDatabase {
-	enchants := make([]*proto.UIEnchant, 0, len(db.Enchants))
-	for _, v := range db.Enchants {
-		enchants = append(enchants, v)
-	}
-	slices.SortFunc(enchants, func(v1, v2 *proto.UIEnchant) bool {
-		return v1.EffectId < v2.EffectId || v1.EffectId == v2.EffectId && v1.Type < v2.Type
-	})
-
-	return &proto.UIDatabase{
-		Items:      mapToSlice(db.Items),
-		Enchants:   enchants,
-		Gems:       mapToSlice(db.Gems),
+	uidb := &proto.UIDatabase{
 		Encounters: db.Encounters,
-		Zones:      mapToSlice(db.Zones),
-		Npcs:       mapToSlice(db.Npcs),
-		ItemIcons:  mapToSlice(db.ItemIcons),
-		SpellIcons: mapToSlice(db.SpellIcons),
 		GlyphIds:   db.GlyphIDs,
 	}
-}
 
-func sliceToMap[T idKeyed](vs []T) map[int32]T {
-	m := make(map[int32]T, len(vs))
-	for _, v := range vs {
-		m[v.GetId()] = v
+	for _, v := range db.Items {
+		uidb.Items = append(uidb.Items, v)
+	}
+	for _, v := range db.Enchants {
+		uidb.Enchants = append(uidb.Enchants, v)
+	}
+	for _, v := range db.Gems {
+		uidb.Gems = append(uidb.Gems, v)
+	}
+	for _, v := range db.Zones {
+		uidb.Zones = append(uidb.Zones, v)
+	}
+	for _, v := range db.Npcs {
+		uidb.Npcs = append(uidb.Npcs, v)
 	}
-	return m
+	for _, v := range db.ItemIcons {
+		uidb.ItemIcons = append(uidb.ItemIcons, v)
+	}
+	for _, v := range db.SpellIcons {
+		uidb.SpellIcons = append(uidb.SpellIcons, v)
+	}
+
+	slices.SortStableFunc(uidb.Items, func(v1, v2 *proto.UIItem) bool {
+		return v1.Id < v2.Id
+	})
+	slices.SortStableFunc(uidb.Enchants, func(v1, v2 *proto.UIEnchant) bool {
+		return v1.EffectId < v2.EffectId || v1.EffectId == v2.EffectId && v1.Type < v2.Type
+	})
+	slices.SortStableFunc(uidb.Gems, func(v1, v2 *proto.UIGem) bool {
+		return v1.Id < v2.Id
+	})
+	slices.SortStableFunc(uidb.Zones, func(v1, v2 *proto.UIZone) bool {
+		return v1.Id < v2.Id
+	})
+	slices.SortStableFunc(uidb.Npcs, func(v1, v2 *proto.UINPC) bool {
+		return v1.Id < v2.Id
+	})
+	slices.SortStableFunc(uidb.ItemIcons, func(v1, v2 *proto.IconData) bool {
+		return v1.Id < v2.Id
+	})
+	slices.SortStableFunc(uidb.SpellIcons, func(v1, v2 *proto.IconData) bool {
+		return v1.Id < v2.Id
+	})
+
+	return uidb
 }
 
 func ReadDatabaseFromJson(jsonStr string) *WowDatabase {
@@ -227,22 +263,9 @@ func ReadDatabaseFromJson(jsonStr string) *WowDatabase {
 		panic(err)
 	}
 
-	enchants := make(map[EnchantDBKey]*proto.UIEnchant, len(dbProto.Enchants))
-	for _, v := range dbProto.Enchants {
-		enchants[EnchantToDBKey(v)] = v
-	}
-
-	return &WowDatabase{
-		Items:      sliceToMap(dbProto.Items),
-		Enchants:   enchants,
-		Gems:       sliceToMap(dbProto.Gems),
-		Zones:      sliceToMap(dbProto.Zones),
-		Npcs:       sliceToMap(dbProto.Npcs),
-		ItemIcons:  sliceToMap(dbProto.ItemIcons),
-		SpellIcons: sliceToMap(dbProto.SpellIcons),
-		Encounters: dbProto.Encounters,
-		GlyphIDs:   dbProto.GlyphIds,
-	}
+	db := NewWowDatabase()
+	db.MergeUIProto(dbProto)
+	return db
 }
 
 func (db *WowDatabase) WriteBinaryAndJson(binFilePath, jsonFilePath string) {
@@ -265,31 +288,30 @@ func (db *WowDatabase) WriteJson(jsonFilePath string) {
 	// Also write in JSON format so we can manually inspect the contents.
 	// Write it out line-by-line so we can have 1 line / item, making it more human-readable.
 	uidb := db.ToUIProto()
-
-	var buffer bytes.Buffer
-	buffer.WriteString("{\n")
-
-	tools.WriteProtoArrayToBuffer(uidb.Items, buffer, "items")
-	buffer.WriteString(",\n")
-	tools.WriteProtoArrayToBuffer(uidb.Enchants, buffer, "enchants")
-	buffer.WriteString(",\n")
-	tools.WriteProtoArrayToBuffer(uidb.Gems, buffer, "gems")
-	buffer.WriteString(",\n")
-	tools.WriteProtoArrayToBuffer(uidb.Zones, buffer, "zones")
-	buffer.WriteString(",\n")
-	tools.WriteProtoArrayToBuffer(uidb.Npcs, buffer, "npcs")
-	buffer.WriteString(",\n")
-	tools.WriteProtoArrayToBuffer(uidb.ItemIcons, buffer, "itemIcons")
-	buffer.WriteString(",\n")
-	tools.WriteProtoArrayToBuffer(uidb.SpellIcons, buffer, "spellIcons")
-	buffer.WriteString(",\n")
-	tools.WriteProtoArrayToBuffer(uidb.Encounters, buffer, "encounters")
-	buffer.WriteString(",\n")
-	tools.WriteProtoArrayToBuffer(uidb.GlyphIds, buffer, "glyphIds")
-	buffer.WriteString("\n")
-
-	buffer.WriteString("}")
-	os.WriteFile(jsonFilePath, buffer.Bytes(), 0666)
+	builder := &strings.Builder{}
+	builder.WriteString("{\n")
+
+	tools.WriteProtoArrayToBuilder(uidb.Items, builder, "items")
+	builder.WriteString(",\n")
+	tools.WriteProtoArrayToBuilder(uidb.Enchants, builder, "enchants")
+	builder.WriteString(",\n")
+	tools.WriteProtoArrayToBuilder(uidb.Gems, builder, "gems")
+	builder.WriteString(",\n")
+	tools.WriteProtoArrayToBuilder(uidb.Zones, builder, "zones")
+	builder.WriteString(",\n")
+	tools.WriteProtoArrayToBuilder(uidb.Npcs, builder, "npcs")
+	builder.WriteString(",\n")
+	tools.WriteProtoArrayToBuilder(uidb.ItemIcons, builder, "itemIcons")
+	builder.WriteString(",\n")
+	tools.WriteProtoArrayToBuilder(uidb.SpellIcons, builder, "spellIcons")
+	builder.WriteString(",\n")
+	tools.WriteProtoArrayToBuilder(uidb.Encounters, builder, "encounters")
+	builder.WriteString(",\n")
+	tools.WriteProtoArrayToBuilder(uidb.GlyphIds, builder, "glyphIds")
+	builder.WriteString("\n")
+
+	builder.WriteString("}")
+	os.WriteFile(jsonFilePath, []byte(builder.String()), 0666)
 }
 
 func toSlice(stats Stats) []float64 {

tools/database/gen_db/main.go

@@ -77,7 +77,7 @@ func main() {
 	for _, response := range itemTooltips {
 		if response.IsEquippable() {
 			// Only included items that are in wowheads gearplanner db
-			// Wowhead doesn't seem to have a field/flag to signify 'not available / in game' but their gearplanner db has them filtered
+			// Wowhead doesnt seem to have a field/flag to signify 'not available / in game' but their gearplanner db has them filtered
 			item := response.ToItemProto()
 			if _, ok := wowheadDB.Items[strconv.Itoa(int(item.Id))]; ok {
 				db.MergeItem(item)
@@ -289,7 +289,10 @@ func simmableItemFilter(_ int32, item *proto.UIItem) bool {
 	return true
 }
 func simmableGemFilter(_ int32, gem *proto.UIGem) bool {
-	return gem.Quality >= proto.ItemQuality_ItemQualityUncommon
+	if gem.Quality < proto.ItemQuality_ItemQualityUncommon {
+		return false
+	}
+	return true
 }
 
 type TalentConfig struct {

tools/database/wowhead_tooltips.go

@@ -161,15 +161,13 @@ var staminaRegex = regexp.MustCompile(`<!--stat7-->\+([0-9]+) Stamina`)
 var spellPowerRegex = regexp.MustCompile(`Increases spell power by ([0-9]+)\.`)
 var spellPowerRegex2 = regexp.MustCompile(`Increases spell power by <!--rtg45-->([0-9]+)\.`)
 
-/*
 // Not sure these exist anymore?
 var arcaneSpellPowerRegex = regexp.MustCompile(`Increases Arcane power by ([0-9]+)\.`)
 var fireSpellPowerRegex = regexp.MustCompile(`Increases Fire power by ([0-9]+)\.`)
 var frostSpellPowerRegex = regexp.MustCompile(`Increases Frost power by ([0-9]+)\.`)
 var holySpellPowerRegex = regexp.MustCompile(`Increases Holy power by ([0-9]+)\.`)
 var natureSpellPowerRegex = regexp.MustCompile(`Increases Nature power by ([0-9]+)\.`)
 var shadowSpellPowerRegex = regexp.MustCompile(`Increases Shadow power by ([0-9]+)\.`)
-*/
 
 var hitRegex = regexp.MustCompile(`Improves hit rating by <!--rtg31-->([0-9]+)\.`)
 var critRegex = regexp.MustCompile(`Improves critical strike rating by <!--rtg32-->([0-9]+)\.`)