diff --git a/apps/frontend/src/app/Components/Character/CharacterCard.tsx b/apps/frontend/src/app/Components/Character/CharacterCard.tsx index 4cbe3405..96490858 100644 --- a/apps/frontend/src/app/Components/Character/CharacterCard.tsx +++ b/apps/frontend/src/app/Components/Character/CharacterCard.tsx @@ -57,6 +57,7 @@ type CharacterCardProps = { characterChildren?: Displayable footer?: Displayable hideStats?: boolean + hideArtifacts?: boolean isTeammateCard?: boolean } export default function CharacterCard({ @@ -69,6 +70,7 @@ export default function CharacterCard({ onClickTeammate, footer, hideStats, + hideArtifacts, isTeammateCard, }: CharacterCardProps) { const { database } = useContext(DatabaseContext) @@ -157,6 +159,7 @@ export default function CharacterCard({ character={character} onClickTeammate={onClickTeammate} hideStats={hideStats} + hideArtifacts={hideArtifacts} weaponChildren={weaponChildren} artifactChildren={artifactChildren} characterChildren={characterChildren} @@ -181,6 +184,7 @@ type ExistingCharacterCardContentProps = { character: ICachedCharacter onClickTeammate?: (characterKey: CharacterKey) => void hideStats?: boolean + hideArtifacts?: boolean weaponChildren?: Displayable artifactChildren?: Displayable characterChildren?: Displayable @@ -195,6 +199,7 @@ function ExistingCharacterCardContent({ character, onClickTeammate, hideStats, + hideArtifacts, weaponChildren, artifactChildren, characterChildren, @@ -218,7 +223,7 @@ function ExistingCharacterCardContent({ padding: hideStats ? `${theme.spacing(1)}!important` : undefined, })} > - + {!hideArtifacts && } {!isTeammateCard && ( diff --git a/apps/frontend/src/app/PageArtifact/ScanningUtil.tsx b/apps/frontend/src/app/PageArtifact/ScanningUtil.tsx index dc8ad3cb..a81ccab1 100644 --- a/apps/frontend/src/app/PageArtifact/ScanningUtil.tsx +++ b/apps/frontend/src/app/PageArtifact/ScanningUtil.tsx @@ -3,7 +3,7 @@ import { BorrowManager } from '@genshin-optimizer/util' import type { RecognizeResult, Scheduler } from 'tesseract.js' import { createScheduler, createWorker } from 'tesseract.js' -const workerCount = 2 +const workerCount = 20 const schedulers = new BorrowManager( async (language): Promise => { @@ -32,7 +32,7 @@ export async function textsFromImage( ): Promise { const canvas = imageDataToCanvas(imageData) const rec = await schedulers.borrow( - 'genshin_fast_09_04_21', + 'eng', async (scheduler) => (await ( await scheduler diff --git a/apps/frontend/src/app/PageCharacter/CharacterDisplay/Tabs/TabOptimize/index.tsx b/apps/frontend/src/app/PageCharacter/CharacterDisplay/Tabs/TabOptimize/index.tsx index d8aacddf..7c3e50e8 100644 --- a/apps/frontend/src/app/PageCharacter/CharacterDisplay/Tabs/TabOptimize/index.tsx +++ b/apps/frontend/src/app/PageCharacter/CharacterDisplay/Tabs/TabOptimize/index.tsx @@ -530,21 +530,11 @@ export default function TabBuild() { - - - {/* 2 */} - {/* Level Filter */} @@ -567,7 +557,37 @@ export default function TabBuild() { /> + + + buildSettingDispatch({ mainStatAssumptionLevel })} + disabled={generatingBuilds} + /> + + {t`mainStat.levelAssTooltip.title`} + {t`mainStat.levelAssTooltip.desc`} + + } + /> + + + + {/* 2 */} + {/* Main Stat Filters */} @@ -576,25 +596,6 @@ export default function TabBuild() { >{t`mainStat.title`} - - - buildSettingDispatch({ mainStatAssumptionLevel })} - disabled={generatingBuilds} - /> - - {t`mainStat.levelAssTooltip.title`} - {t`mainStat.levelAssTooltip.desc`} - - } - /> - - {/* main stat selector */} +
{/* Footer */} {isSM && targetSelector} diff --git a/apps/frontend/src/app/PageHome/index.tsx b/apps/frontend/src/app/PageHome/index.tsx index e638450e..5dbd45d7 100644 --- a/apps/frontend/src/app/PageHome/index.tsx +++ b/apps/frontend/src/app/PageHome/index.tsx @@ -8,7 +8,6 @@ import { useTheme, } from '@mui/material' import ReactGA from 'react-ga4' -import { Trans, useTranslation } from 'react-i18next' import CardDark from '../Components/Card/CardDark' import InventoryCard from './InventoryCard' diff --git a/apps/frontend/src/assets/eng.traineddata.gz b/apps/frontend/src/assets/eng.traineddata.gz new file mode 100644 index 00000000..32aeef67 Binary files /dev/null and b/apps/frontend/src/assets/eng.traineddata.gz differ diff --git a/libs/gi-art-scanner/src/lib/artifactBoxPredictor.ts b/libs/gi-art-scanner/src/lib/artifactBoxPredictor.ts new file mode 100644 index 00000000..30887f0a --- /dev/null +++ b/libs/gi-art-scanner/src/lib/artifactBoxPredictor.ts @@ -0,0 +1,131 @@ +import { + convertToBlackAndWhite, + edgeDetection, + imageDataToCanvas, +} from '@genshin-optimizer/img-util' + +interface Point { + x: number + y: number +} +interface Rectangle { + topLeft: Point + bottomRight: Point +} +type artifactPredictorResult = { + artifactImageData: ImageData + debugImgs: Record +} + +export function artifactBoxPredictor( + imageData: ImageData +): artifactPredictorResult { + const debugImgs = {} as Record + + imageData = + imageData.width > imageData.height + ? boxPredictor(imageData, debugImgs) + : imageData + + return { + artifactImageData: imageData, + debugImgs: debugImgs, + } +} + +function boxPredictor( + imageData: ImageData, + debugImgs: Record +): ImageData { + const edgeDetectedImageData = edgeDetection(imageData) + + debugImgs['Edge Detection Full Screen'] = imageDataToCanvas( + edgeDetectedImageData + ).toDataURL() + + const bwEdgeData = convertToBlackAndWhite( + new ImageData( + new Uint8ClampedArray(edgeDetectedImageData.data), + edgeDetectedImageData.width, + edgeDetectedImageData.height + ) + ) + + const { topLeft, bottomRight } = findLargestRectangle(bwEdgeData) + + const canvas = document.createElement('canvas') + const context = canvas.getContext('2d')! + const width = bottomRight.x - topLeft.x + 1 + const height = bottomRight.y - topLeft.y + 1 + + canvas.width = width + canvas.height = height + + context.drawImage( + imageDataToCanvas(imageData), + topLeft.x, + topLeft.y, + width, + height, + 0, + 0, + width, + height + ) + + const largestRectangleImageData = context.getImageData(0, 0, width, height) + debugImgs['Largest Rectangle'] = canvas.toDataURL() + + return largestRectangleImageData +} + +function findLargestRectangle(imageData: ImageData): Rectangle { + const width = imageData.width + const height = imageData.height + const data = imageData.data + + let maxArea = 0 + let maxRectangle: Rectangle = { + topLeft: { x: 0, y: 0 }, + bottomRight: { x: 0, y: 0 }, + } + + function isWhite(x: number, y: number): boolean { + const pixelIndex = (y * width + x) * 4 + return data[pixelIndex] === 255 + } + + function expandFromPoint(x: number, y: number): Rectangle { + let left = x + let right = x + let top = y + let bottom = y + + while (left > 0 && isWhite(left - 1, y)) left-- + while (right < width - 1 && isWhite(right + 1, y)) right++ + while (top > 0 && isWhite(x, top - 1)) top-- + while (bottom < height - 1 && isWhite(x, bottom + 1)) bottom++ + + return { + topLeft: { x: left, y: top }, + bottomRight: { x: right, y: bottom }, + } + } + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + if (isWhite(x, y)) { + const rectangle = expandFromPoint(x, y) + const area = + (rectangle.bottomRight.x - rectangle.topLeft.x + 1) * + (rectangle.bottomRight.y - rectangle.topLeft.y + 1) + if (area > maxArea) { + maxArea = area + maxRectangle = rectangle + } + } + } + } + + return maxRectangle +} diff --git a/libs/gi-art-scanner/src/lib/artifactPredictor.ts b/libs/gi-art-scanner/src/lib/artifactPredictor.ts new file mode 100644 index 00000000..b3e558aa --- /dev/null +++ b/libs/gi-art-scanner/src/lib/artifactPredictor.ts @@ -0,0 +1,310 @@ +import { + convertToBlackAndWhite, + crop, + edgeDetection, + extractBox, + findSplitHeight, + imageDataToCanvas, + scaleImage, + splitImageVertical, +} from '@genshin-optimizer/img-util' +import { artifactBoxPredictor } from './artifactBoxPredictor' + +type artifactPredictorResult = { + prediction: any + debugImgs: Record + artifactImageData: ImageData +} + +// const artifactAspectRatio = 1.73 +const artifactNameHeaderRatio = 0.2 + +export async function artifactPredictor( + imageData: ImageData, + textsFromImage: ( + imageData: ImageData, + options?: object | undefined + ) => Promise +): Promise { + const { artifactImageData, debugImgs } = artifactBoxPredictor(imageData) + + const edgeDetectedImageData = edgeDetection( + convertToBlackAndWhite( + new ImageData( + new Uint8ClampedArray(artifactImageData.data), + artifactImageData.width, + artifactImageData.height + ), + false, + 200 + ) + ) + const splitHeight = findSplitHeight(edgeDetectedImageData) + const [headerCard, whiteCard] = splitImageVertical( + artifactImageData, + splitHeight + ) + const [ArtifactNameCard, ArtifactMainStatCard] = splitImageVertical( + headerCard, + artifactNameHeaderRatio * headerCard.height + ) + const [ArtifactSubstats, ArtifactSetLocation] = splitImageVertical( + whiteCard, + ArtifactMainStatCard.height + ) + const [ArtifactSet, ArtifactLocation] = splitImageVertical( + ArtifactSetLocation, + ArtifactSetLocation.height - ArtifactNameCard.height + ) + + debugImgs['Edge Detection'] = imageDataToCanvas( + edgeDetectedImageData + ).toDataURL() + debugImgs['Header Card'] = imageDataToCanvas(headerCard).toDataURL() + debugImgs['White Card'] = imageDataToCanvas(whiteCard).toDataURL() + debugImgs['Artifact Name Card'] = + imageDataToCanvas(ArtifactNameCard).toDataURL() + debugImgs['Artifact Main Stat Card'] = + imageDataToCanvas(ArtifactMainStatCard).toDataURL() + debugImgs['Artifact Substats'] = + imageDataToCanvas(ArtifactSubstats).toDataURL() + debugImgs['Artifact Set & Location'] = + imageDataToCanvas(ArtifactSetLocation).toDataURL() + debugImgs['Artifact Set'] = imageDataToCanvas(ArtifactSet).toDataURL() + debugImgs['Artifact Location'] = + imageDataToCanvas(ArtifactLocation).toDataURL() + + // Data about each part of the Artifact + const ArtifactDetections = [ + { + name: 'ArtifactName', + start: 0, + end: 1.0, + crop: 1.0, + ocr: true, + invert: true, + bw: true, + cropRight: false, + threshold: 160, + scale: false, + scaleFactor: 1, + extractBox: true, + extractColor: 'black', + padding: 10, + image: ArtifactNameCard, + }, + { + name: 'ArtifactSlot', + start: 0, + end: 0.2, + crop: 0.8, + ocr: true, + invert: true, + bw: true, + cropRight: false, + threshold: 160, + scale: false, + scaleFactor: 1, + extractBox: true, + extractColor: 'black', + padding: 10, + image: ArtifactMainStatCard, + }, + { + name: 'ArtifactMainStat', + start: 0.4, + end: 0.55, + crop: 0.5, + ocr: true, + invert: true, + bw: true, + cropRight: false, + threshold: 150, + scale: true, + scaleFactor: 2, + extractBox: true, + extractColor: 'black', + padding: 10, + image: ArtifactMainStatCard, + }, + { + name: 'ArtifactMainStatValue', + start: 0.525, + end: 0.775, + crop: 0.5, + ocr: true, + invert: true, + bw: true, + cropRight: false, + threshold: 160, + scale: true, + scaleFactor: 2, + extractBox: true, + extractColor: 'black', + padding: 10, + image: ArtifactMainStatCard, + }, + { + name: 'ArtifactRarity', + start: 0.775, + end: 0.95, + crop: 0.4, + ocr: false, + invert: true, + bw: false, + cropRight: false, + threshold: 128, + scale: false, + scaleFactor: 1, + extractBox: false, + extractColor: 'white', + padding: 10, + image: ArtifactMainStatCard, + }, + { + name: 'ArtifactLevel', + start: 0.03, + end: 0.275, + crop: 0.2, + ocr: true, + invert: true, + bw: true, + cropRight: false, + threshold: 128, + scale: true, + scaleFactor: 2, + extractBox: true, + extractColor: 'white', + padding: 0, + image: ArtifactSubstats, + }, + { + name: 'ArtifactLock', + start: 0.03, + end: 0.275, + crop: 0.8, + ocr: false, + invert: true, + bw: false, + cropRight: true, + threshold: 128, + scale: false, + scaleFactor: 1, + extractBox: false, + extractColor: 'white', + padding: 0, + image: ArtifactSubstats, + }, + { + name: 'ArtifactSubstats', + start: 0.275, + end: 1.0, + crop: 1.0, + ocr: true, + invert: false, + bw: true, + cropRight: false, + threshold: 160, + scale: false, + scaleFactor: 1, + extractBox: true, + extractColor: 'black', + padding: 10, + image: ArtifactSubstats, + }, + { + name: 'ArtifactSet', + start: 0, + end: 1.0, + crop: 1.0, + ocr: true, + invert: false, + bw: true, + cropRight: false, + threshold: 160, + scale: false, + scaleFactor: 1, + extractBox: true, + extractColor: 'black', + padding: 10, + image: ArtifactSet, + }, + { + name: 'ArtifactLocation', + start: 0, + end: 1, + crop: 1.0, + ocr: true, + invert: false, + bw: true, + cropRight: false, + threshold: 160, + scale: false, + scaleFactor: 1, + extractBox: true, + extractColor: 'black', + padding: 10, + image: ArtifactLocation, + }, + ] + + // Processing Pipeline + const imageSegments = ArtifactDetections.map((segment, index) => { + const totalHeight = segment.image.height + const totalWidth = segment.image.width + const res = crop(imageDataToCanvas(segment.image), { + x1: segment.cropRight ? Math.floor(segment.crop * totalWidth) : 5, + x2: segment.cropRight + ? totalWidth + : Math.floor(segment.crop * totalWidth), + y1: Math.floor(segment.start * totalHeight), + y2: Math.floor(segment.end * totalHeight), + }) + const bwRes = segment.bw + ? convertToBlackAndWhite(res, segment.invert, segment.threshold) + : res + const extractedBoxRes = segment.extractBox + ? extractBox( + bwRes, + segment.extractColor as 'white' | 'black', + segment.padding + ) + : bwRes + const scaledRes = segment.scale + ? scaleImage(extractedBoxRes, segment.scaleFactor) + : extractedBoxRes + debugImgs[ArtifactDetections[index].name] = + imageDataToCanvas(scaledRes).toDataURL() + return { + name: segment.name, + textPromise: segment.ocr + ? textsFromImage(scaledRes) + : Promise.resolve(['']), + } + }) + + // OCR + const segmentTexts = await Promise.all( + imageSegments.map(async (segment) => { + const textArray = await segment.textPromise + const cleanedTextArray = textArray.map((text) => { + return text.replace(/\n/g, '') + }) + return { + name: segment.name, + text: cleanedTextArray.length ? cleanedTextArray : [''], + } + }) + ) + + const res: { [key: string]: string[] } = {} + segmentTexts.forEach((segment) => { + res[segment.name] = segment.text + }) + + return { + prediction: res, + debugImgs: debugImgs, + artifactImageData: artifactImageData, + } +} diff --git a/libs/gi-art-scanner/src/lib/processImg.ts b/libs/gi-art-scanner/src/lib/processImg.ts index f4ec2f9c..85a8b291 100644 --- a/libs/gi-art-scanner/src/lib/processImg.ts +++ b/libs/gi-art-scanner/src/lib/processImg.ts @@ -1,14 +1,9 @@ import type { IArtifact } from '@genshin-optimizer/gi-good' import { clamp } from '@genshin-optimizer/util' import type { ReactNode } from 'react' - -import type { Color } from '@genshin-optimizer/img-util' import { - bandPass, crop, darkerColor, - drawHistogram, - drawline, fileToURL, findHistogramRange, histogramAnalysis, @@ -17,21 +12,7 @@ import { lighterColor, urlToImageData, } from '@genshin-optimizer/img-util' -import { - blueTitleDarkerColor, - blueTitleLighterColor, - cardWhite, - equipColor, - goldenTitleDarkerColor, - goldenTitleLighterColor, - greenTextColor, - lockColor, - purpleTitleDarkerColor, - purpleTitleLighterColor, - starColor, - textColorDark, - textColorLight, -} from './consts' +import { equipColor, lockColor, starColor } from './consts' import type { TextKey } from './findBestArtifact' import { findBestArtifact } from './findBestArtifact' import { @@ -42,6 +23,7 @@ import { parseSlotKeys, parseSubstats, } from './parse' +import { artifactPredictor } from './artifactPredictor' export type Processed = { fileName: string @@ -61,284 +43,61 @@ export async function processEntry( imageData: ImageData, options?: object | undefined ) => Promise, - debug = false + debug = true ): Promise { const { f, fName } = entry const imageURL = await fileToURL(f) const imageData = await urlToImageData(imageURL) - const debugImgs = debug ? ({} as Record) : undefined - const artifactCardImageData = verticallyCropArtifactCard(imageData, debugImgs) - const artifactCardCanvas = imageDataToCanvas(artifactCardImageData) - - const titleHistogram = findTitle(artifactCardImageData) - const [titleTop, titleBot] = titleHistogram - ? findHistogramRange(titleHistogram, 0.7, 1) // smaller threshold - : [0, 0] - - const whiteCardHistogram = histogramContAnalysis( - artifactCardImageData, - darkerColor(cardWhite), - lighterColor(cardWhite), - false - ) - const [whiteCardTop, whiteCardBotOri] = findHistogramRange( - whiteCardHistogram, - 0.8, - 2 + const { prediction, debugImgs, artifactImageData } = await artifactPredictor( + imageData, + textsFromImage ) - let whiteCardBot = whiteCardBotOri + + if (debug) console.log(prediction) const equipHistogram = histogramContAnalysis( - imageData, + artifactImageData, darkerColor(equipColor), lighterColor(equipColor), false ) - - const hasEquip = equipHistogram.some( - (i) => i > artifactCardImageData.width * 0.5 - ) - const [equipTop, equipBot] = findHistogramRange(equipHistogram) - - if (hasEquip) { - whiteCardBot = equipBot - } else { - // try to match green text. - // this value is not used because it can be noisy due to possible card background. - - const greentextHisto = histogramAnalysis( - artifactCardImageData, - darkerColor(greenTextColor), - lighterColor(greenTextColor), - false - ) - - const [greenTextTop, greenTextBot] = findHistogramRange(greentextHisto, 0.2) - const greenTextBuffer = greenTextBot - greenTextTop - if (greenTextBot > whiteCardBot) - whiteCardBot = clamp( - greenTextBot + greenTextBuffer, - 0, - artifactCardImageData.height - ) - } - - const artifactCardCropped = crop(artifactCardCanvas, { - y1: titleTop, - y2: whiteCardBot, - }) - - const equippedCropped = hasEquip - ? crop(artifactCardCanvas, { - y1: equipTop, - y2: equipBot, - }) - : undefined - /** - * Technically this is a way to get both the set+slot - */ - // const goldenTitleCropped = cropHorizontal( - // artifactCardCanvas, - // titleTop, - // titleBot - // ) - - // if (debug) - // debugImgs['goldenTitlecropped'] = - // imageDataToCanvas(goldenTitleCropped).toDataURL() - - const headerCropped = crop(artifactCardCanvas, { - // crop out the right 40% of the header, to reduce noise from the artifact image - x1: 0, - x2: artifactCardCanvas.width * 0.6, - y1: titleBot, - y2: whiteCardTop, - }) - - if (debugImgs) { - const canvas = imageDataToCanvas(artifactCardImageData) - titleHistogram && - drawHistogram( - canvas, - titleHistogram, - { - r: 0, - g: 150, - b: 150, - a: 100, - }, - false - ) - - drawHistogram( - canvas, - whiteCardHistogram, - { r: 150, g: 0, b: 0, a: 100 }, - false - ) - drawHistogram(canvas, equipHistogram, { r: 0, g: 0, b: 100, a: 100 }, false) - - drawline(canvas, titleTop, { r: 0, g: 255, b: 0, a: 200 }, false) - drawline( - canvas, - hasEquip ? equipBot : whiteCardBot, - { r: 0, g: 0, b: 255, a: 200 }, - false + const equipped = + equipHistogram.some((i) => i > artifactImageData.width * 0.5) || + prediction.ArtifactLocation.some((item: string) => + item.toLowerCase().includes('equipped') ) - drawline(canvas, whiteCardTop, { r: 255, g: 0, b: 200, a: 200 }, false) - - debugImgs['artifactCardAnalysis'] = canvas.toDataURL() - } - - if (debugImgs) - debugImgs['headerCropped'] = imageDataToCanvas(headerCropped).toDataURL() - - const whiteCardCropped = crop(artifactCardCanvas, { - y1: whiteCardTop, - y2: whiteCardBot, - }) - - const greentextHisto = histogramAnalysis( - whiteCardCropped, - darkerColor(greenTextColor), - lighterColor(greenTextColor), - false - ) - const [greenTextTop, greenTextBot] = findHistogramRange(greentextHisto, 0.2) - - if (debugImgs) { - const canvas = imageDataToCanvas(whiteCardCropped) - drawHistogram(canvas, greentextHisto, { r: 100, g: 0, b: 0, a: 100 }, false) - drawline(canvas, greenTextTop, { r: 0, g: 255, b: 0, a: 200 }, false) - drawline(canvas, greenTextBot, { r: 0, g: 0, b: 255, a: 200 }, false) - debugImgs['whiteCardAnalysis'] = canvas.toDataURL() - } - - const greenTextBuffer = greenTextBot - greenTextTop - - const greenTextCropped = crop(imageDataToCanvas(whiteCardCropped), { - y1: greenTextTop - greenTextBuffer, - y2: greenTextBot + greenTextBuffer, - }) - - const substatsCardCropped = crop(imageDataToCanvas(whiteCardCropped), { - y2: greenTextTop, - }) const lockHisto = histogramAnalysis( - whiteCardCropped, + await urlToImageData(debugImgs['ArtifactLock']), darkerColor(lockColor), lighterColor(lockColor) ) const locked = lockHisto.filter((v) => v > 5).length > 5 - if (debugImgs) { - const canvas = imageDataToCanvas(substatsCardCropped) - drawHistogram(canvas, lockHisto, { r: 0, g: 100, b: 0, a: 100 }) - debugImgs['substatsCardCropped'] = canvas.toDataURL() - } - - const bwHeader = bandPass( - headerCropped, - { r: 140, g: 140, b: 140 }, - { r: 255, g: 255, b: 255 }, - 'bw' - ) - const bwGreenText = bandPass( - greenTextCropped, - { r: 30, g: 100, b: 30 }, - { r: 200, g: 255, b: 200 }, - 'bw' - ) - const bwEquipped = - equippedCropped && - bandPass( - equippedCropped, - darkerColor(textColorDark), - lighterColor(textColorLight), - 'bw' - ) - if (debugImgs) { - debugImgs['bwHeader'] = imageDataToCanvas(bwHeader).toDataURL() - debugImgs['greenTextCropped'] = - imageDataToCanvas(greenTextCropped).toDataURL() - debugImgs['bwGreenText'] = imageDataToCanvas(bwGreenText).toDataURL() - if (bwEquipped) - debugImgs['bwEquipped'] = imageDataToCanvas(bwEquipped).toDataURL() - } - - const [whiteTexts, substatTexts, artifactSetTexts, equippedTexts] = - await Promise.all([ - // slotkey, mainStatValue, level - textsFromImage(bwHeader), - // substats - textsFromImage(substatsCardCropped), - // artifact set, look for greenish texts - textsFromImage(bwGreenText), - // equipment - bwEquipped && textsFromImage(bwEquipped), - ]) - - const rarity = parseRarity(headerCropped, debugImgs) + const rarity = parseRarity(await urlToImageData(debugImgs['ArtifactRarity'])) const [artifact, texts] = findBestArtifact( new Set([rarity]), - parseSetKeys(artifactSetTexts), - parseSlotKeys(whiteTexts), - parseSubstats(substatTexts), - parseMainStatKeys(whiteTexts), - parseMainStatValues(whiteTexts), - equippedTexts ? parseLocation(equippedTexts) : '', + parseSetKeys([prediction.ArtifactSet[0]]), + parseSlotKeys(prediction.ArtifactSlot), + parseSubstats(prediction.ArtifactSubstats), + parseMainStatKeys(prediction.ArtifactMainStat), + parseMainStatValues(prediction.ArtifactMainStatValue), + equipped ? parseLocation(prediction.ArtifactLocation) : '', locked ) return { fileName: fName, - imageURL: imageDataToCanvas(artifactCardCropped).toDataURL(), + imageURL: imageDataToCanvas(artifactImageData).toDataURL(), artifact, texts, - debugImgs, + debugImgs: debugImgs, } } -function verticallyCropArtifactCard( - imageData: ImageData, - debugImgs?: Record -) { - const histogram = histogramContAnalysis( - imageData, - darkerColor(cardWhite), - lighterColor(cardWhite) - ) - const [a, b] = findHistogramRange(histogram) - - const cropped = crop(imageDataToCanvas(imageData), { x1: a, x2: b }) - - if (debugImgs) { - const canvas = imageDataToCanvas(imageData) - - drawHistogram(canvas, histogram, { - r: 255, - g: 0, - b: 0, - a: 100, - }) - drawline(canvas, a, { r: 0, g: 255, b: 0, a: 150 }) - drawline(canvas, b, { r: 0, g: 0, b: 255, a: 150 }) - - debugImgs['fullAnalysis'] = canvas.toDataURL() - - // debugImgs['horicropped'] = imageDataToCanvas(cropped).toDataURL() - } - - return cropped -} - -function parseRarity( - headerData: ImageData, - debugImgs?: Record -) { +function parseRarity(headerData: ImageData) { const hist = histogramContAnalysis( headerData, darkerColor(starColor), @@ -357,11 +116,6 @@ function parseRarity( darkerColor(starColor), lighterColor(starColor) ) - if (debugImgs) { - const canvas = imageDataToCanvas(stars) - drawHistogram(canvas, starsHistogram, { r: 100, g: 0, b: 0, a: 100 }) - debugImgs['rarity'] = canvas.toDataURL() - } const maxThresh = Math.max(...starsHistogram) * 0.5 let count = 0 let onStar = false @@ -379,30 +133,3 @@ function parseRarity( } return clamp(count, 1, 5) } - -function findTitle(artifactCardImageData: ImageData) { - const width = artifactCardImageData.width - const widthThreshold = width * 0.7 - - function findTitleColored(darkerTitleColor: Color, LighterTitleColor: Color) { - const hist = histogramContAnalysis( - artifactCardImageData, - darkerColor(darkerTitleColor, 20), - lighterColor(LighterTitleColor, 20), - false, - [0, 0.3] // only scan the top 30% of the img - ) - if (hist.find((v) => v > widthThreshold)) return hist - return null - } - const titleColors = [ - [goldenTitleDarkerColor, goldenTitleLighterColor], - [purpleTitleDarkerColor, purpleTitleLighterColor], - [blueTitleDarkerColor, blueTitleLighterColor], - ] as const - // Return first detected title color - return titleColors.reduce( - (a, curr) => (a ? a : findTitleColored(curr[0], curr[1])), - null as null | number[] - ) -} diff --git a/libs/img-util/src/imageData.ts b/libs/img-util/src/imageData.ts index 8a78a774..26eb6af4 100644 --- a/libs/img-util/src/imageData.ts +++ b/libs/img-util/src/imageData.ts @@ -80,3 +80,189 @@ export function imageDataToCanvas(imageData: ImageData): HTMLCanvasElement { canvas.getContext('2d')!.putImageData(imageData, 0, 0) return canvas // produces a PNG file } + +export function convertToBlackAndWhite( + inputImageData: ImageData, + invert = false, + threshold = 128 +): ImageData { + const data = inputImageData.data + + for (let i = 0; i < data.length; i += 4) { + const luminance = (data[i] + data[i + 1] + data[i + 2]) / 3 + const color = luminance > threshold !== invert ? 255 : 0 + data[i] = data[i + 1] = data[i + 2] = color + } + + return inputImageData +} + +export function scaleImage( + inputImageData: ImageData, + scaleFactor: number +): ImageData { + const canvas = document.createElement('canvas') + canvas.width = Math.floor(inputImageData.width * scaleFactor) + canvas.height = Math.floor(inputImageData.height * scaleFactor) + + const ctx = canvas.getContext('2d')! + ctx.drawImage( + imageDataToCanvas(inputImageData), + 0, + 0, + inputImageData.width, + inputImageData.height, + 0, + 0, + canvas.width, + canvas.height + ) + return ctx.getImageData(0, 0, canvas.width, canvas.height) +} + +export function extractBox( + inputImageData: ImageData, + extractColor: 'white' | 'black', + pad: number +): ImageData { + const inputWidth = inputImageData.width + const inputHeight = inputImageData.height + const inputData = inputImageData.data + + // Find the bounding box of the specified color region + let minX = inputWidth + let minY = inputHeight + let maxX = 0 + let maxY = 0 + + const targetColor = extractColor === 'white' ? [255, 255, 255] : [0, 0, 0] + + for (let y = 0; y < inputHeight; y++) { + for (let x = 0; x < inputWidth; x++) { + const index = (y * inputWidth + x) * 4 + + const isTargetColor = + inputData[index] === targetColor[0] && + inputData[index + 1] === targetColor[1] && + inputData[index + 2] === targetColor[2] + + if (isTargetColor) { + minX = Math.min(minX, x) + minY = Math.min(minY, y) + maxX = Math.max(maxX, x) + maxY = Math.max(maxY, y) + } + } + } + + // Expand the bounding box by the specified padding + minX = Math.max(0, minX - pad) + minY = Math.max(0, minY - pad) + maxX = Math.min(inputWidth - 1, maxX + pad) + maxY = Math.min(inputHeight - 1, maxY + pad) + + // Create a new ImageData to store the extracted color box + const extractedWidth = Math.abs(maxX - minX + 1) + const extractedHeight = Math.abs(maxY - minY + 1) + const extractedImageData = new ImageData(extractedWidth, extractedHeight) + + // Copy the color box from the original ImageData to the new ImageData + for (let y = minY; y <= maxY; y++) { + for (let x = minX; x <= maxX; x++) { + const srcIndex = (y * inputWidth + x) * 4 + const destIndex = ((y - minY) * extractedWidth + (x - minX)) * 4 + + extractedImageData.data[destIndex] = inputData[srcIndex] + extractedImageData.data[destIndex + 1] = inputData[srcIndex + 1] + extractedImageData.data[destIndex + 2] = inputData[srcIndex + 2] + extractedImageData.data[destIndex + 3] = inputData[srcIndex + 3] + } + } + + return extractedImageData +} + +// Function to perform basic edge detection using the Sobel operator +export function edgeDetection(imageData: ImageData): ImageData { + const width = imageData.width + const height = imageData.height + const data = imageData.data + + const sobelKernelX = [-1, 0, 1, -2, 0, 2, -1, 0, 1] + const sobelKernelY = [-1, -2, -1, 0, 0, 0, 1, 2, 1] + + const resultData = new Uint8ClampedArray(data) + + for (let y = 1; y < height - 1; y++) { + for (let x = 1; x < width - 1; x++) { + let sumX = 0 + let sumY = 0 + + for (let ky = 0; ky < 3; ky++) { + for (let kx = 0; kx < 3; kx++) { + const idx = ((y + ky - 1) * width + (x + kx - 1)) * 4 + const weightX = sobelKernelX[ky * 3 + kx] + const weightY = sobelKernelY[ky * 3 + kx] + + sumX += data[idx] * weightX + sumY += data[idx] * weightY + } + } + + const magnitude = Math.sqrt(sumX * sumX + sumY * sumY) + const index = (y * width + x) * 4 + + resultData[index] = magnitude + resultData[index + 1] = magnitude + resultData[index + 2] = magnitude + } + } + + return new ImageData(resultData, width, height) +} + +export function findSplitHeight(bwImageData: ImageData, match = 80): number { + const width = bwImageData.width + const height = bwImageData.height + const data = bwImageData.data + let splitHeight = 0 + + // Start checking after some gap from the top + for (let y = 20; y < height; y++) { + let whitePixelCount = 0 + for (let x = 0; x < width; x++) { + const isWhitePixel = data[(y * width + x) * 4] === 255 + if (isWhitePixel) whitePixelCount++ + } + // Check if more than match% of the pixels in the row are white + const whitePixelPercentage = (whitePixelCount / width) * 100 + if (whitePixelPercentage > match) { + splitHeight = y + break + } + } + + return splitHeight +} + +export function splitImageVertical( + imageData: ImageData, + splitHeight: number +): ImageData[] { + if (splitHeight === 0) { + return [imageData, new ImageData(imageData.width, 1)] + } + const canvas = document.createElement('canvas') + canvas.width = imageData.width + canvas.height = imageData.height + const ctx = canvas.getContext('2d', { willReadFrequently: true })! + ctx.putImageData(imageData, 0, 0) + + const firstPartImageData = crop(canvas, { y1: 0, y2: splitHeight }) + const secondPartImageData = crop(canvas, { + y1: splitHeight, + y2: imageData.height, + }) + + return [firstPartImageData, secondPartImageData] +}