Merge pull request #65 from gurgalex/ui_refactor

UI refactor + crash fixes + teleportation UI

Author: gurgalex

CHANGELOG.md

@@ -1,3 +1,13 @@
+# v0.11.2
+## fix
+- Crash when Creatures screen would pop up a dialog box
+- Crash on summoning screen if no existing trait or creature name was found
+- Fix for mod crashing with low-end computers taking long to startup new processes
+
+## features
+- Add more dedicated UI screens
+  - Teleportation Shrine Realm Select UI
+
 # v0.11.1
 ## fix
 - fix rapid repeat audio of realm object detection

README.md

@@ -25,6 +25,7 @@ A program to aid in making [Siralim Ultimate](https://store.steampowered.com/app
   - Creatures screen (know which creature's traits, artifact, spell gems are being configured)
   - Reorder creature screen
   - GodForge avatar select
+  - Teleportation Shrine realm select
 
 ## Item sounds
 [video of sounds playing](https://youtu.be/2vVCJtCocbA)

VERSION

@@ -1 +1 @@
-0.11.1
+0.11.2

subot/main.py

@@ -5,25 +5,22 @@
 from dataclasses import dataclass
 from typing import Optional
 
+import time
+import math
+
 import cv2
 import numpy as np
 import numpy.typing
+from numpy.typing import NDArray
 from winrt.windows.media.ocr import OcrEngine
 from winrt.windows.globalization import Language
 from winrt.windows.graphics.imaging import *
 from winrt.windows.security.cryptography import CryptographicBuffer
 
-from enum import Enum, auto
+from logging import getLogger
 
+root = getLogger()
 
-class OCRMode(Enum):
-    SUMMON = auto()
-    UNKNOWN = auto()
-    INSPECT = auto()
-    CREATURES_DISPLAY = auto()
-    SELECT_GODFORGE_AVATAR = auto()
-    CREATURE_REORDER_SELECT = auto()
-    CREATURE_REORDER_WITH = auto()
 
 # Modified from https://gist.github.com/dantmnf/23f060278585d6243ffd9b0c538beab2
 
@@ -192,7 +189,7 @@ def english_installed() -> bool:
 
 
 def detect_green_text(image: np.typing.ArrayLike, x_start: float = 0.0, x_end: float = 1.0, y_start: float = 0.0,
-                      y_end: float = 1.0) -> np.typing.ArrayLike:
+                      y_end: float = 1.0) -> NDArray:
     """Using a source image of RGB color, extract highlighted menu items which are a green color"""
     lower_green = np.array([60, 50, 100])
     upper_green = np.array([60, 255, 255])
@@ -243,16 +240,66 @@ def detect_title(frame: np.typing.ArrayLike) -> np.typing.ArrayLike:
     return mask
 
 
-def detect_dialog_text(frame: np.typing.ArrayLike) -> np.typing.ArrayLike:
-    y_start = int(frame.shape[0] * 0.70)
-    y_end = int(frame.shape[0] * 0.95)
-    x_start = int(frame.shape[1] * 0.01)
-    x_end = int(frame.shape[1] * 0.995)
+def timeit(func):
+    def wrap_timer(*args, **kwargs):
+        t1 = time.time()
+        value = func(*args, **kwargs)
+        t2 = time.time()
+        took = t2 - t1
+        print(f"{func.__name__!r} took {math.ceil(took * 1000)}ms")
+        return value
+
+    return wrap_timer
+
+
+def detect_dialog_text(frame: NDArray, gray_frame: NDArray, ocr_engine: OCR) -> Optional[str]:
+    """detect dialog text from frame
+
+    :param gray_frame BGR whole window frame
+    :param ocr_engine: engine that can perform OCR on image
+    """
+    y_start = int(gray_frame.shape[0] * 0.70)
+    y_end = int(gray_frame.shape[0] * 0.95)
+    x_start = int(gray_frame.shape[1] * 0.01)
+    x_end = int(gray_frame.shape[1] * 0.995)
     dialog_area = frame[y_start:y_end, x_start:x_end]
+    # output = np.ones(dialog_area.shape, dtype='uint8')
+    # output[dialog_area > 180] = 255
+    # mask = output
 
     img = cv2.cvtColor(dialog_area, cv2.COLOR_BGR2HLS)
     sensitivity = 30
     lower_white = np.array([0, 255 - sensitivity, 0])
     upper_white = np.array([0, 255, 0])
     mask = cv2.inRange(img, lower_white, upper_white)
-    return mask
+
+    # resize_factor = 2
+    # mask = cv2.resize(mask, (mask.shape[1] * resize_factor, mask.shape[0] * resize_factor),
+    #                   interpolation=cv2.INTER_LINEAR)
+    ocr_result = ocr_engine.recognize_cv2_image(mask)
+    try:
+        first_line = ocr_result.lines[0]
+        first_word = first_line.words[0]
+        bbox = first_word.bounding_rect
+        root.debug(f"dialog box: {ocr_result.text}")
+
+        # health bar text - rect(69, 87, 73, 16), rect(282, 87, 71, 16)
+        # dialog box text - rect(14, 23, 75, 16)
+
+        offset_x = mask.shape[0] * 0.40
+        root.debug(f"{offset_x=}")
+        is_not_dialog_box = bbox.x > offset_x
+        root.debug(f"{is_not_dialog_box=}")
+        if is_not_dialog_box:
+            return None
+        return ocr_result.merged_text
+        # if is_not_dialog_box and not self.has_menu_entry_text:
+        #     return
+    except IndexError:
+        root.debug("no dialog text")
+        return None
+        # no text was found
+        if not self.has_menu_entry_text and not self.has_dialog_text and not self.quest_text:
+            root.info("Pause, menu system. both not present")
+            self.audio_system.silence()
+        return

subot/ocr.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from .base  import SpeakAuto
+from .base import SpeakAuto, FrameInfo, OCRMode
 from .shared import detect_creature_party_selection
 from subot.audio import AudioSystem
 from subot.ocr import OCR
@@ -9,6 +9,8 @@
 
 
 class OCRCreatureRecorderSelectFirst(SpeakAuto):
+    mode = OCRMode.CREATURE_REORDER_SELECT
+
     def __init__(self, audio_system: AudioSystem, config: Config, ocr_engine: OCR):
         self.auto_text: str = ""
         self.audio_system = audio_system
@@ -17,9 +19,9 @@ def __init__(self, audio_system: AudioSystem, config: Config, ocr_engine: OCR):
         self.prev_creature_pos: Optional[int] = None
         self.creature_pos: Optional[int] = None
 
-    def ocr(self, frame: numpy.typing.ArrayLike):
+    def ocr(self, parent: FrameInfo):
         self.prev_creature_pos = self.creature_pos
-        self.creature_pos = detect_creature_party_selection(frame)
+        self.creature_pos = detect_creature_party_selection(parent.frame)
 
     def speak_auto(self):
         if self.creature_pos != self.prev_creature_pos:
@@ -28,6 +30,8 @@ def speak_auto(self):
 
 
 class OCRCreatureRecorderSwapWith(SpeakAuto):
+    mode = OCRMode.CREATURE_REORDER_WITH
+
     def __init__(self, audio_system: AudioSystem, config: Config, ocr_engine: OCR):
         self.auto_text: str = ""
         self.audio_system = audio_system
@@ -36,9 +40,9 @@ def __init__(self, audio_system: AudioSystem, config: Config, ocr_engine: OCR):
         self.prev_creature_pos: Optional[int] = None
         self.creature_pos: Optional[int] = None
 
-    def ocr(self, frame: numpy.typing.ArrayLike):
+    def ocr(self, parent: FrameInfo):
         self.prev_creature_pos = self.creature_pos
-        self.creature_pos = detect_creature_party_selection(frame)
+        self.creature_pos = detect_creature_party_selection(parent.frame)
 
     def speak_auto(self):
         if self.creature_pos != self.prev_creature_pos:

subot/ui_areas/CreatureReorderSelectFirst.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from .base  import SpeakAuto
+from .base import SpeakAuto, FrameInfo, OCRMode
 from .shared import detect_creature_party_selection
 from subot.audio import AudioSystem
 from subot.ocr import OCR
@@ -9,6 +9,8 @@
 
 
 class OCRGodForgeSelectSystem(SpeakAuto):
+    mode = OCRMode.SELECT_GODFORGE_AVATAR
+
     def __init__(self, audio_system: AudioSystem, config: Config, ocr_engine: OCR):
         self.auto_text: str = ""
         self.audio_system = audio_system
@@ -17,12 +19,11 @@ def __init__(self, audio_system: AudioSystem, config: Config, ocr_engine: OCR):
         self.prev_creature_pos: Optional[int] = None
         self.creature_pos: Optional[int] = None
 
-    def ocr(self, frame: numpy.typing.ArrayLike):
+    def ocr(self, parent: FrameInfo):
         self.prev_creature_pos = self.creature_pos
-        self.creature_pos = detect_creature_party_selection(frame)
+        self.creature_pos = detect_creature_party_selection(parent.frame)
 
     def speak_auto(self):
         if self.creature_pos != self.prev_creature_pos:
             text = f"Creature {self.creature_pos}, selected to GodForge"
             self.audio_system.speak_nonblocking(text)
-