Fix hmt encoding

CHANGELOG.MD

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 This project also inherits changes from [Binilla](https://github.com/Sigmmma/binilla).
 
+## [1.9.2]
+### Changed
+ - Fix encoding problems with hmt files that aren't utf-16-le encoded.
+
 ## [1.9.1]
 ### Changed
  - Fix missing license in distributions.

mozzarilla/__init__.py

@@ -10,8 +10,8 @@
 # ##############
 #   metadata   #
 # ##############
-__author__ = "Devin Bobadilla, Michelle van der Graaf"
+__author__ = "Sigmmma"
 #           YYYY.MM.DD
-__date__ = "2020.03.07"
-__version__ = (1, 9, 1)
+__date__ = "2020.04.28"
+__version__ = (1, 9, 2)
 __website__ = "https://github.com/Sigmmma/mozzarilla"

mozzarilla/windows/tools/compile_hud_message_text.py

@@ -8,6 +8,7 @@
 #
 
 import os
+import charset_normalizer
 
 from pathlib import Path
 from traceback import format_exc
@@ -44,8 +45,17 @@ def hud_message_text_from_hmt(app, fp=None):
 
         print("Creating hud_message_text from this hmt file:")
         print("    %s" % fp)
-        with fp.open("r", encoding="utf-16-le") as f:
-            hmt_string_data = f.read()
+
+        with fp.open("rb") as f:
+            contents = f.read()
+            guess = charset_normalizer.detect(contents)
+            # utf-16 is our fallback as that is the proper encoding for
+            # these files and has the biggest detection failure rate.
+            hmt_string_data = contents.decode(guess['encoding'] or "utf-16")
+            # Reading files this way doesn't remove carriage returns.
+            # We have to wipe them out like this.
+            hmt_string_data = hmt_string_data.replace("\r", "")
+
     except Exception:
         print(format_exc())
         print("    Could not load hmt file.")

requirements.txt

@@ -2,3 +2,4 @@ supyr_struct
 binilla
 reclaimer
 arbytmap
+charset_normalizer

setup.py

@@ -54,8 +54,8 @@
         },
     platforms=["POSIX", "Windows"],
     keywords=["binilla", "binary", "data structure"],
-    install_requires=['reclaimer', 'binilla', 'arbytmap', 'supyr_struct'],
-    requires=['reclaimer', 'arbytmap', 'binilla'],
+    install_requires=['reclaimer', 'binilla', 'arbytmap', 'supyr_struct', "charset_normalizer"],
+    requires=['reclaimer', 'arbytmap', 'binilla', "charset_normalizer"],
     provides=['mozzarilla'],
     python_requires=">=3.5",
     classifiers=[