Bugfixes and improvements

- Updated localization system from another project
- Updated original records data

.gitignore

@@ -2,6 +2,7 @@
 .xmake/
 build/
 package/
+install/
 vsxmake2015/
 vsxmake2017/
 vsxmake2019/

src/ArchiveDialog.cpp

@@ -166,7 +166,7 @@ bool ArchiveDialog::Load(const std::filesystem::path &loadPath)
   if (loadPath.empty())
     return false;
 
-  UTFGlyphRangesBuilder::Get().AddText(loadPath.native());
+  GlyphRangesBuilder::Get().AddText(loadPath.native());
 
   progressMessage = LocalizationManager::Get().Localize("ARCHIVE_DIALOG_LOAD_PROGRESS_READING_ARCHIVE");
   progressNext = 0;
@@ -197,7 +197,7 @@ bool ArchiveDialog::Load(const std::filesystem::path &loadPath)
       // TODO - this is causing a lot of synchronizations at the end of loading
       //        best would be to do this on the main thread
       for (const auto& archivePath : archivePaths)
-        UTFGlyphRangesBuilder::Get().AddText(archivePath.native());
+        GlyphRangesBuilder::Get().AddText(archivePath.native());
 
       path = loadPath;
     }
@@ -316,7 +316,7 @@ bool ArchiveDialog::Save(const std::filesystem::path &savePath, bool async)
   if (savePath.empty())
     return false;
 
-  UTFGlyphRangesBuilder::Get().AddText(savePath.native());
+  GlyphRangesBuilder::Get().AddText(savePath.native());
 
   progressMessage = LocalizationManager::Get().Localize("ARCHIVE_DIALOG_SAVE_PROGRESS_SAVING_ARCHIVE");
   progressNext = 0;
@@ -437,11 +437,11 @@ void ArchiveDialog::DrawBaseDialog(std::wstring_view dialogName, std::wstring_vi
     {
       auto progressCurrent = progressNext.load();
 
-      const auto progressSummary = LocalizationManager::Get().LocalizeFormat("ARCHIVE_DIALOG_PROGRESS_SUMMARY", progressCurrent, progressTotal);
+      const auto& progressSummary = LocalizationManager::Get().LocalizeFormat("ARCHIVE_DIALOG_PROGRESS_SUMMARY", progressCurrent, progressTotal);
       if (progressMessage.empty())
         ImGui::TextUnformatted(progressSummary.c_str());
       else
-        ImGui::TextUnformatted(std::format("{}\n{}", progressSummary, progressMessage).c_str());
+        ImGui::TextUnformatted(std::format("{}\n{}", progressSummary.native(), progressMessage.native()).c_str());
     }
     else if (!progressMessage.empty())
       ImGui::TextUnformatted(progressMessage.c_str());

src/ArchiveDialog.hpp

@@ -73,7 +73,7 @@ class ArchiveDialog
   ArchiveDirectory archiveRoot;
 
   std::recursive_mutex progressMessageMutex;
-  std::string progressMessage;
+  String8 progressMessage;
   std::atomic_uint64_t progressNext = 0;
   std::atomic_uint64_t progressNextTotal = 0;
   std::atomic_bool progressNextActive = false;

src/Main.cpp

@@ -15,7 +15,7 @@
 
 bool BuildFontAtlas()
 {
-  if (!UTFGlyphRangesBuilder::Get().NeedsBuild())
+  if (!GlyphRangesBuilder::Get().NeedsBuild())
     return true;
 
   std::vector<std::filesystem::path> searchPaths;
@@ -28,7 +28,7 @@ bool BuildFontAtlas()
 
   ImFontGlyphRangesBuilder builder;
 
-  const auto glyphRanges = UTFGlyphRangesBuilder::Get().Build();
+  const auto glyphRanges = GlyphRangesBuilder::Get().Build();
   for (const auto& glyphRange : glyphRanges)
   {
     uint32_t translatedGlyphRange[3]{glyphRange.first, glyphRange.second, 0};

src/Options.cpp

@@ -26,7 +26,7 @@ void CommonSettings::Load(const toml::table &aInputRoot)
   transcodeToOriginalFormat = commonTable["transcode_to_original_format"].value_or(transcodeToOriginalFormat);
   directImport = commonTable["direct_import"].value_or(directImport);
 
-  LocalizationManager::Get().SetLanguage(commonTable["language"].value_or(LocalizationManager::Get().GetLanguage()));
+  LocalizationManager::Get().SetLanguage(commonTable["language"].value_or(LocalizationManager::Get().GetLanguage().native()));
 }
 
 void CommonSettings::Save(toml::table &aOutputRoot) const
@@ -42,7 +42,7 @@ void CommonSettings::Save(toml::table &aOutputRoot) const
   commonTable.emplace("transcode_to_original_format", transcodeToOriginalFormat);
   commonTable.emplace("direct_import", directImport);
 
-  commonTable.emplace("language", LocalizationManager::Get().GetLanguage());
+  commonTable.emplace("language", LocalizationManager::Get().GetLanguage().native());
 
   aOutputRoot.emplace("common", std::move(commonTable));
 }
@@ -62,7 +62,7 @@ void CommonSettings::DrawDialog()
     const auto availableLanguages = LocalizationManager::Get().GetAvailableLanguages();
     for (const auto& language : availableLanguages)
     {
-      bool selected = UTFCaseInsensitiveCompare(LocalizationManager::Get().GetLanguage(), language) == 0;
+      bool selected = (LocalizationManager::Get().GetLanguage() <=> language) == std::strong_ordering::equivalent;
       if (ImGui::Selectable(language.c_str(), &selected))
         LocalizationManager::Get().SetLanguage(language);
     }

src/Options.hpp

@@ -5,8 +5,9 @@
 
 #pragma once
 
-struct CommonSettings
+class CommonSettings
 {
+public:
   void Load(const toml::table &aInputRoot);
   void Save(toml::table &aOutputRoot) const;
   void ResetToDefaults();
@@ -25,19 +26,14 @@ struct CommonSettings
   bool directImport{false};
 };
 
-struct Options
+class Options : public Singleton<Options>
 {
+public:
   void Load();
   void Save() const;
   void ResetToDefaults();
 
   void DrawDialog();
 
-  static Options &Get()
-  {
-    static Options options;
-    return options;
-  }
-
   CommonSettings common{};
 };

src/Precompiled.hpp

@@ -14,8 +14,6 @@ extern "C"
 #include <imgui.h>
 #include <scn/scn.h>
 #include <sndfile.hh>
-
-#define TOML_EXCEPTIONS 0
 #include <toml++/toml.h>
 
 #include <array>
@@ -31,9 +29,10 @@ extern "C"
 #include <variant>
 #include <vector>
 
-#include "Options.hpp"
-#include "Utils.hpp"
+#include "Singleton.hpp"
+#include "UTF/UTF.hpp"
 
-#include "Localization.hpp"
+using namespace UTF;
 
-using LocalizationManager = UTFLocalizationManager<char>;
+#include "Options.hpp"
+#include "Utils.hpp"

src/Singleton.hpp

@@ -0,0 +1,18 @@
+//
+// Created by Andrej Redeky.
+// Copyright © 2015-2023 Feldarian Softworks. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+//
+
+#pragma once
+
+template <typename InstanceType>
+class Singleton
+{
+public:
+  static InstanceType& Get()
+  {
+    static InstanceType instance;
+    return instance;
+  }
+};

src/UTF/Common.hpp

@@ -0,0 +1,49 @@
+//
+// Created by Andrej Redeky.
+// Copyright © 2015-2023 Feldarian Softworks. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+//
+
+#pragma once
+
+namespace UTF
+{
+
+template <typename TypeInput, typename... TypesToCompareWith>
+concept IsAnyOfTypes = (std::same_as<TypeInput, TypesToCompareWith> || ...);
+
+template <typename UTFCharType>
+concept IsUTF8CharType = IsAnyOfTypes<UTFCharType, char, char8_t, int8_t, uint8_t>;
+
+static_assert(std::same_as<char16_t, UChar>, "ICU UChar typedef is not char16_t!");
+
+#ifdef _WIN32
+
+template <typename UTFCharType>
+concept IsUTF16CharType = IsAnyOfTypes<UTFCharType, wchar_t, char16_t, int16_t, uint16_t, UChar>;
+
+template <typename UTFCharType>
+concept IsUTF32CharType = IsAnyOfTypes<UTFCharType, char32_t, int32_t, uint32_t, UChar32>;
+
+#else // _WIN32
+
+static_assert(std::same_as<wchar_t, UChar32>, "STD wchar_t had unexpected size!");
+
+template <typename UTFCharType>
+concept IsUTF16CharType = IsAnyOfTypes<UTFCharType, char16_t, int16_t, uint16_t, UChar>;
+
+template <typename UTFCharType>
+concept IsUTF32CharType = IsAnyOfTypes<UTFCharType, wchar_t, char32_t, int32_t, uint32_t, UChar32>;
+
+#endif // _WIN32
+
+template <typename UTFCharType>
+concept IsUTFNativeCharType = IsAnyOfTypes<UTFCharType, char, wchar_t>;
+
+template <typename UTFCharType>
+concept IsUTFCharType = IsUTF8CharType<UTFCharType> || IsUTF16CharType<UTFCharType> || IsUTF32CharType<UTFCharType> || IsUTFNativeCharType<UTFCharType>;
+
+static constexpr auto CodepointInvalid{ 0u };
+static constexpr auto CodepointMax{ 0x10FFFFu };
+
+}

src/UTF/GlyphRangesBuilder.hpp

@@ -0,0 +1,168 @@
+//
+// Created by Andrej Redeky.
+// Copyright © 2015-2023 Feldarian Softworks. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+//
+
+#pragma once
+
+namespace UTF
+{
+
+class GlyphRangesBuilder : public Singleton<GlyphRangesBuilder>
+{
+public:
+
+  GlyphRangesBuilder()
+  {
+    Initialize();
+  }
+
+  void Clear()
+  {
+    glyphsInUse.reset();
+
+    Initialize();
+  }
+
+  bool NeedsBuild() const
+  {
+    return newGlyphsAdded;
+  }
+
+  std::vector<std::pair<uint32_t, uint32_t>> Build()
+  {
+    std::vector<std::pair<uint32_t, uint32_t>> glyphRanges;
+
+    std::lock_guard lock(dataMutex);
+
+    std::pair<uint32_t, uint32_t> glyphRange{CodepointInvalid, CodepointInvalid};
+    for (uint32_t glyph = 1; glyph <= CodepointMax; ++glyph)
+    {
+      if (glyphsInUse.test(glyph))
+      {
+        if (glyphRange.first == CodepointInvalid)
+          glyphRange.first = glyph;
+
+        glyphRange.second = glyph;
+      }
+      else
+      {
+        if (glyphRange.first != CodepointInvalid)
+        {
+          glyphRanges.emplace_back(glyphRange);
+          glyphRange = {CodepointInvalid, CodepointInvalid};
+        }
+      }
+    }
+
+    if (glyphRange.first != CodepointInvalid)
+      glyphRanges.emplace_back(glyphRange);
+
+    newGlyphsAdded = false;
+    return glyphRanges;
+  }
+
+  void AddRange(std::pair<uint32_t, uint32_t> glyphRange)
+  {
+    for (auto glyph = glyphRange.first; glyph <= glyphRange.second; ++glyph)
+      Add(glyph);
+  }
+
+  void Add(uint32_t glyph)
+  {
+    if (glyph == CodepointInvalid || glyph > CodepointMax)
+      return;
+
+    std::lock_guard lock(dataMutex);
+    newGlyphsAdded |= !glyphsInUse.test(glyph);
+    glyphsInUse.set(glyph);
+  }
+
+  template <typename UTFCharType>
+  requires IsUTFCharType<UTFCharType>
+  void AddText(const std::basic_string_view<UTFCharType> utf)
+  {
+    if constexpr (IsUTF8CharType<UTFCharType>)
+    {
+      const auto *utfData = utf.data();
+      const auto utfSize = utf.size();
+      for (size_t utfOffset = 0; utfOffset < utfSize;)
+      {
+        uint32_t glyph = CodepointInvalid;
+        U8_NEXT(utfData, utfOffset, utfSize, glyph);
+
+        Add(glyph);
+      }
+    }
+
+    if constexpr (IsUTF16CharType<UTFCharType>)
+    {
+      const auto *utfData = utf.data();
+      const auto utfSize = utf.size();
+      for (size_t utfOffset = 0; utfOffset < utfSize;)
+      {
+        uint32_t glyph = CodepointInvalid;
+        U16_NEXT(utfData, utfOffset, utfSize, glyph);
+
+        Add(glyph);
+      }
+    }
+
+    if constexpr (IsUTF32CharType<UTFCharType>)
+    {
+      for (const auto glyph : utf)
+        Add(static_cast<uint32_t>(glyph));
+    }
+  }
+
+  template <typename UTFCharType>
+  requires IsUTFCharType<UTFCharType>
+  void AddText(const std::basic_string<UTFCharType> &utf)
+  {
+    AddText(std::basic_string_view<UTFCharType>(utf));
+  }
+
+  template <typename UTFCharType, size_t UTFSize>
+  requires IsUTFCharType<UTFCharType>
+  void AddText(const UTFCharType (&utf)[UTFSize])
+  {
+    AddText(std::basic_string_view<UTFCharType>(utf, UTFSize - 1));
+  }
+
+  template <typename UTFCharType>
+  requires IsUTFCharType<UTFCharType>
+  void AddText(const UTFCharType *utf, const size_t length)
+  {
+    AddText(std::basic_string_view<UTFCharType>(utf, length));
+  }
+
+  void AddText(const std::filesystem::path& path)
+  {
+    AddText(path.native());
+  }
+
+  template <typename UTFStorageType, typename UTFCharType, bool CaseSensitive>
+  requires IsUTFCharType<UTFCharType> && IsAnyOfTypes<UTFStorageType, std::basic_string<UTFCharType>, std::basic_string_view<UTFCharType>>
+  void AddText(const StringWrapper<UTFStorageType, UTFCharType, CaseSensitive>& utf)
+  {
+    AddText(utf.native());
+  }
+
+private:
+  void Initialize()
+  {
+    // Basic Latin (ASCII) + Latin-1 Supplement
+    AddRange({0x0020, 0x00FF});
+
+    // Invalid Unicode Character
+    Add(0xFFFD);
+  }
+
+  std::bitset<CodepointMax + 1> glyphsInUse;
+  bool newGlyphsAdded = false;
+
+  mutable std::mutex dataMutex;
+};
+
+}