From a26a5660e601eb4526a6001117eed7a157ef36d6 Mon Sep 17 00:00:00 2001 From: RadRussianRus Date: Wed, 10 Aug 2022 01:08:23 +0300 Subject: [PATCH] [Core] Language system Telegram Desktop uses static language system with code generation, which requires almost full app rebuild on any string modification, no matter how small it is. Since Kotatogram is options-driven, static language system will slow the development. This language system solves the problem by using JSON and runtime string search instead. --- Telegram/CMakeLists.txt | 2 + Telegram/Resources/langs/rewrites/en.json | 3 + Telegram/Resources/qrc/telegram/telegram.qrc | 3 + Telegram/SourceFiles/core/application.cpp | 2 + Telegram/SourceFiles/kotato/kotato_lang.cpp | 294 ++++++++++++++++++ Telegram/SourceFiles/kotato/kotato_lang.h | 166 ++++++++++ .../SourceFiles/lang/lang_cloud_manager.cpp | 4 + Telegram/SourceFiles/lang/lang_tag.cpp | 29 ++ Telegram/SourceFiles/lang/lang_tag.h | 1 + 9 files changed, 504 insertions(+) create mode 100644 Telegram/Resources/langs/rewrites/en.json create mode 100644 Telegram/SourceFiles/kotato/kotato_lang.cpp create mode 100644 Telegram/SourceFiles/kotato/kotato_lang.h diff --git a/Telegram/CMakeLists.txt b/Telegram/CMakeLists.txt index 6fe5f0b7f..bd1aefa61 100644 --- a/Telegram/CMakeLists.txt +++ b/Telegram/CMakeLists.txt @@ -1023,6 +1023,8 @@ PRIVATE iv/iv_delegate_impl.h iv/iv_instance.cpp iv/iv_instance.h + kotato/kotato_lang.cpp + kotato/kotato_lang.h lang/lang_cloud_manager.cpp lang/lang_cloud_manager.h lang/lang_instance.cpp diff --git a/Telegram/Resources/langs/rewrites/en.json b/Telegram/Resources/langs/rewrites/en.json new file mode 100644 index 000000000..e58768fe8 --- /dev/null +++ b/Telegram/Resources/langs/rewrites/en.json @@ -0,0 +1,3 @@ +{ + "dummy_last_string": "" +} diff --git a/Telegram/Resources/qrc/telegram/telegram.qrc b/Telegram/Resources/qrc/telegram/telegram.qrc index 7d5710aea..8a8a61953 100644 --- a/Telegram/Resources/qrc/telegram/telegram.qrc +++ b/Telegram/Resources/qrc/telegram/telegram.qrc @@ -60,4 +60,7 @@ ../../default_shortcuts-custom.json ../../../../lib/xdg/io.github.kotatogram.desktop + + ../../langs/rewrites/en.json + diff --git a/Telegram/SourceFiles/core/application.cpp b/Telegram/SourceFiles/core/application.cpp index 7b831c3a9..99ddef9ae 100644 --- a/Telegram/SourceFiles/core/application.cpp +++ b/Telegram/SourceFiles/core/application.cpp @@ -7,6 +7,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ #include "core/application.h" +#include "kotato/kotato_lang.h" #include "data/data_abstract_structure.h" #include "data/data_photo.h" #include "data/data_document.h" @@ -257,6 +258,7 @@ void Application::run() { style::internal::StartFonts(); ValidateScale(); + Kotato::Lang::Load(Lang::GetInstance().baseId(), Lang::GetInstance().id()); refreshGlobalProxy(); // Depends on app settings being read. diff --git a/Telegram/SourceFiles/kotato/kotato_lang.cpp b/Telegram/SourceFiles/kotato/kotato_lang.cpp new file mode 100644 index 000000000..edbec05f8 --- /dev/null +++ b/Telegram/SourceFiles/kotato/kotato_lang.cpp @@ -0,0 +1,294 @@ +/* +This file is part of Kotatogram Desktop, +the unofficial app based on Telegram Desktop. + +For license and copyright information please follow this link: +https://github.com/kotatogram/kotatogram-desktop/blob/dev/LEGAL +*/ +#include "kotato/kotato_lang.h" + +#include "base/parse_helper.h" +#include "lang/lang_tag.h" + +#include +#include +#include +#include + +namespace Kotato { +namespace Lang { +namespace { + +const auto kDefaultLanguage = qsl("en"); +const std::vector kPostfixes = { + "#zero", + "#one", + "#two", + "#few", + "#many", + "#other" +}; + +QString BaseLangCode; +QString LangCode; + +QMap DefaultValues; +QMap CurrentValues; + +rpl::event_stream<> LangChanges; + +QString LangDir() { + return cWorkingDir() + "tdata/ktg_lang/"; +} + +void ParseLanguageData( + const QString &langCode, + bool isDefault) { + const auto filename = isDefault + ? qsl(":/ktg_lang/%1.json").arg(langCode) + : LangDir() + (qsl("%1.json").arg(langCode)); + + QFile file(filename); + if (!file.exists()) { + return; + } + if (!file.open(QIODevice::ReadOnly)) { + LOG(("Kotato::Lang Info: file %1 could not be read.").arg(filename)); + return; + } + auto error = QJsonParseError{ 0, QJsonParseError::NoError }; + const auto document = QJsonDocument::fromJson( + base::parse::stripComments(file.readAll()), + &error); + file.close(); + + if (error.error != QJsonParseError::NoError) { + LOG(("Kotato::Lang Info: file %1 has failed to parse. Error: %2" + ).arg(filename + ).arg(error.errorString())); + return; + } else if (!document.isObject()) { + LOG(("Kotato::Lang Info: file %1 has failed to parse. Error: object expected" + ).arg(filename)); + return; + } + + const auto applyValue = [&](const QString &name, const QString &translation) { + if (langCode == kDefaultLanguage) { + DefaultValues.insert(name, translation); + } else { + CurrentValues.insert(name, translation); + } + }; + + const auto langKeys = document.object(); + const auto keyList = langKeys.keys(); + + for (auto i = keyList.constBegin(), e = keyList.constEnd(); i != e; ++i) { + const auto key = *i; + if (key.startsWith("dummy_")) { + continue; + } + + const auto value = langKeys.constFind(key); + + if ((*value).isString()) { + + applyValue(key, (*value).toString()); + + } else if ((*value).isObject()) { + + const auto keyPlurals = (*value).toObject(); + const auto pluralList = keyPlurals.keys(); + + for (auto pli = pluralList.constBegin(), ple = pluralList.constEnd(); pli != ple; ++pli) { + const auto plural = *pli; + const auto pluralValue = keyPlurals.constFind(plural); + + if (!(*pluralValue).isString()) { + LOG(("Kotato::Lang Info: wrong value for key %1 in %2 in file %3, string expected") + .arg(plural).arg(key).arg(filename)); + continue; + } + + const auto name = QString(key + "#" + plural); + const auto translation = (*pluralValue).toString(); + + applyValue(name, translation); + } + } else { + LOG(("Kotato::Lang Info: wrong value for key %1 in file %2, string or object expected") + .arg(key).arg(filename)); + } + } +} + +void UnpackDefault() { + const auto langDir = LangDir(); + if (!QDir().exists(langDir)) QDir().mkpath(langDir); + + const auto langs = QDir(":/ktg_lang").entryList(QStringList() << "*.json", QDir::Files); + auto neededLangs = QStringList() << kDefaultLanguage << LangCode << BaseLangCode; + neededLangs.removeDuplicates(); + + for (auto language : langs) { + language.chop(5); + if (!neededLangs.contains(language)) { + continue; + } + + const auto path = langDir + language + ".default.json"; + auto input = QFile(qsl(":/ktg_lang/%1.json").arg(language)); + auto output = QFile(path); + if (input.open(QIODevice::ReadOnly)) { + auto inputData = input.readAll(); + if (output.open(QIODevice::WriteOnly)) { + output.write(inputData); + output.close(); + } + input.close(); + } + } +} + +} // namespace + +void Load(const QString &baseLangCode, const QString &langCode) { + BaseLangCode = baseLangCode; + if (BaseLangCode.endsWith("-raw")) { + BaseLangCode.chop(4); + } + + LangCode = langCode.isEmpty() ? baseLangCode : langCode; + if (LangCode.endsWith("-raw")) { + LangCode.chop(4); + } + + DefaultValues.clear(); + CurrentValues.clear(); + + if (BaseLangCode != kDefaultLanguage) { + ParseLanguageData(kDefaultLanguage, true); + ParseLanguageData(kDefaultLanguage, false); + } + + ParseLanguageData(BaseLangCode, true); + ParseLanguageData(BaseLangCode, false); + + if (LangCode != BaseLangCode) { + ParseLanguageData(LangCode, true); + ParseLanguageData(LangCode, false); + } + + UnpackDefault(); + LangChanges.fire({}); +} + +QString Translate(const QString &key, Var var1, Var var2, Var var3, Var var4) { + auto phrase = (CurrentValues.contains(key) && !CurrentValues.value(key).isEmpty()) + ? CurrentValues.value(key) + : DefaultValues.value(key); + + for (const auto &v : { var1, var2, var3, var4 }) { + if (!v.key.isEmpty()) { + auto skipNext = false; + const auto key = qsl("{%1}").arg(v.key); + const auto neededLength = phrase.length() - key.length(); + for (auto i = 0; i <= neededLength; i++) { + if (skipNext) { + skipNext = false; + continue; + } + + if (phrase.at(i) == QChar('\\')) { + skipNext = true; + } else if (phrase.at(i) == QChar('{') && phrase.mid(i, key.length()) == key) { + phrase.replace(i, key.length(), v.value); + break; + } + } + } + } + + return phrase; +} + +QString Translate(const QString &key, float64 value, Var var1, Var var2, Var var3, Var var4) { + const auto shift = ::Lang::PluralShift(value); + return Translate(key + kPostfixes.at(shift), var1, var2, var3); +} + +TextWithEntities TranslateWithEntities(const QString &key, EntVar var1, EntVar var2, EntVar var3, EntVar var4) { + TextWithEntities phrase = { + (CurrentValues.contains(key) && !CurrentValues.value(key).isEmpty()) + ? CurrentValues.value(key) + : DefaultValues.value(key) + }; + + for (const auto &v : { var1, var2, var3, var4 }) { + if (!v.key.isEmpty()) { + auto skipNext = false; + const auto key = qsl("{%1}").arg(v.key); + const auto neededLength = phrase.text.length() - key.length(); + for (auto i = 0; i <= neededLength; i++) { + if (skipNext) { + skipNext = false; + continue; + } + + if (phrase.text.at(i) == QChar('\\')) { + skipNext = true; + } else if (phrase.text.at(i) == QChar('{') && phrase.text.mid(i, key.length()) == key) { + phrase.text.replace(i, key.length(), v.value.text); + const auto endOld = i + key.length(); + const auto endNew = i + v.value.text.length(); + + // Shift existing entities + if (endNew > endOld) { + const auto diff = endNew - endOld; + for (auto &entity : phrase.entities) { + if (entity.offset() > endOld) { + entity.shiftRight(diff); + } else if (entity.offset() <= i && entity.offset() + entity.length() >= endOld) { + entity.extendToRight(diff); + } + } + } else if (endNew < endOld) { + const auto diff = endOld - endNew; + for (auto &entity : phrase.entities) { + if (entity.offset() > endNew) { + entity.shiftLeft(diff); + } else if (entity.offset() <= i && entity.offset() + entity.length() >= endNew) { + entity.shrinkFromRight(diff); + } + } + } + + // Add new entities + for (auto entity : v.value.entities) { + phrase.entities.append(EntityInText( + entity.type(), + entity.offset() + i, + entity.length(), + entity.data())); + } + break; + } + } + } + } + + return phrase; +} + +TextWithEntities TranslateWithEntities(const QString &key, float64 value, EntVar var1, EntVar var2, EntVar var3, EntVar var4) { + const auto shift = ::Lang::PluralShift(value); + return TranslateWithEntities(key + kPostfixes.at(shift), var1, var2, var3, var4); +} + +rpl::producer<> Events() { + return LangChanges.events(); +} + +} // namespace Lang +} // namespace Kotato diff --git a/Telegram/SourceFiles/kotato/kotato_lang.h b/Telegram/SourceFiles/kotato/kotato_lang.h new file mode 100644 index 000000000..195e25ae0 --- /dev/null +++ b/Telegram/SourceFiles/kotato/kotato_lang.h @@ -0,0 +1,166 @@ +/* +This file is part of Kotatogram Desktop, +the unofficial app based on Telegram Desktop. + +For license and copyright information please follow this link: +https://github.com/kotatogram/kotatogram-desktop/blob/dev/LEGAL +*/ +#pragma once + +namespace Kotato { +namespace Lang { + +struct Var { + Var() {}; + Var(const QString &k, const QString &v) { + key = k; + value = v; + } + + QString key; + QString value; +}; + +struct EntVar { + EntVar() {}; + EntVar(const QString &k, TextWithEntities v) { + key = k; + value = v; + } + + QString key; + TextWithEntities value; +}; + +void Load(const QString &baseLangCode, const QString &langCode); + +QString Translate( + const QString &key, + Var var1 = Var(), + Var var2 = Var(), + Var var3 = Var(), + Var var4 = Var()); +QString Translate( + const QString &key, + float64 value, + Var var1 = Var(), + Var var2 = Var(), + Var var3 = Var(), + Var var4 = Var()); + +TextWithEntities TranslateWithEntities( + const QString &key, + EntVar var1 = EntVar(), + EntVar var2 = EntVar(), + EntVar var3 = EntVar(), + EntVar var4 = EntVar()); +TextWithEntities TranslateWithEntities( + const QString &key, + float64 value, + EntVar var1 = EntVar(), + EntVar var2 = EntVar(), + EntVar var3 = EntVar(), + EntVar var4 = EntVar()); + +rpl::producer<> Events(); + +} // namespace Lang +} // namespace Kotato + +// Shorthands + +inline QString ktr( + const QString &key, + ::Kotato::Lang::Var var1 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var2 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var3 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var4 = ::Kotato::Lang::Var()) { + return ::Kotato::Lang::Translate(key, var1, var2, var3, var4); +} + +inline QString ktr( + const QString &key, + float64 value, + ::Kotato::Lang::Var var1 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var2 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var3 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var4 = ::Kotato::Lang::Var()) { + return ::Kotato::Lang::Translate(key, value, var1, var2, var3, var4); +} + +inline TextWithEntities ktre( + const QString &key, + ::Kotato::Lang::EntVar var1 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var2 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var3 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var4 = ::Kotato::Lang::EntVar()) { + return ::Kotato::Lang::TranslateWithEntities(key, var1, var2, var3, var4); +} + +inline TextWithEntities ktre( + const QString &key, + float64 value, + ::Kotato::Lang::EntVar var1 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var2 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var3 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var4 = ::Kotato::Lang::EntVar()) { + return ::Kotato::Lang::TranslateWithEntities(key, value, var1, var2, var3, var4); +} + +inline rpl::producer rktr( + const QString &key, + ::Kotato::Lang::Var var1 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var2 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var3 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var4 = ::Kotato::Lang::Var()) { + return rpl::single( + ::Kotato::Lang::Translate(key, var1, var2, var3, var4) + ) | rpl::then( + ::Kotato::Lang::Events() | rpl::map( + [=]{ return ::Kotato::Lang::Translate(key, var1, var2, var3, var4); }) + ); +} + +inline rpl::producer rktr( + const QString &key, + float64 value, + ::Kotato::Lang::Var var1 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var2 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var3 = ::Kotato::Lang::Var(), + ::Kotato::Lang::Var var4 = ::Kotato::Lang::Var()) { + return rpl::single( + ::Kotato::Lang::Translate(key, value, var1, var2, var3, var4) + ) | rpl::then( + ::Kotato::Lang::Events() | rpl::map( + [=]{ return ::Kotato::Lang::Translate(key, value, var1, var2, var3, var4); }) + ); +} + +inline rpl::producer rktre( + const QString &key, + ::Kotato::Lang::EntVar var1 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var2 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var3 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var4 = ::Kotato::Lang::EntVar()) { + return rpl::single( + ::Kotato::Lang::TranslateWithEntities(key, var1, var2, var3, var4) + ) | rpl::then( + ::Kotato::Lang::Events() | rpl::map( + [=]{ return ::Kotato::Lang::TranslateWithEntities(key, var1, var2, var3, var4); }) + ); +} + +inline rpl::producer rktre( + const QString &key, + float64 value, + ::Kotato::Lang::EntVar var1 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var2 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var3 = ::Kotato::Lang::EntVar(), + ::Kotato::Lang::EntVar var4 = ::Kotato::Lang::EntVar()) { + return rpl::single( + ::Kotato::Lang::TranslateWithEntities(key, value, var1, var2, var3, var4) + ) | rpl::then( + ::Kotato::Lang::Events() | rpl::map( + [=]{ return ::Kotato::Lang::TranslateWithEntities(key, value, var1, var2, var3, var4); }) + ); +} diff --git a/Telegram/SourceFiles/lang/lang_cloud_manager.cpp b/Telegram/SourceFiles/lang/lang_cloud_manager.cpp index 8c98b96d1..c1d3c13fe 100644 --- a/Telegram/SourceFiles/lang/lang_cloud_manager.cpp +++ b/Telegram/SourceFiles/lang/lang_cloud_manager.cpp @@ -7,6 +7,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ #include "lang/lang_cloud_manager.h" +#include "kotato/kotato_lang.h" #include "lang/lang_instance.h" #include "lang/lang_file_parser.h" #include "lang/lang_text_entity.h" @@ -447,6 +448,7 @@ void CloudManager::sendSwitchingToLanguageRequest() { const auto finalize = [=] { if (canApplyWithoutRestart(language.id)) { performSwitchAndAddToRecent(language); + Kotato::Lang::Load(Lang::GetInstance().baseId(), Lang::GetInstance().id()); } else { performSwitchAndRestart(language); } @@ -482,6 +484,7 @@ void CloudManager::switchToLanguage(const Language &data) { performSwitchToCustom(); } else if (canApplyWithoutRestart(data.id)) { performSwitchAndAddToRecent(data); + Kotato::Lang::Load(Lang::GetInstance().baseId(), Lang::GetInstance().id()); } else { QVector keys; keys.reserve(3); @@ -535,6 +538,7 @@ void CloudManager::performSwitchToCustom() { } if (canApplyWithoutRestart(u"#custom"_q)) { _langpack.switchToCustomFile(filePath); + Kotato::Lang::Load(Lang::GetInstance().baseId(), Lang::GetInstance().id()); } else { const auto values = loader.found(); const auto getValue = [&](ushort key) { diff --git a/Telegram/SourceFiles/lang/lang_tag.cpp b/Telegram/SourceFiles/lang/lang_tag.cpp index 7b1a10acb..513d5d90f 100644 --- a/Telegram/SourceFiles/lang/lang_tag.cpp +++ b/Telegram/SourceFiles/lang/lang_tag.cpp @@ -987,6 +987,35 @@ PluralResult Plural( return { shift, FormatDouble(value) }; } +ushort PluralShift(float64 value, bool isShortened) { + // To correctly select a shift for PluralType::Short + // we must first round the number. + const auto shortened = (isShortened) + ? FormatCountToShort(qRound(value)) + : ShortenedCount(); + + // Simplified. + const auto n = std::abs(shortened.number ? float64(shortened.number) : value); + const auto i = int(std::floor(n)); + const auto integer = (int(std::ceil(n)) == i); + const auto formatted = integer ? QString() : FormatDouble(n); + const auto dot = formatted.indexOf('.'); + const auto fraction = (dot >= 0) ? formatted.mid(dot + 1) : QString(); + const auto v = fraction.size(); + const auto w = v; + const auto f = NonZeroPartToInt(fraction); + const auto t = f; + + const auto useNonDefaultPlural = (ChoosePlural != ChoosePluralDefault); + return (useNonDefaultPlural ? ChoosePlural : ChoosePluralDefault)( + (integer ? i : -1), + i, + v, + w, + f, + t); +} + void UpdatePluralRules(const QString &languageId) { static auto kMap = GeneratePluralRulesMap(); auto parent = uint64(0); diff --git a/Telegram/SourceFiles/lang/lang_tag.h b/Telegram/SourceFiles/lang/lang_tag.h index 297ae1ff2..c252711ff 100644 --- a/Telegram/SourceFiles/lang/lang_tag.h +++ b/Telegram/SourceFiles/lang/lang_tag.h @@ -34,6 +34,7 @@ PluralResult Plural( ushort keyBase, float64 value, lngtag_count type); +ushort PluralShift(float64 value, bool isShortened = false); void UpdatePluralRules(const QString &languageId); template