From 40bd37315a1b8a733a14636291f824162a38354b Mon Sep 17 00:00:00 2001 From: Eric Kotato Date: Thu, 14 Mar 2024 07:01:15 +0300 Subject: [PATCH 001/596] Use %NUMBER_OF_PROCESSORS% for jom --- Telegram/build/prepare/prepare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Telegram/build/prepare/prepare.py b/Telegram/build/prepare/prepare.py index bb63f2d62..173f34c2b 100644 --- a/Telegram/build/prepare/prepare.py +++ b/Telegram/build/prepare/prepare.py @@ -1454,8 +1454,8 @@ win: -nomake tests ^ -platform win32-msvc - jom -j16 - jom -j16 install + jom -j%NUMBER_OF_PROCESSORS% + jom -j%NUMBER_OF_PROCESSORS% install mac: find ../../patches/qtbase_5.15.13 -type f -print0 | sort -z | xargs -0 git apply cd .. From c9e566f8aa9372bd767203848c06a5f2f8d20d09 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Thu, 14 Mar 2024 08:50:38 +0400 Subject: [PATCH 002/596] Fix snap build Qt has removed support for kernels less than 4.11 by removing the -no-feature-statx flag. Remove the usage all the related flags. --- snap/snapcraft.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index f869d7697..a4ef1f060 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -512,10 +512,6 @@ parts: -release \ -opensource \ -confirm-license \ - -no-feature-getentropy \ - -no-feature-renameat2 \ - -no-feature-statx \ - -no-feature-egl-extension-platform-wayland \ -openssl-linked \ -nomake examples \ -nomake tests From 9414ef570940eef138d04825289d489460309868 Mon Sep 17 00:00:00 2001 From: Eric Kotato Date: Thu, 14 Mar 2024 07:32:25 +0300 Subject: [PATCH 003/596] Using system font stack for Instant View --- Telegram/Resources/iv_html/page.css | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/Telegram/Resources/iv_html/page.css b/Telegram/Resources/iv_html/page.css index bdc44f184..7d4bd22c0 100644 --- a/Telegram/Resources/iv_html/page.css +++ b/Telegram/Resources/iv_html/page.css @@ -1,5 +1,11 @@ +:root { + --font-sans: -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, tahoma, arial, sans-serif; + --font-serif: Iowan Old Style, Apple Garamond, Baskerville, Georgia, Times New Roman, Droid Serif, Times, Source Serif Pro, serif, Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol; + --font-mono: Menlo, Cascadia Code, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; +} + body { - font-family: 'Helvetica Neue'; + font-family: var(--font-sans); font-size: 17px; line-height: 25px; padding: 0; @@ -145,7 +151,7 @@ article { } article h1, article h2 { - font-family: 'Georgia'; + font-family: var(--font-serif); font-size: 28px; line-height: 31px; margin: 21px 18px 12px; @@ -237,7 +243,7 @@ article ol p { } article pre, article pre.hljs { - font-family: Menlo; + font-family: var(--font-mono); margin: 14px 0; padding: 7px 18px; background: #F5F8FC; @@ -265,7 +271,7 @@ article pre + pre { } article h3, article h4 { - font-family: 'Georgia'; + font-family: var(--font-serif); font-size: 24px; line-height: 30px; margin: 18px 18px 9px; @@ -284,7 +290,7 @@ article ol h4 { margin: 10px 0 5px; } article blockquote { - font-family: 'Georgia'; + font-family: var(--font-serif); margin: 18px 18px 16px; padding-left: 22px; position: relative; @@ -310,7 +316,7 @@ article.rtl blockquote:before { left: auto; } article aside { - font-family: 'Georgia'; + font-family: var(--font-serif); margin: 18px 18px 16px; padding: 0 18px; text-align: center; @@ -326,7 +332,7 @@ article blockquote cite, article aside cite, article footer cite, article .pullquote cite { - font-family: 'Helvetica Neue'; + font-family: var(--font-sans); font-size: 15px; display: block; color: var(--td-window-sub-text-fg); @@ -601,7 +607,7 @@ ol figcaption { padding-right: 0; } figcaption > cite { - font-family: 'Helvetica Neue'; + font-family: var(--font-sans); font-size: 12px; display: block; line-height: 15px; @@ -855,7 +861,7 @@ section.related { margin: 7px 0 12px; } section.related h4 { - font-family: 'Helvetica Neue'; + font-family: var(--font-sans); font-size: 17px; line-height: 26px; font-weight: 500; @@ -996,7 +1002,7 @@ section.channel > a > div.join span:before { content: var(--td-lng-group-call-join); } section.channel > a > h4 { - font-family: 'Helvetica Neue'; + font-family: var(--font-sans); font-size: 17px; line-height: 26px; font-weight: 500; From c0b612c45753a9b45c0b965787c52b3055238320 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Thu, 14 Mar 2024 08:37:38 +0400 Subject: [PATCH 004/596] Update cppgir --- .../linux/notifications_manager_linux.cpp | 30 +++++-------------- cmake | 2 +- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/Telegram/SourceFiles/platform/linux/notifications_manager_linux.cpp b/Telegram/SourceFiles/platform/linux/notifications_manager_linux.cpp index 85d0fc037..aed48c948 100644 --- a/Telegram/SourceFiles/platform/linux/notifications_manager_linux.cpp +++ b/Telegram/SourceFiles/platform/linux/notifications_manager_linux.cpp @@ -469,29 +469,18 @@ void NotificationData::close() { } void NotificationData::setImage(QImage image) { - using DestroyNotify = gi::detail::callback< - void(), - gi::transfer_full_t, - std::tuple<> - >; - if (_notification) { const auto imageData = std::make_shared(); QBuffer buffer(imageData.get()); buffer.open(QIODevice::WriteOnly); image.save(&buffer, "PNG"); - const auto callbackWrap = gi::unwrap( - DestroyNotify([imageData] {}), - gi::scope_notified); - _notification.set_icon( Gio::BytesIcon::new_( - gi::wrap(g_bytes_new_with_free_func( - imageData->constData(), + GLib::Bytes::new_with_free_func( + reinterpret_cast(imageData->constData()), imageData->size(), - &callbackWrap->destroy, - callbackWrap), gi::transfer_full))); + [imageData] {}))); return; } @@ -506,10 +495,6 @@ void NotificationData::setImage(QImage image) { image.convertTo(QImage::Format_RGB888); } - const auto callbackWrap = gi::unwrap( - DestroyNotify([image] {}), - gi::scope_notified); - _hints.insert_value(_imageKey, GLib::Variant::new_tuple({ GLib::Variant::new_int32(image.width()), GLib::Variant::new_int32(image.height()), @@ -517,13 +502,12 @@ void NotificationData::setImage(QImage image) { GLib::Variant::new_boolean(image.hasAlphaChannel()), GLib::Variant::new_int32(8), GLib::Variant::new_int32(image.hasAlphaChannel() ? 4 : 3), - gi::wrap(g_variant_new_from_data( - G_VARIANT_TYPE_BYTESTRING, - image.constBits(), + GLib::Variant::new_from_data( + GLib::VariantType::new_("ay"), + reinterpret_cast(image.constBits()), image.sizeInBytes(), true, - &callbackWrap->destroy, - callbackWrap), gi::transfer_none), + [image] {}), })); } diff --git a/cmake b/cmake index c82faf706..95c83fd26 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit c82faf706fd10618ad5f97a7972fb09c56843271 +Subproject commit 95c83fd26424c26d4fc4454cef601529616683aa From e0447539343dad225d60c8d4f65937134e96acfc Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Thu, 14 Mar 2024 10:17:15 +0400 Subject: [PATCH 005/596] Fix macOS GH action --- .github/workflows/mac.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 68bab389c..61561be9f 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -64,7 +64,10 @@ jobs: - name: First set up. run: | sudo chown -R `whoami`:admin /usr/local/share - brew install automake ninja pkg-config nasm meson + + brew update + brew upgrade || true + brew install automake meson nasm ninja pkg-config # Disable spotlight. sudo mdutil -a -i off From 458b990b8c4f5816689a49a9e2517a8b973f26da Mon Sep 17 00:00:00 2001 From: kukuruzka <60149954+kukuruzka165@users.noreply.github.com> Date: Mon, 4 Mar 2024 19:39:07 +0200 Subject: [PATCH 006/596] Improve transliteration --- Telegram/SourceFiles/core/utils.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Telegram/SourceFiles/core/utils.cpp b/Telegram/SourceFiles/core/utils.cpp index bcc0068e0..53e0008ca 100644 --- a/Telegram/SourceFiles/core/utils.cpp +++ b/Telegram/SourceFiles/core/utils.cpp @@ -598,8 +598,10 @@ QString rusKeyboardLayoutSwitch(const QString &from) { fastRusKeyboardSwitch.insert(QString::fromUtf8("ю").at(0), '.'); fastRusKeyboardSwitch.insert(QString::fromUtf8("І").at(0), 'S'); fastRusKeyboardSwitch.insert(QString::fromUtf8("і").at(0), 's'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ї").at(0), ']'); + fastRusKeyboardSwitch.insert(QString::fromUtf8("Ї").at(0), '}'); fastRusKeyboardSwitch.insert(QString::fromUtf8("ї").at(0), ']'); + fastRusKeyboardSwitch.insert(QString::fromUtf8("Є").at(0), '"'); + fastRusKeyboardSwitch.insert(QString::fromUtf8("є").at(0), '\''); } QString result; From bf4b4ae9361813badee92d6c7322091689486d7f Mon Sep 17 00:00:00 2001 From: kukuruzka <60149954+kukuruzka165@users.noreply.github.com> Date: Tue, 5 Mar 2024 19:37:47 +0200 Subject: [PATCH 007/596] full support for ukrainian transliteration --- Telegram/SourceFiles/core/utils.cpp | 175 ++++++---------------------- 1 file changed, 36 insertions(+), 139 deletions(-) diff --git a/Telegram/SourceFiles/core/utils.cpp b/Telegram/SourceFiles/core/utils.cpp index 53e0008ca..c82c7ed65 100644 --- a/Telegram/SourceFiles/core/utils.cpp +++ b/Telegram/SourceFiles/core/utils.cpp @@ -335,6 +335,7 @@ namespace { QHash fastLetterRusEng; QMap fastDoubleLetterRusEng; QHash fastRusKeyboardSwitch; + QHash fastUkrKeyboardSwitch; } QString translitLetterRusEng(QChar letter, QChar next, int32 &toSkip) { @@ -466,149 +467,38 @@ QString translitRusEng(const QString &rus) { return result; } -QString rusKeyboardLayoutSwitch(const QString &from) { - if (fastRusKeyboardSwitch.isEmpty()) { - fastRusKeyboardSwitch.insert('Q', QString::fromUtf8("Й").at(0)); - fastRusKeyboardSwitch.insert('W', QString::fromUtf8("Ц").at(0)); - fastRusKeyboardSwitch.insert('E', QString::fromUtf8("У").at(0)); - fastRusKeyboardSwitch.insert('R', QString::fromUtf8("К").at(0)); - fastRusKeyboardSwitch.insert('T', QString::fromUtf8("Е").at(0)); - fastRusKeyboardSwitch.insert('Y', QString::fromUtf8("Н").at(0)); - fastRusKeyboardSwitch.insert('U', QString::fromUtf8("Г").at(0)); - fastRusKeyboardSwitch.insert('I', QString::fromUtf8("Ш").at(0)); - fastRusKeyboardSwitch.insert('O', QString::fromUtf8("Щ").at(0)); - fastRusKeyboardSwitch.insert('P', QString::fromUtf8("З").at(0)); - fastRusKeyboardSwitch.insert('{', QString::fromUtf8("Х").at(0)); - fastRusKeyboardSwitch.insert('}', QString::fromUtf8("Ъ").at(0)); - fastRusKeyboardSwitch.insert('A', QString::fromUtf8("Ф").at(0)); - fastRusKeyboardSwitch.insert('S', QString::fromUtf8("Ы").at(0)); - fastRusKeyboardSwitch.insert('D', QString::fromUtf8("В").at(0)); - fastRusKeyboardSwitch.insert('F', QString::fromUtf8("А").at(0)); - fastRusKeyboardSwitch.insert('G', QString::fromUtf8("П").at(0)); - fastRusKeyboardSwitch.insert('H', QString::fromUtf8("Р").at(0)); - fastRusKeyboardSwitch.insert('J', QString::fromUtf8("О").at(0)); - fastRusKeyboardSwitch.insert('K', QString::fromUtf8("Л").at(0)); - fastRusKeyboardSwitch.insert('L', QString::fromUtf8("Д").at(0)); - fastRusKeyboardSwitch.insert(':', QString::fromUtf8("Ж").at(0)); - fastRusKeyboardSwitch.insert('"', QString::fromUtf8("Э").at(0)); - fastRusKeyboardSwitch.insert('Z', QString::fromUtf8("Я").at(0)); - fastRusKeyboardSwitch.insert('X', QString::fromUtf8("Ч").at(0)); - fastRusKeyboardSwitch.insert('C', QString::fromUtf8("С").at(0)); - fastRusKeyboardSwitch.insert('V', QString::fromUtf8("М").at(0)); - fastRusKeyboardSwitch.insert('B', QString::fromUtf8("И").at(0)); - fastRusKeyboardSwitch.insert('N', QString::fromUtf8("Т").at(0)); - fastRusKeyboardSwitch.insert('M', QString::fromUtf8("Ь").at(0)); - fastRusKeyboardSwitch.insert('<', QString::fromUtf8("Б").at(0)); - fastRusKeyboardSwitch.insert('>', QString::fromUtf8("Ю").at(0)); - fastRusKeyboardSwitch.insert('q', QString::fromUtf8("й").at(0)); - fastRusKeyboardSwitch.insert('w', QString::fromUtf8("ц").at(0)); - fastRusKeyboardSwitch.insert('e', QString::fromUtf8("у").at(0)); - fastRusKeyboardSwitch.insert('r', QString::fromUtf8("к").at(0)); - fastRusKeyboardSwitch.insert('t', QString::fromUtf8("е").at(0)); - fastRusKeyboardSwitch.insert('y', QString::fromUtf8("н").at(0)); - fastRusKeyboardSwitch.insert('u', QString::fromUtf8("г").at(0)); - fastRusKeyboardSwitch.insert('i', QString::fromUtf8("ш").at(0)); - fastRusKeyboardSwitch.insert('o', QString::fromUtf8("щ").at(0)); - fastRusKeyboardSwitch.insert('p', QString::fromUtf8("з").at(0)); - fastRusKeyboardSwitch.insert('[', QString::fromUtf8("х").at(0)); - fastRusKeyboardSwitch.insert(']', QString::fromUtf8("ъ").at(0)); - fastRusKeyboardSwitch.insert('a', QString::fromUtf8("ф").at(0)); - fastRusKeyboardSwitch.insert('s', QString::fromUtf8("ы").at(0)); - fastRusKeyboardSwitch.insert('d', QString::fromUtf8("в").at(0)); - fastRusKeyboardSwitch.insert('f', QString::fromUtf8("а").at(0)); - fastRusKeyboardSwitch.insert('g', QString::fromUtf8("п").at(0)); - fastRusKeyboardSwitch.insert('h', QString::fromUtf8("р").at(0)); - fastRusKeyboardSwitch.insert('j', QString::fromUtf8("о").at(0)); - fastRusKeyboardSwitch.insert('k', QString::fromUtf8("л").at(0)); - fastRusKeyboardSwitch.insert('l', QString::fromUtf8("д").at(0)); - fastRusKeyboardSwitch.insert(';', QString::fromUtf8("ж").at(0)); - fastRusKeyboardSwitch.insert('\'', QString::fromUtf8("э").at(0)); - fastRusKeyboardSwitch.insert('z', QString::fromUtf8("я").at(0)); - fastRusKeyboardSwitch.insert('x', QString::fromUtf8("ч").at(0)); - fastRusKeyboardSwitch.insert('c', QString::fromUtf8("с").at(0)); - fastRusKeyboardSwitch.insert('v', QString::fromUtf8("м").at(0)); - fastRusKeyboardSwitch.insert('b', QString::fromUtf8("и").at(0)); - fastRusKeyboardSwitch.insert('n', QString::fromUtf8("т").at(0)); - fastRusKeyboardSwitch.insert('m', QString::fromUtf8("ь").at(0)); - fastRusKeyboardSwitch.insert(',', QString::fromUtf8("б").at(0)); - fastRusKeyboardSwitch.insert('.', QString::fromUtf8("ю").at(0)); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Й").at(0), 'Q'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ц").at(0), 'W'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("У").at(0), 'E'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("К").at(0), 'R'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Е").at(0), 'T'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Н").at(0), 'Y'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Г").at(0), 'U'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ш").at(0), 'I'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Щ").at(0), 'O'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("З").at(0), 'P'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Х").at(0), '{'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ъ").at(0), '}'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ф").at(0), 'A'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ы").at(0), 'S'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("В").at(0), 'D'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("А").at(0), 'F'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("П").at(0), 'G'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Р").at(0), 'H'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("О").at(0), 'J'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Л").at(0), 'K'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Д").at(0), 'L'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ж").at(0), ':'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Э").at(0), '"'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Я").at(0), 'Z'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ч").at(0), 'X'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("С").at(0), 'C'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("М").at(0), 'V'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("И").at(0), 'B'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Т").at(0), 'N'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ь").at(0), 'M'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Б").at(0), '<'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ю").at(0), '>'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("й").at(0), 'q'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ц").at(0), 'w'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("у").at(0), 'e'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("к").at(0), 'r'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("е").at(0), 't'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("н").at(0), 'y'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("г").at(0), 'u'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ш").at(0), 'i'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("щ").at(0), 'o'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("з").at(0), 'p'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("х").at(0), '['); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ъ").at(0), ']'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ф").at(0), 'a'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ы").at(0), 's'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("в").at(0), 'd'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("а").at(0), 'f'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("п").at(0), 'g'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("р").at(0), 'h'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("о").at(0), 'j'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("л").at(0), 'k'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("д").at(0), 'l'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ж").at(0), ';'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("э").at(0), '\''); - fastRusKeyboardSwitch.insert(QString::fromUtf8("я").at(0), 'z'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ч").at(0), 'x'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("с").at(0), 'c'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("м").at(0), 'v'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("и").at(0), 'b'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("т").at(0), 'n'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ь").at(0), 'm'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("б").at(0), ','); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ю").at(0), '.'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("І").at(0), 'S'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("і").at(0), 's'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Ї").at(0), '}'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("ї").at(0), ']'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("Є").at(0), '"'); - fastRusKeyboardSwitch.insert(QString::fromUtf8("є").at(0), '\''); - } +QString engAlphabet = "qwertyuiop[]asdfghjkl;'zxcvbnm,."; +QString engAlphabetUpper = engAlphabet.toUpper(); +void initializeKeyboardSwitch() { + if (fastRusKeyboardSwitch.isEmpty()) { + QString rusAlphabet = "йцукенгшщзхъфывапролджэячсмитьбю"; + QString rusAlphabetUpper = rusAlphabet.toUpper(); + for (int i = 0; i < rusAlphabet.size(); ++i) { + fastRusKeyboardSwitch.insert(engAlphabetUpper[i], rusAlphabetUpper[i]); + fastRusKeyboardSwitch.insert(engAlphabet[i], rusAlphabet[i]); + fastRusKeyboardSwitch.insert(rusAlphabetUpper[i], engAlphabetUpper[i]); + fastRusKeyboardSwitch.insert(rusAlphabet[i], engAlphabet[i]); + } + } + if (fastUkrKeyboardSwitch.isEmpty()) { + QString ukrAlphabet = "йцукенгшщзхїфівапролджєячсмитьбю"; + QString ukrAlphabetUpper = ukrAlphabet.toUpper(); + for (int i = 0; i < ukrAlphabet.size(); ++i) { + fastUkrKeyboardSwitch.insert(engAlphabetUpper[i], ukrAlphabetUpper[i]); + fastUkrKeyboardSwitch.insert(engAlphabet[i], ukrAlphabet[i]); + fastUkrKeyboardSwitch.insert(ukrAlphabetUpper[i], engAlphabetUpper[i]); + fastUkrKeyboardSwitch.insert(ukrAlphabet[i], engAlphabet[i]); + } + } +} + +QString switchKeyboardLayout(const QString& from, QHash& keyboardSwitch) { QString result; result.reserve(from.size()); for (QString::const_iterator i = from.cbegin(), e = from.cend(); i != e; ++i) { - QHash::const_iterator j = fastRusKeyboardSwitch.constFind(*i); - if (j == fastRusKeyboardSwitch.cend()) { + QHash::const_iterator j = keyboardSwitch.constFind(*i); + if (j == keyboardSwitch.cend()) { result += *i; } else { result += j.value(); @@ -616,3 +506,10 @@ QString rusKeyboardLayoutSwitch(const QString &from) { } return result; } + +QString rusKeyboardLayoutSwitch(const QString& from) { + initializeKeyboardSwitch(); + QString rus = switchKeyboardLayout(from, fastRusKeyboardSwitch); + QString ukr = switchKeyboardLayout(from, fastUkrKeyboardSwitch); + return rus == ukr ? rus : rus + ' ' + ukr; +} From d5b23b5bde162ac94295665b3579ba4707db79c6 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 10:55:21 +0400 Subject: [PATCH 008/596] Use a separate lang key for IV channel join button. --- Telegram/Resources/iv_html/page.css | 2 +- Telegram/Resources/langs/lang.strings | 1 + Telegram/SourceFiles/iv/iv_controller.cpp | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Telegram/Resources/iv_html/page.css b/Telegram/Resources/iv_html/page.css index 7d4bd22c0..85a6124cc 100644 --- a/Telegram/Resources/iv_html/page.css +++ b/Telegram/Resources/iv_html/page.css @@ -999,7 +999,7 @@ section.channel > a > div.join:hover { text-decoration: underline; } section.channel > a > div.join span:before { - content: var(--td-lng-group-call-join); + content: var(--td-lng-iv-join-channel); } section.channel > a > h4 { font-family: var(--font-sans); diff --git a/Telegram/Resources/langs/lang.strings b/Telegram/Resources/langs/lang.strings index 2237f68ca..437479980 100644 --- a/Telegram/Resources/langs/lang.strings +++ b/Telegram/Resources/langs/lang.strings @@ -4869,6 +4869,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL "lng_iv_open_in_browser" = "Open in Browser"; "lng_iv_share" = "Share"; +"lng_iv_join_channel" = "Join"; // Wnd specific diff --git a/Telegram/SourceFiles/iv/iv_controller.cpp b/Telegram/SourceFiles/iv/iv_controller.cpp index db69e9b97..3d7a30318 100644 --- a/Telegram/SourceFiles/iv/iv_controller.cpp +++ b/Telegram/SourceFiles/iv/iv_controller.cpp @@ -76,7 +76,7 @@ namespace { { "toast-fg", &st::toastFg }, }; static const auto phrases = base::flat_map>{ - { "group-call-join", tr::lng_group_call_join }, + { "iv-join-channel", tr::lng_iv_join_channel }, }; static const auto serialize = [](const style::color *color) { const auto qt = (*color)->c; From 5133f74f67c66ede2c9dc0fac861aa7fbc7934b7 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 10:56:02 +0400 Subject: [PATCH 009/596] Fix IV theming on macOS because of broken HTML. --- Telegram/SourceFiles/iv/iv_controller.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/SourceFiles/iv/iv_controller.cpp b/Telegram/SourceFiles/iv/iv_controller.cpp index 3d7a30318..1bafb769d 100644 --- a/Telegram/SourceFiles/iv/iv_controller.cpp +++ b/Telegram/SourceFiles/iv/iv_controller.cpp @@ -161,7 +161,7 @@ namespace { return R"( From 5c8e6c301228d456d433b8881fd351af92f371f6 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 10:56:24 +0400 Subject: [PATCH 010/596] Fix page encoding in IV on macOS. --- Telegram/SourceFiles/iv/iv_controller.cpp | 2 +- Telegram/SourceFiles/iv/iv_instance.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Telegram/SourceFiles/iv/iv_controller.cpp b/Telegram/SourceFiles/iv/iv_controller.cpp index 1bafb769d..67c423636 100644 --- a/Telegram/SourceFiles/iv/iv_controller.cpp +++ b/Telegram/SourceFiles/iv/iv_controller.cpp @@ -494,7 +494,7 @@ void Controller::createWebview(const QString &dataPath) { || index >= _pages.size()) { return Webview::DataResult::Failed; } - return finishWith(WrapPage(_pages[index]), "text/html"); + return finishWith(WrapPage(_pages[index]), "text/html; charset=utf-8"); } else if (id.starts_with("page") && id.ends_with(".json")) { auto index = 0; const auto result = std::from_chars( diff --git a/Telegram/SourceFiles/iv/iv_instance.cpp b/Telegram/SourceFiles/iv/iv_instance.cpp index d48cb85c8..49b945d5e 100644 --- a/Telegram/SourceFiles/iv/iv_instance.cpp +++ b/Telegram/SourceFiles/iv/iv_instance.cpp @@ -708,7 +708,7 @@ void Shown::streamMap(QString params, Webview::DataRequest request) { void Shown::sendEmbed(QByteArray hash, Webview::DataRequest request) { const auto i = _embeds.find(hash); if (i != end(_embeds)) { - requestDone(std::move(request), i->second, "text/html"); + requestDone(std::move(request), i->second, "text/html; charset=utf-8"); } else { requestFail(std::move(request)); } From 5c32423597d185ddef8832b152ce1a301e22a0f5 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 11:22:52 +0400 Subject: [PATCH 011/596] Integrate IV menu hiding by click inside WebView. --- Telegram/Resources/iv_html/page.css | 6 ++++++ Telegram/Resources/iv_html/page.js | 19 +++++++++++++++++++ Telegram/SourceFiles/iv/iv_controller.cpp | 19 ++++++++++++++++++- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/Telegram/Resources/iv_html/page.css b/Telegram/Resources/iv_html/page.css index 85a6124cc..0e0b3322a 100644 --- a/Telegram/Resources/iv_html/page.css +++ b/Telegram/Resources/iv_html/page.css @@ -88,6 +88,12 @@ html.custom_scroll ::-webkit-scrollbar-thumb:hover { opacity: 1; } } +#menu_page_blocker { + z-index: 999; + position: fixed; + width: 100%; + height: 100%; +} #top_shadow { z-index: 999; position: fixed; diff --git a/Telegram/Resources/iv_html/page.js b/Telegram/Resources/iv_html/page.js index efac9fa65..ac75d6d93 100644 --- a/Telegram/Resources/iv_html/page.js +++ b/Telegram/Resources/iv_html/page.js @@ -8,6 +8,11 @@ var IV = { var target = e.target; var context = ''; while (target) { + if (target.id == 'menu_page_blocker') { + IV.notify({ event: 'menu_page_blocker_click' }); + IV.menuShown(false); + return; + } if (target.tagName == 'AUDIO' || target.tagName == 'VIDEO') { return; } @@ -599,6 +604,20 @@ var IV = { back: function () { window.history.back(); }, + menuShown: function (shown) { + var already = document.getElementById('menu_page_blocker'); + if (already && shown) { + return; + } else if (already) { + document.body.removeChild(already); + return; + } else if (!shown) { + return; + } + var blocker = document.createElement('div'); + blocker.id = 'menu_page_blocker'; + document.body.appendChild(blocker); + }, videos: {}, videosPlaying: {}, diff --git a/Telegram/SourceFiles/iv/iv_controller.cpp b/Telegram/SourceFiles/iv/iv_controller.cpp index 67c423636..e24124ecd 100644 --- a/Telegram/SourceFiles/iv/iv_controller.cpp +++ b/Telegram/SourceFiles/iv/iv_controller.cpp @@ -429,6 +429,10 @@ void Controller::createWebview(const QString &dataPath) { const auto url = object.value("url").toString(); const auto context = object.value("context").toString(); processLink(url, context); + } else if (event == "menu_page_blocker_click") { + if (_menu) { + _menu->hideMenu(); + } } else if (event == u"ready"_q) { _ready = true; auto script = QByteArray(); @@ -443,6 +447,9 @@ void Controller::createWebview(const QString &dataPath) { if (base::take(_reloadInitialWhenReady)) { script += reloadScript(0); } + if (_menu) { + script += "IV.menuShown(true);"; + } if (!script.isEmpty()) { _webview->eval(script); } @@ -648,7 +655,7 @@ bool Controller::active() const { } void Controller::showJoinedTooltip() { - if (_webview) { + if (_webview && _ready) { _webview->eval("IV.showTooltip('" + EscapeForScriptString( tr::lng_action_you_joined(tr::now).toUtf8()) @@ -679,6 +686,9 @@ void Controller::showMenu() { _menu = base::make_unique_q( _window.get(), st::popupMenuWithIcons); + if (_webview && _ready) { + _webview->eval("IV.menuShown(true);"); + } _menu->setDestroyedCallback(crl::guard(_window.get(), [ this, weakButton = Ui::MakeWeak(_menuToggle.data()), @@ -686,6 +696,13 @@ void Controller::showMenu() { if (_menu == menu && weakButton) { weakButton->setForceRippled(false); } + if (const auto widget = _webview ? _webview->widget() : nullptr) { + InvokeQueued(widget, crl::guard(_window.get(), [=] { + if (_webview && _ready) { + _webview->eval("IV.menuShown(false);"); + } + })); + } })); _menuToggle->setForceRippled(true); From 782556495adae0894b57c4bf13a353588444545b Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 11:39:11 +0400 Subject: [PATCH 012/596] Improve joined channels tracking in IV. --- Telegram/Resources/iv_html/page.js | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Telegram/Resources/iv_html/page.js b/Telegram/Resources/iv_html/page.js index ac75d6d93..4b43286b7 100644 --- a/Telegram/Resources/iv_html/page.js +++ b/Telegram/Resources/iv_html/page.js @@ -127,12 +127,17 @@ var IV = { } }, toggleChannelJoined: function (id, joined) { + IV.channelsJoined['channel' + id] = joined; + IV.checkChannelButtons(); + }, + checkChannelButtons: function() { const channels = document.getElementsByClassName('channel'); - const full = 'channel' + id; for (var i = 0; i < channels.length; ++i) { const channel = channels[i]; - if (String(channel.getAttribute('data-context')) === full) { - channel.classList.toggle('joined', joined); + const full = String(channel.getAttribute('data-context')); + const value = IV.channelsJoined[full]; + if (value !== undefined) { + channel.classList.toggle('joined', value); } } }, @@ -419,6 +424,15 @@ var IV = { && (fromEl.getAttribute('data-src') == toEl.getAttribute('data-src'))) { return false; + } else if (fromEl.tagName == 'SECTION' + && fromEl.classList.contains('channel') + && fromEl.hasAttribute('data-context') + && toEl.tagName == 'SECTION' + && toEl.classList.contains('channel') + && toEl.hasAttribute('data-context') + && (String(fromEl.getAttribute('data-context')) + == String(toEl.getAttribute('data-context')))) { + return false; } else if (fromEl.classList.contains('loaded')) { toEl.classList.add('loaded'); } @@ -573,6 +587,7 @@ var IV = { } else { IV.initMedia(); } + IV.checkChannelButtons(); if (scroll === undefined) { IV.jumpToHash(hash, true); } else { @@ -623,6 +638,7 @@ var IV = { videosPlaying: {}, cache: {}, + channelsJoined: {}, index: 0, position: 0 }; From fe4b853393f1cf93874af35952e5f4a9e86d925f Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Wed, 13 Mar 2024 21:29:46 +0300 Subject: [PATCH 013/596] Improved section of scheduled messages when forum opened in flat mode. --- .../history/view/history_view_context_menu.cpp | 6 ++++-- .../SourceFiles/history/view/history_view_element.h | 1 + .../SourceFiles/history/view/history_view_message.cpp | 10 +++++----- .../history/view/history_view_scheduled_section.cpp | 11 ++++++++--- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Telegram/SourceFiles/history/view/history_view_context_menu.cpp b/Telegram/SourceFiles/history/view/history_view_context_menu.cpp index 09c3b89f9..5a7852b82 100644 --- a/Telegram/SourceFiles/history/view/history_view_context_menu.cpp +++ b/Telegram/SourceFiles/history/view/history_view_context_menu.cpp @@ -104,7 +104,8 @@ bool HasEditMessageAction( || !request.selectedItems.empty() || (context != Context::History && context != Context::Replies - && context != Context::ShortcutMessages)) { + && context != Context::ShortcutMessages + && context != Context::ScheduledTopic)) { return false; } const auto peer = item->history()->peer; @@ -1340,7 +1341,8 @@ void AddPollActions( } if ((context != Context::History) && (context != Context::Replies) - && (context != Context::Pinned)) { + && (context != Context::Pinned) + && (context != Context::ScheduledTopic)) { return; } if (poll->closed()) { diff --git a/Telegram/SourceFiles/history/view/history_view_element.h b/Telegram/SourceFiles/history/view/history_view_element.h index 5a6ea07d6..f47937516 100644 --- a/Telegram/SourceFiles/history/view/history_view_element.h +++ b/Telegram/SourceFiles/history/view/history_view_element.h @@ -60,6 +60,7 @@ enum class Context : char { SavedSublist, TTLViewer, ShortcutMessages, + ScheduledTopic, }; enum class OnlyEmojiAndSpaces : char { diff --git a/Telegram/SourceFiles/history/view/history_view_message.cpp b/Telegram/SourceFiles/history/view/history_view_message.cpp index 18c3dc57a..2e8f79f30 100644 --- a/Telegram/SourceFiles/history/view/history_view_message.cpp +++ b/Telegram/SourceFiles/history/view/history_view_message.cpp @@ -861,9 +861,7 @@ QSize Message::performCountOptimalSize() { void Message::refreshTopicButton() { const auto item = data(); - if (isAttachedToPrevious() - || (context() != Context::History) - || item->isScheduled()) { + if (isAttachedToPrevious() || context() != Context::History) { _topicButton = nullptr; } else if (const auto topic = item->topic()) { if (!_topicButton) { @@ -2044,7 +2042,8 @@ bool Message::hasFromPhoto() const { case Context::TTLViewer: case Context::Pinned: case Context::Replies: - case Context::SavedSublist: { + case Context::SavedSublist: + case Context::ScheduledTopic: { const auto item = data(); if (item->isPost()) { return false; @@ -3243,7 +3242,8 @@ bool Message::hasFromName() const { case Context::TTLViewer: case Context::Pinned: case Context::Replies: - case Context::SavedSublist: { + case Context::SavedSublist: + case Context::ScheduledTopic: { const auto item = data(); const auto peer = item->history()->peer; if (hasOutLayout() && !item->from()->isChannel()) { diff --git a/Telegram/SourceFiles/history/view/history_view_scheduled_section.cpp b/Telegram/SourceFiles/history/view/history_view_scheduled_section.cpp index c93b149eb..2a6328b38 100644 --- a/Telegram/SourceFiles/history/view/history_view_scheduled_section.cpp +++ b/Telegram/SourceFiles/history/view/history_view_scheduled_section.cpp @@ -252,7 +252,8 @@ void ScheduledWidget::setupComposeControls() { & ~ChatRestriction::SendPolls; const auto canSendAnything = Data::CanSendAnyOf( _history->peer, - allWithoutPolls); + allWithoutPolls, + false); const auto restriction = Data::RestrictionError( _history->peer, ChatRestriction::SendOther); @@ -650,7 +651,11 @@ void ScheduledWidget::send() { const auto error = GetErrorTextForSending( _history->peer, { - .topicRootId = _forumTopic ? _forumTopic->topicRootId() : MsgId(), + .topicRootId = _forumTopic + ? _forumTopic->topicRootId() + : history()->isForum() + ? MsgId(1) + : MsgId(), .forward = nullptr, .text = &textWithTags, .ignoreSlowmodeCountdown = true, @@ -1143,7 +1148,7 @@ QRect ScheduledWidget::floatPlayerAvailableRect() { } Context ScheduledWidget::listContext() { - return Context::History; + return _forumTopic ? Context::ScheduledTopic : Context::History; } bool ScheduledWidget::listScrollTo(int top, bool syntetic) { From d92418dc6d4030bafe5cc65b55facf85564a1d48 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 12:49:51 +0400 Subject: [PATCH 014/596] Disable media context menus in IV. --- Telegram/Resources/iv_html/page.js | 1 + Telegram/SourceFiles/iv/iv_prepare.cpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Telegram/Resources/iv_html/page.js b/Telegram/Resources/iv_html/page.js index 4b43286b7..26cab565c 100644 --- a/Telegram/Resources/iv_html/page.js +++ b/Telegram/Resources/iv_html/page.js @@ -330,6 +330,7 @@ var IV = { + '" type="video/mp4" />' + ''; var media = element.firstChild; + media.oncontextmenu = function () { return false; }; media.oncanplay = IV.checkVideos; media.onloadeddata = IV.checkVideos; } diff --git a/Telegram/SourceFiles/iv/iv_prepare.cpp b/Telegram/SourceFiles/iv/iv_prepare.cpp index e4e69b2de..9e276bce7 100644 --- a/Telegram/SourceFiles/iv/iv_prepare.cpp +++ b/Telegram/SourceFiles/iv/iv_prepare.cpp @@ -516,6 +516,7 @@ QByteArray Parser::block( const auto id = Number(photo.id); result = tag("a", { { "href", href }, + { "oncontextmenu", data.vurl() ? QByteArray() : "return false;" }, { "data-context", data.vurl() ? QByteArray() : "viewer-photo" + id }, }, result) + caption(data.vcaption()); return result; @@ -576,6 +577,7 @@ QByteArray Parser::block( const auto href = resource("video" + id); result = tag("a", { { "href", href }, + { "oncontextmenu", "return false;" }, { "data-context", "viewer-video" + id }, }, result); } @@ -745,6 +747,7 @@ QByteArray Parser::block(const MTPDpageBlockAudio &data) { const auto src = documentFullUrl(audio); return tag("figure", tag("audio", { { "src", src }, + { "oncontextmenu", "return false;" }, { "controls", std::nullopt }, }) + caption(data.vcaption())); } From f6b9702b5290407afbbdc6ca64e218bf691bd560 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 12:51:34 +0400 Subject: [PATCH 015/596] Unminimize IV when opening another one. --- Telegram/SourceFiles/iv/iv_controller.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Telegram/SourceFiles/iv/iv_controller.cpp b/Telegram/SourceFiles/iv/iv_controller.cpp index e24124ecd..494007872 100644 --- a/Telegram/SourceFiles/iv/iv_controller.cpp +++ b/Telegram/SourceFiles/iv/iv_controller.cpp @@ -577,6 +577,11 @@ void Controller::showInWindow(const QString &dataPath, Prepared page) { } void Controller::activate() { + if (_window->isMinimized()) { + _window->showNormal(); + } else if (_window->isHidden()) { + _window->show(); + } _window->raise(); _window->activateWindow(); _window->setFocus(); From a118d3d848591e5619037ace14f9ce68d9fd85f2 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 14:26:00 +0400 Subject: [PATCH 016/596] Close IV when opening URL in the browser. --- Telegram/SourceFiles/iv/iv_instance.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Telegram/SourceFiles/iv/iv_instance.cpp b/Telegram/SourceFiles/iv/iv_instance.cpp index 49b945d5e..673535cc7 100644 --- a/Telegram/SourceFiles/iv/iv_instance.cpp +++ b/Telegram/SourceFiles/iv/iv_instance.cpp @@ -833,6 +833,7 @@ void Instance::show( break; case Type::OpenLinkExternal: QDesktopServices::openUrl(event.url); + closeAll(); break; case Type::OpenMedia: if (const auto window = Core::App().activeWindow()) { From 396f52b8ca04515b9d9c0b4afd14443cf3c22fe3 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 14:27:46 +0400 Subject: [PATCH 017/596] Fix title updating on IV menu show. --- Telegram/lib_ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/lib_ui b/Telegram/lib_ui index 2687a19b1..0e175983d 160000 --- a/Telegram/lib_ui +++ b/Telegram/lib_ui @@ -1 +1 @@ -Subproject commit 2687a19b1435276e5a998d04b7d47df92e0c8391 +Subproject commit 0e175983d97605c20ee0321ca4485345536087b0 From 200bb31e0841b4d5c849ac979983dd836e230c25 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 15:01:03 +0400 Subject: [PATCH 018/596] Fix theming in code blocks. --- Telegram/Resources/iv_html/page.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/Resources/iv_html/page.css b/Telegram/Resources/iv_html/page.css index 0e0b3322a..ae3d64655 100644 --- a/Telegram/Resources/iv_html/page.css +++ b/Telegram/Resources/iv_html/page.css @@ -252,7 +252,7 @@ article pre.hljs { font-family: var(--font-mono); margin: 14px 0; padding: 7px 18px; - background: #F5F8FC; + background: var(--td-box-divider-bg); font-size: 16px; white-space: pre-wrap; word-wrap: break-word; From 73ca78215fa9340d520fc4eefa0c4e3749a283e5 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 18:35:57 +0400 Subject: [PATCH 019/596] Set window title in IV. --- Telegram/Resources/langs/lang.strings | 1 + Telegram/SourceFiles/iv/iv_controller.cpp | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Telegram/Resources/langs/lang.strings b/Telegram/Resources/langs/lang.strings index 437479980..39907f573 100644 --- a/Telegram/Resources/langs/lang.strings +++ b/Telegram/Resources/langs/lang.strings @@ -4870,6 +4870,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL "lng_iv_open_in_browser" = "Open in Browser"; "lng_iv_share" = "Share"; "lng_iv_join_channel" = "Join"; +"lng_iv_window_title" = "Instant View"; // Wnd specific diff --git a/Telegram/SourceFiles/iv/iv_controller.cpp b/Telegram/SourceFiles/iv/iv_controller.cpp index 494007872..d3f962641 100644 --- a/Telegram/SourceFiles/iv/iv_controller.cpp +++ b/Telegram/SourceFiles/iv/iv_controller.cpp @@ -244,14 +244,20 @@ void Controller::updateTitleGeometry(int newWidth) const { void Controller::initControls() { _subtitleWrap = std::make_unique(_window.get()); + _subtitleText = _index.value() | rpl::filter( + rpl::mappers::_1 >= 0 + ) | rpl::map([=](int index) { + return _pages[index].name; + }); _subtitle = std::make_unique( _subtitleWrap.get(), - _index.value() | rpl::filter( - rpl::mappers::_1 >= 0 - ) | rpl::map([=](int index) { - return _pages[index].name; - }), + _subtitleText.value(), st::ivSubtitle); + _subtitleText.value( + ) | rpl::start_with_next([=](const QString &subtitle) { + const auto prefix = tr::lng_iv_window_title(tr::now); + _window->setWindowTitle(prefix + ' ' + QChar(0x2014) + ' ' + subtitle); + }, _subtitle->lifetime()); _subtitle->setAttribute(Qt::WA_TransparentForMouseEvents); _menuToggle.create(_subtitleWrap.get(), st::ivMenuToggle); From 461aebd7f3ef6f193fdcdad8592e54323216ce36 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 18:36:17 +0400 Subject: [PATCH 020/596] Hide IV when sharing on macOS, fixes sharing. --- Telegram/SourceFiles/iv/iv_controller.cpp | 52 +++++++++++++++++------ Telegram/SourceFiles/iv/iv_controller.h | 1 + 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/Telegram/SourceFiles/iv/iv_controller.cpp b/Telegram/SourceFiles/iv/iv_controller.cpp index d3f962641..1671c59ba 100644 --- a/Telegram/SourceFiles/iv/iv_controller.cpp +++ b/Telegram/SourceFiles/iv/iv_controller.cpp @@ -762,10 +762,18 @@ void Controller::destroyShareMenu() { setInnerFocus(); } if (_shareWrap) { - _shareWrap->windowHandle()->setParent(nullptr); + if (_shareContainer) { + _shareWrap->windowHandle()->setParent(nullptr); + } _shareWrap = nullptr; _shareContainer = nullptr; } + if (_shareHidesContent) { + _shareHidesContent = false; + if (const auto content = _webview ? _webview->widget() : nullptr) { + content->show(); + } + } } void Controller::showShareMenu() { @@ -773,22 +781,33 @@ void Controller::showShareMenu() { if (_shareWrap || index < 0 || index > _pages.size()) { return; } + _shareHidesContent = Platform::IsMac(); + if (_shareHidesContent) { + if (const auto content = _webview ? _webview->widget() : nullptr) { + content->hide(); + } + } - _shareWrap = std::make_unique(nullptr); + _shareWrap = std::make_unique(_shareHidesContent + ? _window->window() + : nullptr); const auto margins = QMargins(0, st::windowTitleHeight, 0, 0); - _shareWrap->setGeometry(_window->geometry().marginsRemoved(margins)); - _shareWrap->setWindowFlag(Qt::FramelessWindowHint); - _shareWrap->setAttribute(Qt::WA_TranslucentBackground); - _shareWrap->setAttribute(Qt::WA_NoSystemBackground); - _shareWrap->createWinId(); + if (!_shareHidesContent) { + _shareWrap->setWindowFlag(Qt::FramelessWindowHint); + _shareWrap->setAttribute(Qt::WA_TranslucentBackground); + _shareWrap->setAttribute(Qt::WA_NoSystemBackground); + _shareWrap->createWinId(); - _shareContainer.reset(QWidget::createWindowContainer( - _shareWrap->windowHandle(), - _window.get(), - Qt::FramelessWindowHint | Qt::WindowStaysOnTopHint)); + _shareContainer.reset(QWidget::createWindowContainer( + _shareWrap->windowHandle(), + _window.get(), + Qt::FramelessWindowHint | Qt::WindowStaysOnTopHint)); + } _window->sizeValue() | rpl::start_with_next([=](QSize size) { - _shareContainer->setGeometry(QRect(QPoint(), size).marginsRemoved( - margins)); + const auto widget = _shareHidesContent + ? _shareWrap.get() + : _shareContainer.get(); + widget->setGeometry(QRect(QPoint(), size).marginsRemoved(margins)); }, _shareWrap->lifetime()); auto result = _showShareBox({ @@ -803,7 +822,12 @@ void Controller::showShareMenu() { }, _shareWrap->lifetime()); Ui::ForceFullRepaintSync(_shareWrap.get()); - _shareContainer->show(); + + if (_shareHidesContent) { + _shareWrap->show(); + } else { + _shareContainer->show(); + } activate(); } diff --git a/Telegram/SourceFiles/iv/iv_controller.h b/Telegram/SourceFiles/iv/iv_controller.h index 76e53d6e5..f8352f0d3 100644 --- a/Telegram/SourceFiles/iv/iv_controller.h +++ b/Telegram/SourceFiles/iv/iv_controller.h @@ -142,6 +142,7 @@ private: std::unique_ptr _shareContainer; Fn _shareFocus; Fn _shareHide; + bool _shareHidesContent = false; std::vector _pages; base::flat_map _indices; From 2e7323d484ade5577d7d59df3a8271d90883f5a2 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 19:03:46 +0400 Subject: [PATCH 021/596] Fix a glitch in IV sharing on Windows. --- Telegram/SourceFiles/iv/iv_controller.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Telegram/SourceFiles/iv/iv_controller.cpp b/Telegram/SourceFiles/iv/iv_controller.cpp index 1671c59ba..66eef977d 100644 --- a/Telegram/SourceFiles/iv/iv_controller.cpp +++ b/Telegram/SourceFiles/iv/iv_controller.cpp @@ -793,6 +793,7 @@ void Controller::showShareMenu() { : nullptr); const auto margins = QMargins(0, st::windowTitleHeight, 0, 0); if (!_shareHidesContent) { + _shareWrap->setGeometry(_window->geometry().marginsRemoved(margins)); _shareWrap->setWindowFlag(Qt::FramelessWindowHint); _shareWrap->setAttribute(Qt::WA_TranslucentBackground); _shareWrap->setAttribute(Qt::WA_NoSystemBackground); From 6c083e88a3cf1f8c0232a7d4ead4770b96915ff9 Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Thu, 14 Mar 2024 20:05:17 +0300 Subject: [PATCH 022/596] Changed behavior of bot menu button to decrease size to first emoji. --- .../SourceFiles/history/history_widget.cpp | 84 ++++++++++++------- Telegram/SourceFiles/history/history_widget.h | 7 +- 2 files changed, 60 insertions(+), 31 deletions(-) diff --git a/Telegram/SourceFiles/history/history_widget.cpp b/Telegram/SourceFiles/history/history_widget.cpp index b480d1dea..e759ea5f8 100644 --- a/Telegram/SourceFiles/history/history_widget.cpp +++ b/Telegram/SourceFiles/history/history_widget.cpp @@ -203,6 +203,18 @@ const auto kPsaAboutPrefix = "cloud_lng_about_psa_"; }); } +[[nodiscard]] QString FirstEmoji(const QString &s) { + const auto begin = s.data(); + const auto end = begin + s.size(); + for (auto ch = begin; ch != end; ch++) { + auto length = 0; + if (const auto e = Ui::Emoji::Find(ch, end, &length)) { + return e->text(); + } + } + return QString(); +} + } // namespace HistoryWidget::HistoryWidget( @@ -2908,8 +2920,8 @@ void HistoryWidget::updateControlsVisibility() { _botKeyboardShow->hide(); _botKeyboardHide->hide(); _botCommandStart->hide(); - if (_botMenuButton) { - _botMenuButton->hide(); + if (_botMenu.button) { + _botMenu.button->hide(); } if (_tabbedPanel) { _tabbedPanel->hide(); @@ -2977,8 +2989,8 @@ void HistoryWidget::updateControlsVisibility() { } else { _attachToggle->show(); } - if (_botMenuButton) { - _botMenuButton->show(); + if (_botMenu.button) { + _botMenu.button->show(); } if (_sendRestriction) { _sendRestriction->hide(); @@ -3055,8 +3067,8 @@ void HistoryWidget::updateControlsVisibility() { if (_sendAs) { _sendAs->hide(); } - if (_botMenuButton) { - _botMenuButton->hide(); + if (_botMenu.button) { + _botMenu.button->hide(); } _kbScroll->hide(); if (_replyTo || readyToForward() || _kbReplyTo) { @@ -4803,28 +4815,35 @@ bool HistoryWidget::updateCmdStartShown() { } } } + constexpr auto kSmallMenuAfter = 10; const auto commandsChanged = (_cmdStartShown != cmdStartShown); auto buttonChanged = false; if (!bot || (bot->botInfo->botMenuButtonUrl.isEmpty() && bot->botInfo->commands.empty())) { - buttonChanged = (_botMenuButton != nullptr); - _botMenuButton.destroy(); - } else if (!_botMenuButton) { + buttonChanged = (_botMenu.button != nullptr); + _botMenu.button.destroy(); + } else if (!_botMenu.button) { buttonChanged = true; - _botMenuButtonText = bot->botInfo->botMenuButtonText; - _botMenuButton.create( + _botMenu.text = bot->botInfo->botMenuButtonText; + _botMenu.small = (Ui::FieldCharacterCount(_field) > kSmallMenuAfter); + if (_botMenu.small) { + if (const auto e = FirstEmoji(_botMenu.text); !e.isEmpty()) { + _botMenu.text = e; + } + } + _botMenu.button.create( this, - (_botMenuButtonText.isEmpty() + (_botMenu.text.isEmpty() ? tr::lng_bot_menu_button() - : rpl::single(_botMenuButtonText)), + : rpl::single(_botMenu.text)), st::historyBotMenuButton); orderWidgets(); - _botMenuButton->setTextTransform( + _botMenu.button->setTextTransform( Ui::RoundButton::TextTransform::NoTransform); - _botMenuButton->setFullRadius(true); - _botMenuButton->setClickedCallback([=] { + _botMenu.button->setFullRadius(true); + _botMenu.button->setClickedCallback([=] { const auto user = _peer ? _peer->asUser() : nullptr; const auto bot = (user && user->isBot()) ? user : nullptr; if (bot && !bot->botInfo->botMenuButtonUrl.isEmpty()) { @@ -4835,22 +4854,29 @@ bool HistoryWidget::updateCmdStartShown() { _fieldAutocomplete->showFiltered(_peer, "/", true); } }); - _botMenuButton->widthValue( + _botMenu.button->widthValue( ) | rpl::start_with_next([=](int width) { if (width > st::historyBotMenuMaxWidth) { - _botMenuButton->setFullWidth(st::historyBotMenuMaxWidth); + _botMenu.button->setFullWidth(st::historyBotMenuMaxWidth); } else { updateFieldSize(); } - }, _botMenuButton->lifetime()); + }, _botMenu.button->lifetime()); } - const auto textChanged = _botMenuButton - && (_botMenuButtonText != bot->botInfo->botMenuButtonText); + const auto textSmall = Ui::FieldCharacterCount(_field) > kSmallMenuAfter; + const auto textChanged = _botMenu.button + && ((_botMenu.text != bot->botInfo->botMenuButtonText) + || (_botMenu.small != textSmall)); if (textChanged) { - _botMenuButtonText = bot->botInfo->botMenuButtonText; - _botMenuButton->setText(_botMenuButtonText.isEmpty() + _botMenu.text = bot->botInfo->botMenuButtonText; + if (_botMenu.small = textSmall) { + if (const auto e = FirstEmoji(_botMenu.text); !e.isEmpty()) { + _botMenu.text = e; + } + } + _botMenu.button->setText(_botMenu.text.isEmpty() ? tr::lng_bot_menu_button() - : rpl::single(_botMenuButtonText)); + : rpl::single(_botMenu.text)); } _cmdStartShown = cmdStartShown; return commandsChanged || buttonChanged || textChanged; @@ -5166,15 +5192,15 @@ void HistoryWidget::moveFieldControls() { _kbScroll->setGeometryToLeft(0, bottom, width(), keyboardHeight); } -// (_botMenuButton) (_attachToggle|_replaceMedia) (_sendAs) ---- _inlineResults ------------------------------ _tabbedPanel ------ _fieldBarCancel +// (_botMenu.button) (_attachToggle|_replaceMedia) (_sendAs) ---- _inlineResults ------------------------------ _tabbedPanel ------ _fieldBarCancel // (_attachDocument|_attachPhoto) _field (_ttlInfo) (_scheduled) (_silent|_cmdStart|_kbShow) (_kbHide|_tabbedSelectorToggle) _send // (_botStart|_unblock|_joinChannel|_muteUnmute|_reportMessages) auto buttonsBottom = bottom - _attachToggle->height(); auto left = st::historySendRight; - if (_botMenuButton) { + if (_botMenu.button) { const auto skip = st::historyBotMenuSkip; - _botMenuButton->moveToLeft(left + skip, buttonsBottom + skip); left += skip + _botMenuButton->width(); + _botMenu.button->moveToLeft(left + skip, buttonsBottom + skip); left += skip + _botMenu.button->width(); } if (_replaceMedia) { _replaceMedia->moveToLeft(left, buttonsBottom); @@ -5246,8 +5272,8 @@ void HistoryWidget::updateFieldSize() { - st::historySendRight - _send->width() - _tabbedSelectorToggle->width(); - if (_botMenuButton) { - fieldWidth -= st::historyBotMenuSkip + _botMenuButton->width(); + if (_botMenu.button) { + fieldWidth -= st::historyBotMenuSkip + _botMenu.button->width(); } if (_sendAs) { fieldWidth -= _sendAs->width(); diff --git a/Telegram/SourceFiles/history/history_widget.h b/Telegram/SourceFiles/history/history_widget.h index e4ec63ea2..4cbfd46af 100644 --- a/Telegram/SourceFiles/history/history_widget.h +++ b/Telegram/SourceFiles/history/history_widget.h @@ -748,8 +748,11 @@ private: object_ptr _joinChannel; object_ptr _muteUnmute; object_ptr _reportMessages; - object_ptr _botMenuButton = { nullptr }; - QString _botMenuButtonText; + struct { + object_ptr button = { nullptr }; + QString text; + bool small = false; + } _botMenu; object_ptr _attachToggle; object_ptr _replaceMedia = { nullptr }; object_ptr _sendAs = { nullptr }; From 1f0ed3bb315eea05d97107cb8b65c76a520a13e6 Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Thu, 14 Mar 2024 20:36:19 +0300 Subject: [PATCH 023/596] Fixed links in service message about requested peers from bot. --- Telegram/SourceFiles/history/history_item.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Telegram/SourceFiles/history/history_item.cpp b/Telegram/SourceFiles/history/history_item.cpp index d990c56ef..43bebb226 100644 --- a/Telegram/SourceFiles/history/history_item.cpp +++ b/Telegram/SourceFiles/history/history_item.cpp @@ -4628,7 +4628,7 @@ void HistoryItem::setServiceMessageByAction(const MTPmessageAction &action) { auto prepareRequestedPeer = [&]( const MTPDmessageActionRequestedPeer &action) { auto result = PreparedServiceText{}; - result.links.push_back(fromLink()); + result.links.push_back(history()->peer->createOpenLink()); const auto &list = action.vpeers().v; for (auto i = 0, count = int(list.size()); i != count; ++i) { @@ -4664,7 +4664,7 @@ void HistoryItem::setServiceMessageByAction(const MTPmessageAction &action) { lt_chat, result.text, lt_bot, - Ui::Text::Link(history()->peer->name(), 2), + Ui::Text::Link(history()->peer->name(), 1), Ui::Text::WithEntities); return result; }; From 5a5d89c32ced14a1e401a09b561a99166989e6ab Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 20:29:08 +0400 Subject: [PATCH 024/596] Fix crashing regression in Qt 6.7.0. --- Telegram/build/docker/centos_env/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/build/docker/centos_env/Dockerfile b/Telegram/build/docker/centos_env/Dockerfile index 274b4919a..ec9e9a3ad 100644 --- a/Telegram/build/docker/centos_env/Dockerfile +++ b/Telegram/build/docker/centos_env/Dockerfile @@ -54,7 +54,7 @@ FROM builder AS patches RUN git init patches \ && cd patches \ && git remote add origin {{ GIT }}/desktop-app/patches.git \ - && git fetch --depth=1 origin cc0c2f83650bbd52064bb7bc7257c950822cce5b \ + && git fetch --depth=1 origin b8390aef3db43f973486c6e512f4e825ffd09691 \ && git reset --hard FETCH_HEAD \ && rm -rf .git From b955d3297036e9afd67fa75b50d833b3c2935ab2 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 21:55:16 +0400 Subject: [PATCH 025/596] Beta version 4.15.14. - Fix initial text color in dark mode Instant View on macOS. - Fix non-English symbols encoding in Instant View on macOS. - Fix sharing from Instant View on macOS. - Fix crash with long messages on Linux. --- Telegram/Resources/uwp/AppX/AppxManifest.xml | 2 +- Telegram/Resources/winrc/Telegram.rc | 8 ++++---- Telegram/Resources/winrc/Updater.rc | 8 ++++---- Telegram/SourceFiles/core/version.h | 4 ++-- Telegram/build/version | 8 ++++---- changelog.txt | 7 +++++++ 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/Telegram/Resources/uwp/AppX/AppxManifest.xml b/Telegram/Resources/uwp/AppX/AppxManifest.xml index 466034289..e6aa01dc5 100644 --- a/Telegram/Resources/uwp/AppX/AppxManifest.xml +++ b/Telegram/Resources/uwp/AppX/AppxManifest.xml @@ -10,7 +10,7 @@ + Version="4.15.4.0" /> Telegram Desktop Telegram Messenger LLP diff --git a/Telegram/Resources/winrc/Telegram.rc b/Telegram/Resources/winrc/Telegram.rc index fda1d34da..fe9addb7b 100644 --- a/Telegram/Resources/winrc/Telegram.rc +++ b/Telegram/Resources/winrc/Telegram.rc @@ -44,8 +44,8 @@ IDI_ICON1 ICON "..\\art\\icon256.ico" // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,15,3,0 - PRODUCTVERSION 4,15,3,0 + FILEVERSION 4,15,4,0 + PRODUCTVERSION 4,15,4,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -62,10 +62,10 @@ BEGIN BEGIN VALUE "CompanyName", "Telegram FZ-LLC" VALUE "FileDescription", "Telegram Desktop" - VALUE "FileVersion", "4.15.3.0" + VALUE "FileVersion", "4.15.4.0" VALUE "LegalCopyright", "Copyright (C) 2014-2024" VALUE "ProductName", "Telegram Desktop" - VALUE "ProductVersion", "4.15.3.0" + VALUE "ProductVersion", "4.15.4.0" END END BLOCK "VarFileInfo" diff --git a/Telegram/Resources/winrc/Updater.rc b/Telegram/Resources/winrc/Updater.rc index 8ebf2c68b..1289e471e 100644 --- a/Telegram/Resources/winrc/Updater.rc +++ b/Telegram/Resources/winrc/Updater.rc @@ -35,8 +35,8 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,15,3,0 - PRODUCTVERSION 4,15,3,0 + FILEVERSION 4,15,4,0 + PRODUCTVERSION 4,15,4,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -53,10 +53,10 @@ BEGIN BEGIN VALUE "CompanyName", "Telegram FZ-LLC" VALUE "FileDescription", "Telegram Desktop Updater" - VALUE "FileVersion", "4.15.3.0" + VALUE "FileVersion", "4.15.4.0" VALUE "LegalCopyright", "Copyright (C) 2014-2024" VALUE "ProductName", "Telegram Desktop" - VALUE "ProductVersion", "4.15.3.0" + VALUE "ProductVersion", "4.15.4.0" END END BLOCK "VarFileInfo" diff --git a/Telegram/SourceFiles/core/version.h b/Telegram/SourceFiles/core/version.h index 933b976d6..cc1bff4ac 100644 --- a/Telegram/SourceFiles/core/version.h +++ b/Telegram/SourceFiles/core/version.h @@ -22,7 +22,7 @@ constexpr auto AppId = "{53F49750-6209-4FBF-9CA8-7A333C87D1ED}"_cs; constexpr auto AppNameOld = "Telegram Win (Unofficial)"_cs; constexpr auto AppName = "Telegram Desktop"_cs; constexpr auto AppFile = "Telegram"_cs; -constexpr auto AppVersion = 4015003; -constexpr auto AppVersionStr = "4.15.3"; +constexpr auto AppVersion = 4015004; +constexpr auto AppVersionStr = "4.15.4"; constexpr auto AppBetaVersion = true; constexpr auto AppAlphaVersion = TDESKTOP_ALPHA_VERSION; diff --git a/Telegram/build/version b/Telegram/build/version index 2ccb686af..d1f6650e1 100644 --- a/Telegram/build/version +++ b/Telegram/build/version @@ -1,7 +1,7 @@ -AppVersion 4015003 +AppVersion 4015004 AppVersionStrMajor 4.15 -AppVersionStrSmall 4.15.3 -AppVersionStr 4.15.3 +AppVersionStrSmall 4.15.4 +AppVersionStr 4.15.4 BetaChannel 1 AlphaVersion 0 -AppVersionOriginal 4.15.3.beta +AppVersionOriginal 4.15.4.beta diff --git a/changelog.txt b/changelog.txt index 99986b155..62f49bf5e 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,10 @@ +4.15.4 beta (14.03.24) + +- Fix initial text color in dark mode Instant View on macOS. +- Fix non-English symbols encoding in Instant View on macOS. +- Fix sharing from Instant View on macOS. +- Fix crash with long messages on Linux. + 4.15.3 beta (13.03.24) - Instant View on Windows (with WebView2) and macOS. From 20db635366a38f55a7df13ad53f58e86ae2c5332 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 21:58:56 +0400 Subject: [PATCH 026/596] Beta version 4.15.4: Update Qt patches in snap. --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index a4ef1f060..b309c3d11 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -167,7 +167,7 @@ parts: patches: source: https://github.com/desktop-app/patches.git source-depth: 1 - source-commit: cc0c2f83650bbd52064bb7bc7257c950822cce5b + source-commit: b8390aef3db43f973486c6e512f4e825ffd09691 plugin: dump override-pull: | craftctl default From f2a54e3cbbe7d70b795f64713c030734a44a739b Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 14 Mar 2024 22:30:52 +0400 Subject: [PATCH 027/596] Beta version 4.15.4: Fix build with Xcode. --- Telegram/SourceFiles/history/history_widget.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/SourceFiles/history/history_widget.cpp b/Telegram/SourceFiles/history/history_widget.cpp index e759ea5f8..8aa8f7332 100644 --- a/Telegram/SourceFiles/history/history_widget.cpp +++ b/Telegram/SourceFiles/history/history_widget.cpp @@ -4869,7 +4869,7 @@ bool HistoryWidget::updateCmdStartShown() { || (_botMenu.small != textSmall)); if (textChanged) { _botMenu.text = bot->botInfo->botMenuButtonText; - if (_botMenu.small = textSmall) { + if ((_botMenu.small = textSmall)) { if (const auto e = FirstEmoji(_botMenu.text); !e.isEmpty()) { _botMenu.text = e; } From 59e53c1edfc031f98075aa0fa05374232732ea3b Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Thu, 14 Mar 2024 21:49:25 +0400 Subject: [PATCH 028/596] Port NotificationId conversion to cppgir --- .../platform/linux/integration_linux.cpp | 74 +++++++++---------- .../linux/notifications_manager_linux.cpp | 31 +++++++- .../window/notifications_manager.h | 37 +++++----- 3 files changed, 80 insertions(+), 62 deletions(-) diff --git a/Telegram/SourceFiles/platform/linux/integration_linux.cpp b/Telegram/SourceFiles/platform/linux/integration_linux.cpp index fef646b20..929ed15b7 100644 --- a/Telegram/SourceFiles/platform/linux/integration_linux.cpp +++ b/Telegram/SourceFiles/platform/linux/integration_linux.cpp @@ -19,7 +19,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include #include -#include #include #include @@ -28,6 +27,32 @@ namespace { using namespace gi::repository; +std::vector AnyVectorFromVariant(GLib::Variant value) { + std::vector result; + + auto iter = gi::wrap( + g_variant_iter_new(value.gobj_()), + gi::transfer_full); + + const auto uint64Type = GLib::VariantType::new_("t"); + const auto int64Type = GLib::VariantType::new_("x"); + + while (auto value = iter.next_value()) { + value = value.get_variant(); + if (value.is_of_type(uint64Type)) { + result.push_back(std::make_any(value.get_uint64())); + } else if (value.is_of_type(int64Type)) { + result.push_back(std::make_any(value.get_int64())); + } else if (value.is_container()) { + result.push_back( + std::make_any>( + AnyVectorFromVariant(value))); + } + } + + return result; +} + class Application : public Gio::impl::ApplicationImpl { public: Application(); @@ -102,23 +127,8 @@ Application::Application() using Window::Notifications::Manager; using NotificationId = Manager::NotificationId; - using NotificationIdTuple = std::invoke_result_t< - decltype(&NotificationId::toTuple), - NotificationId* - >; - const auto notificationIdVariantType = [] { - try { - return gi::wrap( - Glib::create_variant( - NotificationId().toTuple() - ).get_type().gobj_copy(), - gi::transfer_full - ); - } catch (...) { - return GLib::VariantType(); - } - }(); + const auto notificationIdVariantType = GLib::VariantType::new_("av"); auto notificationActivateAction = Gio::SimpleAction::new_( "notification-activate", @@ -128,17 +138,9 @@ Application::Application() Gio::SimpleAction, GLib::Variant parameter) { Core::Sandbox::Instance().customEnterFromEventLoop([&] { - try { - const auto &app = Core::App(); - app.notifications().manager().notificationActivated( - NotificationId::FromTuple( - Glib::wrap( - parameter.gobj_copy_() - ).get_dynamic() - ) - ); - } catch (...) { - } + Core::App().notifications().manager().notificationActivated( + NotificationId::FromAnyVector( + AnyVectorFromVariant(parameter))); }); }); @@ -152,18 +154,10 @@ Application::Application() Gio::SimpleAction, GLib::Variant parameter) { Core::Sandbox::Instance().customEnterFromEventLoop([&] { - try { - const auto &app = Core::App(); - app.notifications().manager().notificationReplied( - NotificationId::FromTuple( - Glib::wrap( - parameter.gobj_copy_() - ).get_dynamic() - ), - {} - ); - } catch (...) { - } + Core::App().notifications().manager().notificationReplied( + NotificationId::FromAnyVector( + AnyVectorFromVariant(parameter)), + {}); }); }); diff --git a/Telegram/SourceFiles/platform/linux/notifications_manager_linux.cpp b/Telegram/SourceFiles/platform/linux/notifications_manager_linux.cpp index aed48c948..34e9a07eb 100644 --- a/Telegram/SourceFiles/platform/linux/notifications_manager_linux.cpp +++ b/Telegram/SourceFiles/platform/linux/notifications_manager_linux.cpp @@ -28,7 +28,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include #include -#include #include #include @@ -139,6 +138,32 @@ bool UseGNotification() { return KSandbox::isFlatpak() && !ServiceRegistered; } +GLib::Variant AnyVectorToVariant(const std::vector &value) { + return GLib::Variant::new_array( + value | ranges::views::transform([](const std::any &value) { + try { + return GLib::Variant::new_variant( + GLib::Variant::new_uint64(std::any_cast(value))); + } catch (...) { + } + + try { + return GLib::Variant::new_variant( + GLib::Variant::new_int64(std::any_cast(value))); + } catch (...) { + } + + try { + return GLib::Variant::new_variant( + AnyVectorToVariant( + std::any_cast>(value))); + } catch (...) { + } + + return GLib::Variant(nullptr); + }) | ranges::to_vector); +} + class NotificationData final : public base::has_weak_ptr { public: using NotificationId = Window::Notifications::Manager::NotificationId; @@ -239,9 +264,7 @@ bool NotificationData::init( set_category(_notification.gobj_(), "im.received"); } - const auto idVariant = gi::wrap( - Glib::create_variant(_id.toTuple()).gobj_copy(), - gi::transfer_full); + const auto idVariant = AnyVectorToVariant(_id.toAnyVector()); _notification.set_default_action_and_target( "app.notification-activate", diff --git a/Telegram/SourceFiles/window/notifications_manager.h b/Telegram/SourceFiles/window/notifications_manager.h index 9d7224c74..a8db13833 100644 --- a/Telegram/SourceFiles/window/notifications_manager.h +++ b/Telegram/SourceFiles/window/notifications_manager.h @@ -233,19 +233,19 @@ public: const ContextId&, const ContextId&) = default; - [[nodiscard]] auto toTuple() const { - return std::make_tuple( - sessionId, - peerId.value, - topicRootId.bare); + [[nodiscard]] auto toAnyVector() const { + return std::vector{ + std::make_any(sessionId), + std::make_any(peerId.value), + std::make_any(topicRootId.bare), + }; } - template - [[nodiscard]] static auto FromTuple(const T &tuple) { + [[nodiscard]] static auto FromAnyVector(const auto &vector) { return ContextId{ - std::get<0>(tuple), - PeerIdHelper(std::get<1>(tuple)), - std::get<2>(tuple), + std::any_cast(vector[0]), + PeerIdHelper(std::any_cast(vector[1])), + std::any_cast(vector[2]), }; } }; @@ -257,17 +257,18 @@ public: const NotificationId&, const NotificationId&) = default; - [[nodiscard]] auto toTuple() const { - return std::make_tuple( - contextId.toTuple(), - msgId.bare); + [[nodiscard]] auto toAnyVector() const { + return std::vector{ + std::make_any>(contextId.toAnyVector()), + std::make_any(msgId.bare), + }; } - template - [[nodiscard]] static auto FromTuple(const T &tuple) { + [[nodiscard]] static auto FromAnyVector(const auto &vector) { return NotificationId{ - ContextId::FromTuple(std::get<0>(tuple)), - std::get<1>(tuple), + ContextId::FromAnyVector( + std::any_cast>(vector[0])), + std::any_cast(vector[1]), }; } }; From 3689e7dfbca149afc4251de9e09d2ecf211705b5 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Thu, 14 Mar 2024 21:50:18 +0400 Subject: [PATCH 029/596] Use cppgir compatible XDP utilities API --- Telegram/SourceFiles/platform/linux/integration_linux.cpp | 4 ++-- Telegram/SourceFiles/platform/linux/specific_linux.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Telegram/SourceFiles/platform/linux/integration_linux.cpp b/Telegram/SourceFiles/platform/linux/integration_linux.cpp index 929ed15b7..38e5cc7f2 100644 --- a/Telegram/SourceFiles/platform/linux/integration_linux.cpp +++ b/Telegram/SourceFiles/platform/linux/integration_linux.cpp @@ -197,12 +197,12 @@ LinuxIntegration::LinuxIntegration() , _darkModeWatcher( "org.freedesktop.appearance", "color-scheme", - [](uint value) { + [](GLib::Variant value) { #if QT_VERSION >= QT_VERSION_CHECK(6, 5, 0) QWindowSystemInterface::handleThemeChange(); #else // Qt >= 6.5.0 Core::Sandbox::Instance().customEnterFromEventLoop([&] { - Core::App().settings().setSystemDarkMode(value == 1); + Core::App().settings().setSystemDarkMode(value.get_uint32() == 1); }); #endif // Qt < 6.5.0 }) { diff --git a/Telegram/SourceFiles/platform/linux/specific_linux.cpp b/Telegram/SourceFiles/platform/linux/specific_linux.cpp index 750c4dc5f..c6f9390e7 100644 --- a/Telegram/SourceFiles/platform/linux/specific_linux.cpp +++ b/Telegram/SourceFiles/platform/linux/specific_linux.cpp @@ -512,12 +512,12 @@ QString SingleInstanceLocalServerName(const QString &hash) { #if QT_VERSION < QT_VERSION_CHECK(6, 5, 0) std::optional IsDarkMode() { - const auto result = base::Platform::XDP::ReadSetting( + const auto result = base::Platform::XDP::ReadSetting( "org.freedesktop.appearance", "color-scheme"); return result.has_value() - ? std::make_optional(*result == 1) + ? std::make_optional(result->get_uint32() == 1) : std::nullopt; } #endif // Qt < 6.5.0 From d3715cc3624046d34580b2944d88cea4d9d89576 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Thu, 14 Mar 2024 21:50:58 +0400 Subject: [PATCH 030/596] Get rid of glibmm --- CMakeLists.txt | 1 + Telegram/CMakeLists.txt | 5 -- .../platform/linux/specific_linux.cpp | 4 -- Telegram/build/docker/centos_env/Dockerfile | 22 ------ docs/building-linux.md | 3 - snap/snapcraft.yaml | 68 ++----------------- 6 files changed, 7 insertions(+), 96 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dcf88602f..3bf13693b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,7 @@ endif() include(cmake/external/qt/package.cmake) set(desktop_app_skip_libs + glibmm variant ) diff --git a/Telegram/CMakeLists.txt b/Telegram/CMakeLists.txt index 5d30acc96..46e061a09 100644 --- a/Telegram/CMakeLists.txt +++ b/Telegram/CMakeLists.txt @@ -1679,11 +1679,6 @@ elseif (APPLE) ) endif() else() - target_link_libraries(Telegram - PRIVATE - desktop-app::external_glibmm - ) - include(${cmake_helpers_loc}/external/glib/generate_dbus.cmake) generate_dbus(Telegram org.freedesktop.portal. XdpBackground ${third_party_loc}/xdg-desktop-portal/data/org.freedesktop.portal.Background.xml) generate_dbus(Telegram org.freedesktop. XdgNotifications ${src_loc}/platform/linux/org.freedesktop.Notifications.xml) diff --git a/Telegram/SourceFiles/platform/linux/specific_linux.cpp b/Telegram/SourceFiles/platform/linux/specific_linux.cpp index c6f9390e7..8d662cdc4 100644 --- a/Telegram/SourceFiles/platform/linux/specific_linux.cpp +++ b/Telegram/SourceFiles/platform/linux/specific_linux.cpp @@ -35,8 +35,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include #include -#include - #include #include #include @@ -692,8 +690,6 @@ void start() { GLib::set_prgname(cExeName().toStdString()); GLib::set_application_name(AppName.data()); - Glib::init(); - Webview::WebKitGTK::SetSocketPath(u"%1/%2-%3-webview-%4"_q.arg( QDir::tempPath(), h, diff --git a/Telegram/build/docker/centos_env/Dockerfile b/Telegram/build/docker/centos_env/Dockerfile index ec9e9a3ad..9ff771f41 100644 --- a/Telegram/build/docker/centos_env/Dockerfile +++ b/Telegram/build/docker/centos_env/Dockerfile @@ -705,27 +705,6 @@ RUN git clone -b 2.78.1 --depth=1 {{ GIT }}/GNOME/glib.git \ && cd .. \ && rm -rf glib -FROM patches AS glibmm -COPY --link --from=glib {{ LibrariesPath }}/glib-cache / - -RUN git clone -b 2.78.0 --depth=1 {{ GIT }}/GNOME/glibmm.git \ - && cd glibmm \ - && git apply ../patches/glibmm.patch \ - && CFLAGS="$CFLAGS {{ CFLAGS_LTO }}" CXXFLAGS="$CXXFLAGS {{ CFLAGS_LTO }}" meson build \ - --buildtype=plain \ - --default-library=both \ - -Dbuild-documentation=false \ - -Dbuild-examples=false \ - -Dsigc++-3.0:build-documentation=false \ - -Dsigc++-3.0:build-examples=false \ - -Dsigc++-3.0:build-tests=false \ - -Dsigc++-3.0:validation=false \ - -Dmm-common:use-network=true \ - && meson compile -C build \ - && DESTDIR="{{ LibrariesPath }}/glibmm-cache" meson install -C build \ - && cd .. \ - && rm -rf glibmm - FROM builder AS gobject-introspection COPY --link --from=glib {{ LibrariesPath }}/glib-cache / @@ -873,7 +852,6 @@ COPY --link --from=openal {{ LibrariesPath }}/openal-cache / COPY --link --from=openssl {{ LibrariesPath }}/openssl-cache / COPY --link --from=xkbcommon {{ LibrariesPath }}/xkbcommon-cache / COPY --link --from=glib {{ LibrariesPath }}/glib-cache / -COPY --link --from=glibmm {{ LibrariesPath }}/glibmm-cache / COPY --link --from=gobject-introspection {{ LibrariesPath }}/gobject-introspection-cache / COPY --link --from=qt {{ LibrariesPath }}/qt-cache / COPY --link --from=breakpad {{ LibrariesPath }}/breakpad-cache / diff --git a/docs/building-linux.md b/docs/building-linux.md index 4fa4503b5..393bd854a 100644 --- a/docs/building-linux.md +++ b/docs/building-linux.md @@ -36,9 +36,6 @@ Or, to create a debug build, run (also using [your **api_id** and **api_hash**]( -D TDESKTOP_API_ID=YOUR_API_ID \ -D TDESKTOP_API_HASH=YOUR_API_HASH -If you need a backward compatible binary (running on older OS like the official one), you should build the binary with LTO. -To do this, add `-D CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON` option. - The built files will be in the `out` directory. [api_credentials]: api_credentials.md diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b309c3d11..95e95f375 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -99,6 +99,7 @@ parts: - libboost-regex1.74-dev - libfmt-dev - libgirepository1.0-dev + - libglib2.0-dev - libheif-dev - libopus-dev - libpulse-dev @@ -113,6 +114,7 @@ parts: - libasound2 - libavif13 - libboost-regex1.74.0 + - libglib2.0-0 - libheif1 - libopus0 - libpulse0 @@ -156,7 +158,6 @@ parts: after: - ffmpeg - libjxl - - glibmm - mozjpeg - openal - protobuf @@ -176,18 +177,6 @@ parts: rm -rf patches stage: [-./*] - meson-deps: - source: https://github.com/mesonbuild/meson.git - source-depth: 1 - source-branch: 1.2.3 - plugin: nil - build-packages: - - python3-pip - - ninja-build - override-build: | - pip install . - stage: [-./*] - desktop-qt: source: https://github.com/desktop-app/snapcraft-desktop-helpers.git source-depth: 1 @@ -254,53 +243,6 @@ parts: after: - mozjpeg - glib: - source: https://github.com/GNOME/glib.git - source-depth: 1 - source-tag: 2.78.1 - plugin: meson - build-packages: - - mm-common - - ninja-build - meson-parameters: - - --buildtype=release - - --prefix=/usr - - --strip - - -Dtests=false - prime: - - -./usr/include - - -./usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/pkgconfig - - -./usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/*.so - after: - - meson-deps - - glibmm: - source: https://github.com/GNOME/glibmm.git - source-depth: 1 - source-tag: 2.78.0 - plugin: meson - build-packages: - - mm-common - - ninja-build - - libxml-parser-perl - meson-parameters: - - --buildtype=release - - --prefix=/usr - - --strip - - -Dbuild-documentation=false - - -Dbuild-examples=false - - -Dsigc++-3.0:build-documentation=false - - -Dsigc++-3.0:build-examples=false - - -Dsigc++-3.0:build-tests=false - - -Dsigc++-3.0:validation=false - prime: - - -./usr/include - - -./usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/pkgconfig - - -./usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/*.so - after: - - meson-deps - - glib - libjxl: source: https://github.com/libjxl/libjxl.git source-depth: 1 @@ -418,6 +360,7 @@ parts: - libegl-dev - libfontconfig1-dev - libfreetype-dev + - libglib2.0-dev - libglx-dev - libgtk-3-dev - libharfbuzz-dev @@ -456,6 +399,7 @@ parts: - libegl1 - libfontconfig1 - libfreetype6 + - libglib2.0-0 - libglx0 - libgtk-3-0 - libharfbuzz0b @@ -535,7 +479,6 @@ parts: - -./usr/mkspecs - -./usr/modules after: - - glib - mozjpeg - patches @@ -579,6 +522,7 @@ parts: - libegl-dev - libgbm-dev - libgl-dev + - libglib2.0-dev - libopenh264-dev - libopus-dev - libpipewire-0.3-dev @@ -597,6 +541,7 @@ parts: - libegl1 - libgbm1 - libgl1 + - libglib2.0-0 - libopenh264-6 - libopus0 - libssl3 @@ -620,5 +565,4 @@ parts: - -./usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/*.a after: - ffmpeg - - glib - mozjpeg From a117c1ebf2d15c4ca15905d4d965091e26dccb4c Mon Sep 17 00:00:00 2001 From: Dubzer Date: Thu, 14 Mar 2024 22:48:40 +0300 Subject: [PATCH 031/596] Add new Segoe UI to IV css --- Telegram/Resources/iv_html/page.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/Resources/iv_html/page.css b/Telegram/Resources/iv_html/page.css index ae3d64655..6df26aaf9 100644 --- a/Telegram/Resources/iv_html/page.css +++ b/Telegram/Resources/iv_html/page.css @@ -1,5 +1,5 @@ :root { - --font-sans: -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, tahoma, arial, sans-serif; + --font-sans: -apple-system, BlinkMacSystemFont, avenir next, avenir, Segoe UI Variable Text, segoe ui, helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, tahoma, arial, sans-serif; --font-serif: Iowan Old Style, Apple Garamond, Baskerville, Georgia, Times New Roman, Droid Serif, Times, Source Serif Pro, serif, Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol; --font-mono: Menlo, Cascadia Code, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; } From ded7aada5243dd05b535e9f9cc7969c2d1a37654 Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Fri, 15 Mar 2024 02:34:08 +0300 Subject: [PATCH 032/596] Slightly improved filter for choosing users requested by bot. --- Telegram/SourceFiles/api/api_bot.cpp | 8 ++++---- Telegram/SourceFiles/boxes/peers/choose_peer_box.cpp | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Telegram/SourceFiles/api/api_bot.cpp b/Telegram/SourceFiles/api/api_bot.cpp index d49f1bbf7..706b7a7cb 100644 --- a/Telegram/SourceFiles/api/api_bot.cpp +++ b/Telegram/SourceFiles/api/api_bot.cpp @@ -421,10 +421,10 @@ void ActivateBotCommand(ClickHandlerContext context, int row, int column) { MTP_int(itemId), MTP_int(id), MTP_vector_from_range( - result - | ranges::views::transform([]( - not_null peer) { - return MTPInputPeer(peer->input); })) + result | ranges::views::transform([]( + not_null peer) { + return MTPInputPeer(peer->input); + })) )).done([=](const MTPUpdates &result) { peer->session().api().applyUpdates(result); }).send(); diff --git a/Telegram/SourceFiles/boxes/peers/choose_peer_box.cpp b/Telegram/SourceFiles/boxes/peers/choose_peer_box.cpp index 7d4d74453..33ce09ac4 100644 --- a/Telegram/SourceFiles/boxes/peers/choose_peer_box.cpp +++ b/Telegram/SourceFiles/boxes/peers/choose_peer_box.cpp @@ -49,7 +49,7 @@ public: void submit(); QString savedMessagesChatStatus() const override { - return tr::lng_saved_forward_here(tr::now); + return {}; } private: @@ -295,6 +295,8 @@ object_ptr CreatePeerByQueryBox( case Type::User: { const auto user = peer->asUser(); return user + && !user->isInaccessible() + && !user->isNotificationsUser() && checkRestriction(query.userIsBot, user->isBot()) && checkRestriction(query.userIsPremium, user->isPremium()); } From 9e18236b55fadf9d00635b50f495733a2a303c15 Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Fri, 15 Mar 2024 02:34:49 +0300 Subject: [PATCH 033/596] Added ability to open account in new window with middle click. --- Telegram/SourceFiles/settings/settings_information.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Telegram/SourceFiles/settings/settings_information.cpp b/Telegram/SourceFiles/settings/settings_information.cpp index 6a46992cc..9e4a66529 100644 --- a/Telegram/SourceFiles/settings/settings_information.cpp +++ b/Telegram/SourceFiles/settings/settings_information.cpp @@ -650,6 +650,9 @@ void SetupAccountsWrap( if (which == Qt::LeftButton) { callback(raw->clickModifiers()); return; + } else if (which == Qt::MiddleButton) { + callback(Qt::ControlModifier); + return; } else if (which != Qt::RightButton) { return; } From c3ca8c6258cb26ae99c594d79224ad39b41c2843 Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Fri, 15 Mar 2024 03:06:05 +0300 Subject: [PATCH 034/596] Fixed emoji drawing in statuses of group call participants. --- .../calls/group/calls_group_members_row.cpp | 55 ++++++++++--------- .../calls/group/calls_group_members_row.h | 2 +- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/Telegram/SourceFiles/calls/group/calls_group_members_row.cpp b/Telegram/SourceFiles/calls/group/calls_group_members_row.cpp index 96a31fcb9..78484dadc 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_members_row.cpp +++ b/Telegram/SourceFiles/calls/group/calls_group_members_row.cpp @@ -129,7 +129,7 @@ MembersRow::MembersRow( : PeerListRow(participantPeer) , _delegate(delegate) { refreshStatus(); - _aboutText = participantPeer->about(); + _about.setText(st::defaultTextStyle, participantPeer->about()); } MembersRow::~MembersRow() = default; @@ -562,10 +562,10 @@ void MembersRow::paintStatusIcon( } void MembersRow::setAbout(const QString &about) { - if (_aboutText == about) { + if (_about.toString() == about) { return; } - _aboutText = about; + _about.setText(st::defaultTextStyle, about); _delegate->rowUpdateRow(this); } @@ -610,13 +610,11 @@ void MembersRow::paintComplexStatusText( x += skip; availableWidth -= skip; const auto &font = st::normalFont; - const auto about = (style == MembersRowStyle::Video) - ? QString() + const auto useAbout = (style == MembersRowStyle::Video) + ? false : ((_state == State::RaisedHand && !_raisedHandStatus) - || (_state != State::RaisedHand && !_speaking)) - ? _aboutText - : QString(); - if (about.isEmpty() + || (_state != State::RaisedHand && !_speaking)); + if (!useAbout && _state != State::Invited && !_mutedByMe) { paintStatusIcon(p, x, y, st, font, selected, narrowMode); @@ -641,25 +639,30 @@ void MembersRow::paintComplexStatusText( selected); return; } - p.setFont(font); - if (style == MembersRowStyle::Video) { - p.setPen(st::groupCallVideoSubTextFg); - } else if (_mutedByMe) { - p.setPen(st::groupCallMemberMutedIcon); + p.setPen((style == MembersRowStyle::Video) + ? st::groupCallVideoSubTextFg + : _mutedByMe + ? st::groupCallMemberMutedIcon + : st::groupCallMemberNotJoinedStatus); + if (!_mutedByMe && useAbout) { + return _about.draw(p, { + .position = QPoint(x, y), + .outerWidth = outerWidth, + .availableWidth = availableWidth, + .elisionLines = 1, + }); } else { - p.setPen(st::groupCallMemberNotJoinedStatus); + p.setFont(font); + p.drawTextLeft( + x, + y, + outerWidth, + (_mutedByMe + ? tr::lng_group_call_muted_by_me_status(tr::now) + : _delegate->rowIsMe(peer()) + ? tr::lng_status_connecting(tr::now) + : tr::lng_group_call_invited_status(tr::now))); } - p.drawTextLeft( - x, - y, - outerWidth, - (_mutedByMe - ? tr::lng_group_call_muted_by_me_status(tr::now) - : !about.isEmpty() - ? font->elided(about, availableWidth) - : _delegate->rowIsMe(peer()) - ? tr::lng_status_connecting(tr::now) - : tr::lng_group_call_invited_status(tr::now))); } QSize MembersRow::rightActionSize() const { diff --git a/Telegram/SourceFiles/calls/group/calls_group_members_row.h b/Telegram/SourceFiles/calls/group/calls_group_members_row.h index 8dd02f1de..5ba910c8e 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_members_row.h +++ b/Telegram/SourceFiles/calls/group/calls_group_members_row.h @@ -208,7 +208,7 @@ private: Ui::Animations::Simple _speakingAnimation; // For gray-red/green icon. Ui::Animations::Simple _mutedAnimation; // For gray/red icon. Ui::Animations::Simple _activeAnimation; // For icon cross animation. - QString _aboutText; + Ui::Text::String _about; crl::time _speakingLastTime = 0; uint64 _raisedHandRating = 0; int _volume = Group::kDefaultVolume; From 77dc234b2d28e2a9aa3bf6e1b4584fbb6afad16d Mon Sep 17 00:00:00 2001 From: John Preston Date: Fri, 15 Mar 2024 14:42:31 +0400 Subject: [PATCH 035/596] Support AirPods Mute/Unmute toggle. --- Telegram/SourceFiles/calls/calls_call.cpp | 25 ++++++++++++++++ Telegram/SourceFiles/calls/calls_call.h | 8 ++++- .../calls/group/calls_group_call.cpp | 29 +++++++++++++++++++ .../calls/group/calls_group_call.h | 8 ++++- Telegram/lib_webrtc | 2 +- 5 files changed, 69 insertions(+), 3 deletions(-) diff --git a/Telegram/SourceFiles/calls/calls_call.cpp b/Telegram/SourceFiles/calls/calls_call.cpp index ba59556d9..486d5af2a 100644 --- a/Telegram/SourceFiles/calls/calls_call.cpp +++ b/Telegram/SourceFiles/calls/calls_call.cpp @@ -420,6 +420,14 @@ void Call::actuallyAnswer() { }).send(); } +void Call::captureMuteChanged(bool mute) { + setMuted(mute); +} + +rpl::producer Call::captureMuteDeviceId() { + return _captureDeviceId.value(); +} + void Call::setMuted(bool mute) { _muted = mute; if (_instance) { @@ -1033,6 +1041,20 @@ void Call::createAndStartController(const MTPDphoneCall &call) { raw->setIncomingVideoOutput(_videoIncoming->sink()); raw->setAudioOutputDuckingEnabled(settings.callAudioDuckingEnabled()); + + _state.value() | rpl::start_with_next([=](State state) { + const auto track = (state != State::FailedHangingUp) + && (state != State::Failed) + && (state != State::HangingUp) + && (state != State::Ended) + && (state != State::EndedByOtherDevice) + && (state != State::Busy); + Core::App().mediaDevices().setCaptureMuteTracker(this, track); + }, _instanceLifetime); + + _muted.value() | rpl::start_with_next([=](bool muted) { + Core::App().mediaDevices().setCaptureMuted(muted); + }, _instanceLifetime); } void Call::handleControllerStateChange(tgcalls::State state) { @@ -1375,6 +1397,9 @@ void Call::handleControllerError(const QString &error) { } void Call::destroyController() { + _instanceLifetime.destroy(); + Core::App().mediaDevices().setCaptureMuteTracker(this, false); + if (_instance) { _instance->stop([](tgcalls::FinalState) { }); diff --git a/Telegram/SourceFiles/calls/calls_call.h b/Telegram/SourceFiles/calls/calls_call.h index 4be689831..69c2d94da 100644 --- a/Telegram/SourceFiles/calls/calls_call.h +++ b/Telegram/SourceFiles/calls/calls_call.h @@ -59,7 +59,9 @@ enum class CallType { Outgoing, }; -class Call : public base::has_weak_ptr { +class Call final + : public base::has_weak_ptr + , private Webrtc::CaptureMuteTracker { public: class Delegate { public: @@ -249,6 +251,9 @@ private: void setSignalBarCount(int count); void destroyController(); + void captureMuteChanged(bool mute) override; + rpl::producer captureMuteDeviceId() override; + void setupMediaDevices(); void setupOutgoingVideo(); void updateRemoteMediaState( @@ -298,6 +303,7 @@ private: std::unique_ptr _waitingTrack; + rpl::lifetime _instanceLifetime; rpl::lifetime _lifetime; }; diff --git a/Telegram/SourceFiles/calls/group/calls_group_call.cpp b/Telegram/SourceFiles/calls/group/calls_group_call.cpp index 8617aef85..5f7346913 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_call.cpp +++ b/Telegram/SourceFiles/calls/group/calls_group_call.cpp @@ -30,6 +30,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "base/random.h" #include "webrtc/webrtc_video_track.h" #include "webrtc/webrtc_create_adm.h" +#include "webrtc/webrtc_environment.h" #include #include @@ -667,6 +668,8 @@ GroupCall::GroupCall( GroupCall::~GroupCall() { destroyScreencast(); destroyController(); + + Core::App().mediaDevices().setCaptureMuteTracker(this, false); } bool GroupCall::isSharingScreen() const { @@ -2087,6 +2090,32 @@ void GroupCall::setupMediaDevices() { }) | rpl::start_with_next([=](const Webrtc::DeviceResolvedId &deviceId) { _cameraCapture->switchToDevice(deviceId.value.toStdString(), false); }, _lifetime); + + _muted.value() | rpl::start_with_next([=](MuteState state) { + const auto devices = &Core::App().mediaDevices(); + const auto muted = (state != MuteState::Active) + && (state != MuteState::PushToTalk); + const auto track = !muted || (state == MuteState::Muted); + devices->setCaptureMuteTracker(this, track); + devices->setCaptureMuted(muted); + }, _lifetime); +} + +void GroupCall::captureMuteChanged(bool mute) { + const auto oldState = muted(); + if (mute + && (oldState == MuteState::ForceMuted + || oldState == MuteState::RaisedHand + || oldState == MuteState::Muted)) { + return; + } else if (!mute && oldState != MuteState::Muted) { + return; + } + setMutedAndUpdate(mute ? MuteState::Muted : MuteState::Active); +} + +rpl::producer GroupCall::captureMuteDeviceId() { + return _captureDeviceId.value(); } int GroupCall::activeVideoSendersCount() const { diff --git a/Telegram/SourceFiles/calls/group/calls_group_call.h b/Telegram/SourceFiles/calls/group/calls_group_call.h index fd2cd88a0..8f93793bd 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_call.h +++ b/Telegram/SourceFiles/calls/group/calls_group_call.h @@ -12,6 +12,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "base/bytes.h" #include "mtproto/sender.h" #include "mtproto/mtproto_auth_key.h" +#include "webrtc/webrtc_device_common.h" #include "webrtc/webrtc_device_resolver.h" class History; @@ -175,7 +176,9 @@ struct ParticipantVideoParams; [[nodiscard]] uint32 GetAdditionalAudioSsrc( const std::shared_ptr ¶ms); -class GroupCall final : public base::has_weak_ptr { +class GroupCall final + : public base::has_weak_ptr + , private Webrtc::CaptureMuteTracker { public: class Delegate { public: @@ -550,6 +553,9 @@ private: void applySelfUpdate(const MTPDgroupCallParticipant &data); void applyOtherParticipantUpdate(const MTPDgroupCallParticipant &data); + void captureMuteChanged(bool mute) override; + rpl::producer captureMuteDeviceId() override; + void setupMediaDevices(); void setupOutgoingVideo(); void setScreenEndpoint(std::string endpoint); diff --git a/Telegram/lib_webrtc b/Telegram/lib_webrtc index 5493af61d..1cbf5fa7d 160000 --- a/Telegram/lib_webrtc +++ b/Telegram/lib_webrtc @@ -1 +1 @@ -Subproject commit 5493af61df5cb90a30b686296521961763a009e0 +Subproject commit 1cbf5fa7d875074c40e76216a3047bd7c59996d7 From a1a415de82b9ab18589e8d3fe0b43caadaaf566b Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Fri, 15 Mar 2024 17:08:17 +0300 Subject: [PATCH 036/596] Fixed crash in non-stack bar chart view when hide selected point. --- Telegram/SourceFiles/statistics/view/bar_chart_view.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/SourceFiles/statistics/view/bar_chart_view.cpp b/Telegram/SourceFiles/statistics/view/bar_chart_view.cpp index e4ce96160..8beeaa509 100644 --- a/Telegram/SourceFiles/statistics/view/bar_chart_view.cpp +++ b/Telegram/SourceFiles/statistics/view/bar_chart_view.cpp @@ -149,7 +149,7 @@ void BarChartView::paintSelectedXIndex( if (_isStack) { BarChartView::paintChartAndSelected(p, c); - } else { + } else if (selectedXIndex >= 0) { const auto linesFilter = linesFilterController(); auto hq = PainterHighQualityEnabler(p); auto o = ScopedPainterOpacity(p, progress); From f1d8135e30181d16c93c6775f47eb2990da6adc1 Mon Sep 17 00:00:00 2001 From: John Preston Date: Fri, 15 Mar 2024 20:23:48 +0400 Subject: [PATCH 037/596] Fix crash in IV parsing. --- Telegram/SourceFiles/iv/iv_prepare.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/SourceFiles/iv/iv_prepare.cpp b/Telegram/SourceFiles/iv/iv_prepare.cpp index 9e276bce7..2d9595667 100644 --- a/Telegram/SourceFiles/iv/iv_prepare.cpp +++ b/Telegram/SourceFiles/iv/iv_prepare.cpp @@ -851,7 +851,7 @@ QByteArray Parser::block(const MTPDpageRelatedArticle &data) { inner += tag( "span", { { "class", "related-link-source" } }, - EscapeText(utf(*author) + EscapeText((author ? utf(*author) : QByteArray()) + separator + langDateTimeFull(parsed).toUtf8())); } From 24e9a3ea595cd05391839ba15bf7bf69435df092 Mon Sep 17 00:00:00 2001 From: John Preston Date: Fri, 15 Mar 2024 20:27:54 +0400 Subject: [PATCH 038/596] Bot close confirm closes only by Close button. --- Telegram/SourceFiles/ui/chat/attach/attach_bot_webview.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/SourceFiles/ui/chat/attach/attach_bot_webview.cpp b/Telegram/SourceFiles/ui/chat/attach/attach_bot_webview.cpp index ed57fdf2b..a84890032 100644 --- a/Telegram/SourceFiles/ui/chat/attach/attach_bot_webview.cpp +++ b/Telegram/SourceFiles/ui/chat/attach/attach_bot_webview.cpp @@ -1073,7 +1073,7 @@ void Panel::closeWithConfirmation() { }); if (!weak) { return; - } else if (result.id != "cancel") { + } else if (result.id == "close") { _delegate->botClose(); } else { _closeWithConfirmationScheduled = false; From ad101dc8a06ab50384b386186383542b78a20bd0 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Fri, 15 Mar 2024 23:08:20 +0400 Subject: [PATCH 039/596] A way to initialize GLib::VariantIter without gi::wrap is found --- Telegram/SourceFiles/platform/linux/integration_linux.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Telegram/SourceFiles/platform/linux/integration_linux.cpp b/Telegram/SourceFiles/platform/linux/integration_linux.cpp index 38e5cc7f2..47b6e63ef 100644 --- a/Telegram/SourceFiles/platform/linux/integration_linux.cpp +++ b/Telegram/SourceFiles/platform/linux/integration_linux.cpp @@ -30,9 +30,9 @@ using namespace gi::repository; std::vector AnyVectorFromVariant(GLib::Variant value) { std::vector result; - auto iter = gi::wrap( - g_variant_iter_new(value.gobj_()), - gi::transfer_full); + GLib::VariantIter iter; + iter.allocate_(); + iter.init(value); const auto uint64Type = GLib::VariantType::new_("t"); const auto int64Type = GLib::VariantType::new_("x"); From f4ecfeaddd96963f5c05a15f1a31269c1de750ca Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Fri, 15 Mar 2024 19:07:22 +0300 Subject: [PATCH 040/596] Added ability to perform bulk selection from menu in HistoryWidget. --- Telegram/Resources/langs/lang.strings | 1 + .../history/history_inner_widget.cpp | 81 +++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/Telegram/Resources/langs/lang.strings b/Telegram/Resources/langs/lang.strings index 39907f573..190958942 100644 --- a/Telegram/Resources/langs/lang.strings +++ b/Telegram/Resources/langs/lang.strings @@ -3013,6 +3013,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL "lng_context_delete_msg" = "Delete Message"; "lng_context_auto_delete_in" = "auto-delete in {duration}"; "lng_context_select_msg" = "Select Message"; +"lng_context_select_msg_bulk" = "Select up to this message"; "lng_context_report_msg" = "Report Message"; "lng_context_pin_msg" = "Pin Message"; "lng_context_unpin_msg" = "Unpin Message"; diff --git a/Telegram/SourceFiles/history/history_inner_widget.cpp b/Telegram/SourceFiles/history/history_inner_widget.cpp index a2188d7d5..79c25f691 100644 --- a/Telegram/SourceFiles/history/history_inner_widget.cpp +++ b/Telegram/SourceFiles/history/history_inner_widget.cpp @@ -2277,6 +2277,87 @@ void HistoryInner::showContextMenu(QContextMenuEvent *e, bool showFromTouch) { } } }, &st::menuIconSelect); + const auto collectBetween = [=]( + not_null from, + not_null to, + int max) -> HistoryItemsList { + auto current = from; + auto collected = HistoryItemsList(); + collected.reserve(max); + collected.push_back(from); + collected.push_back(to); + const auto toId = to->fullId(); + while (true) { + if (collected.size() > max) { + return {}; + } + const auto view = viewByItem(current); + const auto nextView = nextItem(view); + if (!nextView) { + return {}; + } + const auto nextItem = nextView->data(); + if (nextItem->fullId() == toId) { + return collected; + } + if (nextItem->isRegular() && !nextItem->isService()) { + collected.push_back(nextItem); + } + current = nextItem; + } + }; + + [&] { // Select up to this message. + if (selectedState.count <= 0) { + return; + } + const auto toItem = groupLeaderOrSelf(item); + auto topToBottom = false; + auto nearestItem = (HistoryItem*)(nullptr); + { + auto minDiff = std::numeric_limits::max(); + for (const auto &[item, _] : _selected) { + const auto diff = item->fullId().msg.bare + - toItem->fullId().msg.bare; + if (std::abs(diff) < minDiff) { + nearestItem = item; + minDiff = std::abs(diff); + topToBottom = (diff < 0); + } + } + } + if (!nearestItem) { + return; + } + const auto start = (topToBottom ? nearestItem : toItem); + const auto end = (topToBottom ? toItem : nearestItem); + const auto left = MaxSelectedItems + - selectedState.count + + (topToBottom ? 0 : 1); + if (collectBetween(start, end, left).empty()) { + return; + } + const auto startId = start->fullId(); + const auto endId = end->fullId(); + const auto callback = [=] { + const auto from = session->data().message(startId); + const auto to = session->data().message(endId); + if (from && to) { + for (const auto &i : collectBetween(from, to, left)) { + changeSelectionAsGroup( + &_selected, + i, + SelectAction::Select); + } + update(); + _widget->updateTopBarSelection(); + } + }; + _menu->addAction( + tr::lng_context_select_msg_bulk(tr::now), + callback, + &st::menuIconSelect); + }(); } }; From 393f330d15c6c0412ed85d465b6269971b277b7d Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Sat, 16 Mar 2024 04:38:43 +0300 Subject: [PATCH 041/596] Added ability to go to message from Calls box. --- Telegram/SourceFiles/calls/calls_box_controller.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Telegram/SourceFiles/calls/calls_box_controller.cpp b/Telegram/SourceFiles/calls/calls_box_controller.cpp index 11c6347d5..a3d0d5369 100644 --- a/Telegram/SourceFiles/calls/calls_box_controller.cpp +++ b/Telegram/SourceFiles/calls/calls_box_controller.cpp @@ -572,6 +572,11 @@ base::unique_qptr BoxController::rowContextMenu( _window->show( Box(session, base::duplicate(ids))); }, &st::menuIconDelete); + result->addAction(tr::lng_context_to_msg(tr::now), [=, window = _window] { + if (const auto item = session->data().message(ids.front())) { + window->showMessage(item); + } + }, &st::menuIconShowInChat); return result; } From 55b2b04e2ea67078731e932c045622b465e4854a Mon Sep 17 00:00:00 2001 From: John Preston Date: Sat, 16 Mar 2024 09:54:07 +0400 Subject: [PATCH 042/596] Reapply "Disable system proxy support on Linux" This reverts commit 202a4064a43578980a21d57288569210a506cac5. --- Telegram/SourceFiles/core/sandbox.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Telegram/SourceFiles/core/sandbox.cpp b/Telegram/SourceFiles/core/sandbox.cpp index 6feacdb5b..fb145f7f1 100644 --- a/Telegram/SourceFiles/core/sandbox.cpp +++ b/Telegram/SourceFiles/core/sandbox.cpp @@ -33,6 +33,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include #include #include +#include namespace Core { namespace { @@ -517,8 +518,10 @@ void Sandbox::refreshGlobalProxy() { || proxy.type == MTP::ProxyData::Type::Http) { QNetworkProxy::setApplicationProxy( MTP::ToNetworkProxy(MTP::ToDirectIpProxy(proxy))); - } else if (!Core::IsAppLaunched() - || Core::App().settings().proxy().isSystem()) { + } else if ((!Core::IsAppLaunched() + || Core::App().settings().proxy().isSystem()) + // this works stable only in sandboxed environment where it works through portal + && (!Platform::IsLinux() || KSandbox::isInside() || cDebugMode())) { QNetworkProxyFactory::setUseSystemConfiguration(true); } else { QNetworkProxy::setApplicationProxy(QNetworkProxy::NoProxy); From 82fda96ed88065a761e27cb80173250c81aba8a9 Mon Sep 17 00:00:00 2001 From: John Preston Date: Sat, 16 Mar 2024 09:59:23 +0400 Subject: [PATCH 043/596] Beta version 4.15.5. - Fix a crash in Instant View article parsing. - Support AirPods Mute/Unmute toggle in calls on macOS. - Disable system proxy on Linux to check if it fixes crashes. --- Telegram/Resources/uwp/AppX/AppxManifest.xml | 2 +- Telegram/Resources/winrc/Telegram.rc | 8 ++++---- Telegram/Resources/winrc/Updater.rc | 8 ++++---- Telegram/SourceFiles/core/version.h | 4 ++-- Telegram/build/version | 8 ++++---- changelog.txt | 6 ++++++ 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Telegram/Resources/uwp/AppX/AppxManifest.xml b/Telegram/Resources/uwp/AppX/AppxManifest.xml index e6aa01dc5..1149de562 100644 --- a/Telegram/Resources/uwp/AppX/AppxManifest.xml +++ b/Telegram/Resources/uwp/AppX/AppxManifest.xml @@ -10,7 +10,7 @@ + Version="4.15.5.0" /> Telegram Desktop Telegram Messenger LLP diff --git a/Telegram/Resources/winrc/Telegram.rc b/Telegram/Resources/winrc/Telegram.rc index fe9addb7b..fe116dd6d 100644 --- a/Telegram/Resources/winrc/Telegram.rc +++ b/Telegram/Resources/winrc/Telegram.rc @@ -44,8 +44,8 @@ IDI_ICON1 ICON "..\\art\\icon256.ico" // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,15,4,0 - PRODUCTVERSION 4,15,4,0 + FILEVERSION 4,15,5,0 + PRODUCTVERSION 4,15,5,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -62,10 +62,10 @@ BEGIN BEGIN VALUE "CompanyName", "Telegram FZ-LLC" VALUE "FileDescription", "Telegram Desktop" - VALUE "FileVersion", "4.15.4.0" + VALUE "FileVersion", "4.15.5.0" VALUE "LegalCopyright", "Copyright (C) 2014-2024" VALUE "ProductName", "Telegram Desktop" - VALUE "ProductVersion", "4.15.4.0" + VALUE "ProductVersion", "4.15.5.0" END END BLOCK "VarFileInfo" diff --git a/Telegram/Resources/winrc/Updater.rc b/Telegram/Resources/winrc/Updater.rc index 1289e471e..93b6c21de 100644 --- a/Telegram/Resources/winrc/Updater.rc +++ b/Telegram/Resources/winrc/Updater.rc @@ -35,8 +35,8 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,15,4,0 - PRODUCTVERSION 4,15,4,0 + FILEVERSION 4,15,5,0 + PRODUCTVERSION 4,15,5,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -53,10 +53,10 @@ BEGIN BEGIN VALUE "CompanyName", "Telegram FZ-LLC" VALUE "FileDescription", "Telegram Desktop Updater" - VALUE "FileVersion", "4.15.4.0" + VALUE "FileVersion", "4.15.5.0" VALUE "LegalCopyright", "Copyright (C) 2014-2024" VALUE "ProductName", "Telegram Desktop" - VALUE "ProductVersion", "4.15.4.0" + VALUE "ProductVersion", "4.15.5.0" END END BLOCK "VarFileInfo" diff --git a/Telegram/SourceFiles/core/version.h b/Telegram/SourceFiles/core/version.h index cc1bff4ac..fbb996b7c 100644 --- a/Telegram/SourceFiles/core/version.h +++ b/Telegram/SourceFiles/core/version.h @@ -22,7 +22,7 @@ constexpr auto AppId = "{53F49750-6209-4FBF-9CA8-7A333C87D1ED}"_cs; constexpr auto AppNameOld = "Telegram Win (Unofficial)"_cs; constexpr auto AppName = "Telegram Desktop"_cs; constexpr auto AppFile = "Telegram"_cs; -constexpr auto AppVersion = 4015004; -constexpr auto AppVersionStr = "4.15.4"; +constexpr auto AppVersion = 4015005; +constexpr auto AppVersionStr = "4.15.5"; constexpr auto AppBetaVersion = true; constexpr auto AppAlphaVersion = TDESKTOP_ALPHA_VERSION; diff --git a/Telegram/build/version b/Telegram/build/version index d1f6650e1..24dca8f50 100644 --- a/Telegram/build/version +++ b/Telegram/build/version @@ -1,7 +1,7 @@ -AppVersion 4015004 +AppVersion 4015005 AppVersionStrMajor 4.15 -AppVersionStrSmall 4.15.4 -AppVersionStr 4.15.4 +AppVersionStrSmall 4.15.5 +AppVersionStr 4.15.5 BetaChannel 1 AlphaVersion 0 -AppVersionOriginal 4.15.4.beta +AppVersionOriginal 4.15.5.beta diff --git a/changelog.txt b/changelog.txt index 62f49bf5e..191239576 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,9 @@ +4.15.5 beta (16.03.24) + +- Fix a crash in Instant View article parsing. +- Support AirPods Mute/Unmute toggle in calls on macOS. +- Disable system proxy on Linux to check if it fixes crashes. + 4.15.4 beta (14.03.24) - Fix initial text color in dark mode Instant View on macOS. From 6d5347472d422a5b5ed481153c13b9dd91b8ed60 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Sat, 16 Mar 2024 02:25:32 +0400 Subject: [PATCH 044/596] Fix media viewer geometry logging It has regressed after c64e953174b9e40dc27754dbdcd4a9b9d1bb729b --- .../media/view/media_view_overlay_widget.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp b/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp index 4c8a0cf70..bc1b00968 100644 --- a/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp +++ b/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp @@ -481,6 +481,10 @@ OverlayWidget::OverlayWidget() moveToScreen(true); } } else if (type == QEvent::Resize) { + const auto size = static_cast(e.get())->size(); + DEBUG_LOG(("Viewer Pos: Resized to %1, %2") + .arg(size.width()) + .arg(size.height())); if (_windowed) { savePosition(); } @@ -508,9 +512,6 @@ OverlayWidget::OverlayWidget() const auto type = e->type(); if (type == QEvent::Resize) { const auto size = static_cast(e.get())->size(); - DEBUG_LOG(("Viewer Pos: Resized to %1, %2") - .arg(size.width()) - .arg(size.height())); // Somehow Windows 11 knows the geometry of first widget below // the semi-native title control widgets and it uses @@ -909,6 +910,11 @@ void OverlayWidget::updateGeometry(bool inMove) { if (_fullscreen && (!Platform::IsWindows11OrGreater() || !isHidden())) { updateGeometryToScreen(inMove); } else if (_windowed && _normalGeometryInited) { + DEBUG_LOG(("Viewer Pos: Setting %1, %2, %3, %4") + .arg(_normalGeometry.x()) + .arg(_normalGeometry.y()) + .arg(_normalGeometry.width()) + .arg(_normalGeometry.height())); _window->setGeometry(_normalGeometry); } if constexpr (!Platform::IsMac()) { From ddaf78828a82bf1db2b19e91ae9e5d73889ba4b5 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Sat, 16 Mar 2024 02:29:12 +0400 Subject: [PATCH 045/596] Fix media viewer geometry restoration --- Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp b/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp index bc1b00968..b4965fe6f 100644 --- a/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp +++ b/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp @@ -915,7 +915,7 @@ void OverlayWidget::updateGeometry(bool inMove) { .arg(_normalGeometry.y()) .arg(_normalGeometry.width()) .arg(_normalGeometry.height())); - _window->setGeometry(_normalGeometry); + _window->RpWidget::setGeometry(_normalGeometry); } if constexpr (!Platform::IsMac()) { if (_fullscreen) { From 2638ee292616f1238d7804f0d537a34cb374b98b Mon Sep 17 00:00:00 2001 From: 23rd <23rd@vivaldi.net> Date: Sun, 17 Mar 2024 03:46:11 +0300 Subject: [PATCH 046/596] Improved message edition with pre-selected text. --- .../chat_helpers/message_field.cpp | 25 +++++++++++++++++++ .../SourceFiles/chat_helpers/message_field.h | 4 +++ .../history/history_inner_widget.cpp | 12 ++++++++- .../SourceFiles/history/history_widget.cpp | 13 ++++------ Telegram/SourceFiles/history/history_widget.h | 5 ++-- .../history_view_compose_controls.cpp | 5 +++- .../controls/history_view_compose_controls.h | 2 +- .../history/view/history_view_list_widget.cpp | 5 ++++ .../history/view/history_view_list_widget.h | 2 ++ .../view/history_view_replies_section.cpp | 4 ++- .../view/history_view_scheduled_section.cpp | 4 ++- .../business/settings_shortcut_messages.cpp | 4 ++- 12 files changed, 69 insertions(+), 16 deletions(-) diff --git a/Telegram/SourceFiles/chat_helpers/message_field.cpp b/Telegram/SourceFiles/chat_helpers/message_field.cpp index 5715eea45..c5c04a43c 100644 --- a/Telegram/SourceFiles/chat_helpers/message_field.cpp +++ b/Telegram/SourceFiles/chat_helpers/message_field.cpp @@ -1037,3 +1037,28 @@ base::unique_qptr PremiumRequiredSendRestriction( }); return result; } + +void SelectTextInFieldWithMargins( + not_null field, + const TextSelection &selection) { + if (selection.empty()) { + return; + } + auto textCursor = field->textCursor(); + // Try to set equal margins for top and bottom sides. + const auto charsCountInLine = field->width() + / field->st().font->width('W'); + const auto linesCount = (field->height() / field->st().font->height); + const auto selectedLines = (selection.to - selection.from) + / charsCountInLine; + constexpr auto kMinDiff = ushort(3); + if ((linesCount - selectedLines) > kMinDiff) { + textCursor.setPosition(selection.from + - charsCountInLine * ((linesCount - 1) / 2)); + field->setTextCursor(textCursor); + } + textCursor.setPosition(selection.from); + field->setTextCursor(textCursor); + textCursor.setPosition(selection.to, QTextCursor::KeepAnchor); + field->setTextCursor(textCursor); +} diff --git a/Telegram/SourceFiles/chat_helpers/message_field.h b/Telegram/SourceFiles/chat_helpers/message_field.h index a8f248e82..0c9d8ff85 100644 --- a/Telegram/SourceFiles/chat_helpers/message_field.h +++ b/Telegram/SourceFiles/chat_helpers/message_field.h @@ -154,3 +154,7 @@ private: QWidget *parent, not_null user, not_null controller); + +void SelectTextInFieldWithMargins( + not_null field, + const TextSelection &selection); diff --git a/Telegram/SourceFiles/history/history_inner_widget.cpp b/Telegram/SourceFiles/history/history_inner_widget.cpp index 79c25f691..c5f201f51 100644 --- a/Telegram/SourceFiles/history/history_inner_widget.cpp +++ b/Telegram/SourceFiles/history/history_inner_widget.cpp @@ -2131,7 +2131,17 @@ void HistoryInner::showContextMenu(QContextMenuEvent *e, bool showFromTouch) { if (editItem) { const auto editItemId = editItem->fullId(); _menu->addAction(tr::lng_context_edit_msg(tr::now), [=] { - _widget->editMessage(editItemId); + if (const auto item = session->data().message(editItemId)) { + auto it = _selected.find(item); + const auto selection = ((it != _selected.end()) + && (it->second != FullSelection)) + ? it->second + : TextSelection(); + if (!selection.empty()) { + clearSelected(true); + } + _widget->editMessage(item, selection); + } }, &st::menuIconEdit); } const auto pinItem = (item->canPin() && item->isPinned()) diff --git a/Telegram/SourceFiles/history/history_widget.cpp b/Telegram/SourceFiles/history/history_widget.cpp index 8aa8f7332..1bc4ac21f 100644 --- a/Telegram/SourceFiles/history/history_widget.cpp +++ b/Telegram/SourceFiles/history/history_widget.cpp @@ -6606,7 +6606,7 @@ void HistoryWidget::keyPressEvent(QKeyEvent *e) { && _field->empty() && !_editMsgId && !_replyTo) { - editMessage(item); + editMessage(item, {}); return; } _scroll->keyPressEvent(e); @@ -7570,13 +7570,9 @@ void HistoryWidget::setReplyFieldsFromProcessing() { setInnerFocus(); } -void HistoryWidget::editMessage(FullMsgId itemId) { - if (const auto item = session().data().message(itemId)) { - editMessage(item); - } -} - -void HistoryWidget::editMessage(not_null item) { +void HistoryWidget::editMessage( + not_null item, + const TextSelection &selection) { if (_chooseTheme) { toggleChooseChatTheme(_peer); } else if (_voiceRecordBar->isActive()) { @@ -7625,6 +7621,7 @@ void HistoryWidget::editMessage(not_null item) { updateReplyToName(); updateControlsGeometry(); updateField(); + SelectTextInFieldWithMargins(_field, selection); _saveDraftText = true; _saveDraftStart = crl::now(); diff --git a/Telegram/SourceFiles/history/history_widget.h b/Telegram/SourceFiles/history/history_widget.h index 4cbfd46af..eafa19ac9 100644 --- a/Telegram/SourceFiles/history/history_widget.h +++ b/Telegram/SourceFiles/history/history_widget.h @@ -195,8 +195,9 @@ public: not_null item, TextWithEntities quote = {}, int quoteOffset = 0); - void editMessage(FullMsgId itemId); - void editMessage(not_null item); + void editMessage( + not_null item, + const TextSelection &selection); [[nodiscard]] FullReplyTo replyTo() const; bool lastForceReplyReplied(const FullMsgId &replyTo) const; diff --git a/Telegram/SourceFiles/history/view/controls/history_view_compose_controls.cpp b/Telegram/SourceFiles/history/view/controls/history_view_compose_controls.cpp index 8fe8752c5..185073db1 100644 --- a/Telegram/SourceFiles/history/view/controls/history_view_compose_controls.cpp +++ b/Telegram/SourceFiles/history/view/controls/history_view_compose_controls.cpp @@ -2857,9 +2857,12 @@ void ComposeControls::updateHeight() { } } -void ComposeControls::editMessage(FullMsgId id) { +void ComposeControls::editMessage( + FullMsgId id, + const TextSelection &selection) { if (const auto item = session().data().message(id)) { editMessage(item); + SelectTextInFieldWithMargins(_field, selection); } } diff --git a/Telegram/SourceFiles/history/view/controls/history_view_compose_controls.h b/Telegram/SourceFiles/history/view/controls/history_view_compose_controls.h index 779c8c2d8..b24e0213a 100644 --- a/Telegram/SourceFiles/history/view/controls/history_view_compose_controls.h +++ b/Telegram/SourceFiles/history/view/controls/history_view_compose_controls.h @@ -198,7 +198,7 @@ public: void showFinished(); void raisePanels(); - void editMessage(FullMsgId id); + void editMessage(FullMsgId id, const TextSelection &selection); void cancelEditMessage(); void maybeCancelEditMessage(); // Confirm if changed and cancel. diff --git a/Telegram/SourceFiles/history/view/history_view_list_widget.cpp b/Telegram/SourceFiles/history/view/history_view_list_widget.cpp index 9c2f94086..fb8d99130 100644 --- a/Telegram/SourceFiles/history/view/history_view_list_widget.cpp +++ b/Telegram/SourceFiles/history/view/history_view_list_widget.cpp @@ -2425,6 +2425,11 @@ SelectedItems ListWidget::getSelectedItems() const { return collectSelectedItems(); } +const TextSelection &ListWidget::getSelectedTextRange( + not_null item) const { + return _selectedTextRange; +} + int ListWidget::findItemIndexByY(int y) const { Expects(!_items.empty()); diff --git a/Telegram/SourceFiles/history/view/history_view_list_widget.h b/Telegram/SourceFiles/history/view/history_view_list_widget.h index 12e09accf..b68e689e9 100644 --- a/Telegram/SourceFiles/history/view/history_view_list_widget.h +++ b/Telegram/SourceFiles/history/view/history_view_list_widget.h @@ -248,6 +248,8 @@ public: [[nodiscard]] TextForMimeData getSelectedText() const; [[nodiscard]] MessageIdsList getSelectedIds() const; [[nodiscard]] SelectedItems getSelectedItems() const; + [[nodiscard]] const TextSelection &getSelectedTextRange( + not_null item) const; void cancelSelection(); void selectItem(not_null item); void selectItemAsGroup(not_null item); diff --git a/Telegram/SourceFiles/history/view/history_view_replies_section.cpp b/Telegram/SourceFiles/history/view/history_view_replies_section.cpp index 7b8eb8765..9451bd556 100644 --- a/Telegram/SourceFiles/history/view/history_view_replies_section.cpp +++ b/Telegram/SourceFiles/history/view/history_view_replies_section.cpp @@ -316,7 +316,9 @@ RepliesWidget::RepliesWidget( if (const auto item = session().data().message(fullId)) { const auto media = item->media(); if (!media || media->webpage() || media->allowsEditCaption()) { - _composeControls->editMessage(fullId); + _composeControls->editMessage( + fullId, + _inner->getSelectedTextRange(item)); } } }, _inner->lifetime()); diff --git a/Telegram/SourceFiles/history/view/history_view_scheduled_section.cpp b/Telegram/SourceFiles/history/view/history_view_scheduled_section.cpp index 2a6328b38..59ee93ae9 100644 --- a/Telegram/SourceFiles/history/view/history_view_scheduled_section.cpp +++ b/Telegram/SourceFiles/history/view/history_view_scheduled_section.cpp @@ -172,7 +172,9 @@ ScheduledWidget::ScheduledWidget( if (const auto item = session().data().message(fullId)) { const auto media = item->media(); if (!media || media->webpage() || media->allowsEditCaption()) { - _composeControls->editMessage(fullId); + _composeControls->editMessage( + fullId, + _inner->getSelectedTextRange(item)); } } }, _inner->lifetime()); diff --git a/Telegram/SourceFiles/settings/business/settings_shortcut_messages.cpp b/Telegram/SourceFiles/settings/business/settings_shortcut_messages.cpp index e1fcec944..9422c892c 100644 --- a/Telegram/SourceFiles/settings/business/settings_shortcut_messages.cpp +++ b/Telegram/SourceFiles/settings/business/settings_shortcut_messages.cpp @@ -394,7 +394,9 @@ ShortcutMessages::ShortcutMessages( if (const auto item = _session->data().message(fullId)) { const auto media = item->media(); if (!media || media->webpage() || media->allowsEditCaption()) { - _composeControls->editMessage(fullId); + _composeControls->editMessage( + fullId, + _inner->getSelectedTextRange(item)); } } }, _inner->lifetime()); From 4abc68ab1c987e856f09d6d08672abc23f1bd6d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Novomesk=C3=BD?= Date: Fri, 15 Mar 2024 14:42:03 +0100 Subject: [PATCH 047/596] Upgrade dav1d, libavif, libde265, libheif --- Telegram/build/prepare/prepare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Telegram/build/prepare/prepare.py b/Telegram/build/prepare/prepare.py index 173f34c2b..7ce74596b 100644 --- a/Telegram/build/prepare/prepare.py +++ b/Telegram/build/prepare/prepare.py @@ -696,7 +696,7 @@ mac: """) stage('dav1d', """ - git clone -b 1.2.1 --depth 1 https://code.videolan.org/videolan/dav1d.git + git clone -b 1.4.1 --depth 1 https://code.videolan.org/videolan/dav1d.git cd dav1d win: if "%X8664%" equ "x64" ( @@ -774,7 +774,7 @@ mac: """) stage('libavif', """ - git clone -b v0.11.1 --depth 1 https://github.com/AOMediaCodec/libavif.git + git clone -b v1.0.4 --depth 1 https://github.com/AOMediaCodec/libavif.git cd libavif win: cmake . ^ @@ -804,7 +804,7 @@ mac: """) stage('libde265', """ - git clone --depth 1 -b v1.0.12 https://github.com/strukturag/libde265.git + git clone --depth 1 -b v1.0.15 https://github.com/strukturag/libde265.git cd libde265 win: cmake . ^ @@ -880,7 +880,7 @@ mac: """) stage('libheif', """ - git clone --depth 1 -b v1.16.2 https://github.com/strukturag/libheif.git + git clone --depth 1 -b v1.17.6 https://github.com/strukturag/libheif.git cd libheif win: %THIRDPARTY_DIR%\\msys64\\usr\\bin\\sed.exe -i 's/LIBHEIF_EXPORTS/LIBDE265_STATIC_BUILD/g' libheif/CMakeLists.txt From 0e428aebdba504ee8355964285d4fcca1ca579f2 Mon Sep 17 00:00:00 2001 From: John Preston Date: Sun, 17 Mar 2024 20:34:02 +0400 Subject: [PATCH 048/596] Update patches in Linux build. --- Telegram/build/docker/centos_env/Dockerfile | 2 +- snap/snapcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Telegram/build/docker/centos_env/Dockerfile b/Telegram/build/docker/centos_env/Dockerfile index 9ff771f41..f7b674422 100644 --- a/Telegram/build/docker/centos_env/Dockerfile +++ b/Telegram/build/docker/centos_env/Dockerfile @@ -54,7 +54,7 @@ FROM builder AS patches RUN git init patches \ && cd patches \ && git remote add origin {{ GIT }}/desktop-app/patches.git \ - && git fetch --depth=1 origin b8390aef3db43f973486c6e512f4e825ffd09691 \ + && git fetch --depth=1 origin 75140a10eefcecaad7b3f3aa4586b880a39007ff \ && git reset --hard FETCH_HEAD \ && rm -rf .git diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 95e95f375..dbb12885e 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -168,7 +168,7 @@ parts: patches: source: https://github.com/desktop-app/patches.git source-depth: 1 - source-commit: b8390aef3db43f973486c6e512f4e825ffd09691 + source-commit: 75140a10eefcecaad7b3f3aa4586b880a39007ff plugin: dump override-pull: | craftctl default From 602b0b5d9b83a7c7486d86f7a029c64954a4f142 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Mon, 18 Mar 2024 09:57:44 +0400 Subject: [PATCH 049/596] Use the File::OpenUrl routine in IV --- Telegram/SourceFiles/iv/iv_instance.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Telegram/SourceFiles/iv/iv_instance.cpp b/Telegram/SourceFiles/iv/iv_instance.cpp index 673535cc7..50545ed32 100644 --- a/Telegram/SourceFiles/iv/iv_instance.cpp +++ b/Telegram/SourceFiles/iv/iv_instance.cpp @@ -47,7 +47,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "window/window_session_controller.h" #include "window/window_session_controller_link_info.h" -#include #include namespace Iv { @@ -832,7 +831,7 @@ void Instance::show( processJoinChannel(event.context); break; case Type::OpenLinkExternal: - QDesktopServices::openUrl(event.url); + File::OpenUrl(event.url); closeAll(); break; case Type::OpenMedia: From 07ab8756555f4a410d043d2aa3bf4bd13fc22731 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Mon, 18 Mar 2024 09:57:55 +0400 Subject: [PATCH 050/596] Update lib_base --- Telegram/lib_base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Telegram/lib_base b/Telegram/lib_base index 928fe24bb..216aed4e8 160000 --- a/Telegram/lib_base +++ b/Telegram/lib_base @@ -1 +1 @@ -Subproject commit 928fe24bb723bbebd4bcc87792da54b796d0b5a9 +Subproject commit 216aed4e8d518571e33df26fef918edcac8122f9 From 846a6d8717b6a8e0c125f79f049734f87ec07010 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Mon, 18 Mar 2024 17:57:44 +0400 Subject: [PATCH 051/596] Update scudo to 18.1.1 --- Telegram/ThirdParty/scudo/CMakeLists.txt | 13 +- Telegram/ThirdParty/scudo/allocator_common.h | 85 +++ Telegram/ThirdParty/scudo/allocator_config.h | 53 +- Telegram/ThirdParty/scudo/atomic_helpers.h | 8 +- .../scudo/benchmarks/malloc_benchmark.cpp | 4 - Telegram/ThirdParty/scudo/chunk.h | 13 - Telegram/ThirdParty/scudo/combined.h | 230 ++++---- Telegram/ThirdParty/scudo/common.cpp | 18 - Telegram/ThirdParty/scudo/common.h | 38 +- .../ThirdParty/scudo/condition_variable.h | 60 ++ .../scudo/condition_variable_base.h | 56 ++ .../scudo/condition_variable_linux.cpp | 52 ++ .../scudo/condition_variable_linux.h | 38 ++ Telegram/ThirdParty/scudo/flags.cpp | 3 + Telegram/ThirdParty/scudo/flags.inc | 12 +- Telegram/ThirdParty/scudo/flags_parser.cpp | 30 +- Telegram/ThirdParty/scudo/flags_parser.h | 3 +- .../scudo/fuzz/get_error_info_fuzzer.cpp | 9 +- .../scudo/include/scudo/interface.h | 17 +- Telegram/ThirdParty/scudo/linux.cpp | 50 +- Telegram/ThirdParty/scudo/local_cache.h | 125 +--- Telegram/ThirdParty/scudo/mem_map.h | 5 +- Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp | 12 +- Telegram/ThirdParty/scudo/mem_map_linux.cpp | 153 +++++ Telegram/ThirdParty/scudo/mem_map_linux.h | 67 +++ Telegram/ThirdParty/scudo/mutex.h | 19 +- Telegram/ThirdParty/scudo/options.h | 2 +- Telegram/ThirdParty/scudo/platform.h | 14 + Telegram/ThirdParty/scudo/primary32.h | 298 +++++++--- Telegram/ThirdParty/scudo/primary64.h | 551 +++++++++++------- Telegram/ThirdParty/scudo/release.cpp | 3 +- Telegram/ThirdParty/scudo/release.h | 171 +++--- Telegram/ThirdParty/scudo/report.cpp | 33 +- Telegram/ThirdParty/scudo/report.h | 8 +- Telegram/ThirdParty/scudo/report_linux.cpp | 58 ++ Telegram/ThirdParty/scudo/report_linux.h | 34 ++ .../ThirdParty/scudo/rss_limit_checker.cpp | 37 -- Telegram/ThirdParty/scudo/rss_limit_checker.h | 63 -- Telegram/ThirdParty/scudo/secondary.h | 186 ++++-- Telegram/ThirdParty/scudo/size_class_map.h | 26 +- Telegram/ThirdParty/scudo/stack_depot.h | 5 +- .../ThirdParty/scudo/tests/CMakeLists.txt | 15 +- .../ThirdParty/scudo/tests/chunk_test.cpp | 23 - .../ThirdParty/scudo/tests/combined_test.cpp | 188 ++++-- .../ThirdParty/scudo/tests/common_test.cpp | 23 - .../scudo/tests/condition_variable_test.cpp | 59 ++ .../ThirdParty/scudo/tests/memtag_test.cpp | 10 +- .../ThirdParty/scudo/tests/primary_test.cpp | 43 +- .../ThirdParty/scudo/tests/release_test.cpp | 42 +- .../ThirdParty/scudo/tests/report_test.cpp | 1 - .../scudo/tests/scudo_hooks_test.cpp | 114 ---- .../ThirdParty/scudo/tests/scudo_unit_test.h | 6 + .../scudo/tests/scudo_unit_test_main.cpp | 4 +- .../scudo/tests/size_class_map_test.cpp | 4 - Telegram/ThirdParty/scudo/tests/tsd_test.cpp | 4 + .../scudo/tests/wrappers_c_test.cpp | 201 ++++++- .../scudo/tests/wrappers_cpp_test.cpp | 143 +++-- Telegram/ThirdParty/scudo/trusty.cpp | 6 +- Telegram/ThirdParty/scudo/tsd.h | 17 +- Telegram/ThirdParty/scudo/tsd_shared.h | 5 + Telegram/ThirdParty/scudo/vector.h | 51 +- Telegram/ThirdParty/scudo/wrappers_c.cpp | 3 + Telegram/ThirdParty/scudo/wrappers_c.inc | 108 +++- .../ThirdParty/scudo/wrappers_c_bionic.cpp | 21 +- Telegram/ThirdParty/scudo/wrappers_cpp.cpp | 66 ++- 65 files changed, 2424 insertions(+), 1395 deletions(-) create mode 100644 Telegram/ThirdParty/scudo/allocator_common.h create mode 100644 Telegram/ThirdParty/scudo/condition_variable.h create mode 100644 Telegram/ThirdParty/scudo/condition_variable_base.h create mode 100644 Telegram/ThirdParty/scudo/condition_variable_linux.cpp create mode 100644 Telegram/ThirdParty/scudo/condition_variable_linux.h create mode 100644 Telegram/ThirdParty/scudo/mem_map_linux.cpp create mode 100644 Telegram/ThirdParty/scudo/mem_map_linux.h create mode 100644 Telegram/ThirdParty/scudo/report_linux.cpp create mode 100644 Telegram/ThirdParty/scudo/report_linux.h delete mode 100644 Telegram/ThirdParty/scudo/rss_limit_checker.cpp delete mode 100644 Telegram/ThirdParty/scudo/rss_limit_checker.h create mode 100644 Telegram/ThirdParty/scudo/tests/condition_variable_test.cpp delete mode 100644 Telegram/ThirdParty/scudo/tests/scudo_hooks_test.cpp diff --git a/Telegram/ThirdParty/scudo/CMakeLists.txt b/Telegram/ThirdParty/scudo/CMakeLists.txt index 094c23a73..60092005c 100644 --- a/Telegram/ThirdParty/scudo/CMakeLists.txt +++ b/Telegram/ThirdParty/scudo/CMakeLists.txt @@ -25,7 +25,7 @@ append_list_if(COMPILER_RT_HAS_WNO_PEDANTIC -Wno-pedantic SCUDO_CFLAGS) append_list_if(COMPILER_RT_HAS_FNO_LTO_FLAG -fno-lto SCUDO_CFLAGS) if(COMPILER_RT_DEBUG) - list(APPEND SCUDO_CFLAGS -O0 -DSCUDO_DEBUG=1) + list(APPEND SCUDO_CFLAGS -O0 -DSCUDO_DEBUG=1 -DSCUDO_ENABLE_HOOKS=1) else() list(APPEND SCUDO_CFLAGS -O3) endif() @@ -56,11 +56,15 @@ if(ANDROID) endif() set(SCUDO_HEADERS + allocator_common.h allocator_config.h atomic_helpers.h bytemap.h checksum.h chunk.h + condition_variable.h + condition_variable_base.h + condition_variable_linux.h combined.h common.h flags_parser.h @@ -74,6 +78,7 @@ set(SCUDO_HEADERS mem_map.h mem_map_base.h mem_map_fuchsia.h + mem_map_linux.h mutex.h options.h platform.h @@ -82,7 +87,7 @@ set(SCUDO_HEADERS quarantine.h release.h report.h - rss_limit_checker.h + report_linux.h secondary.h size_class_map.h stack_depot.h @@ -102,6 +107,7 @@ set(SCUDO_HEADERS set(SCUDO_SOURCES checksum.cpp common.cpp + condition_variable_linux.cpp crc32_hw.cpp flags_parser.cpp flags.cpp @@ -109,9 +115,10 @@ set(SCUDO_SOURCES linux.cpp mem_map.cpp mem_map_fuchsia.cpp + mem_map_linux.cpp release.cpp report.cpp - rss_limit_checker.cpp + report_linux.cpp string_utils.cpp timing.cpp ) diff --git a/Telegram/ThirdParty/scudo/allocator_common.h b/Telegram/ThirdParty/scudo/allocator_common.h new file mode 100644 index 000000000..95f4776ac --- /dev/null +++ b/Telegram/ThirdParty/scudo/allocator_common.h @@ -0,0 +1,85 @@ +//===-- allocator_common.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ALLOCATOR_COMMON_H_ +#define SCUDO_ALLOCATOR_COMMON_H_ + +#include "common.h" +#include "list.h" + +namespace scudo { + +template struct TransferBatch { + typedef typename SizeClassAllocator::SizeClassMap SizeClassMap; + typedef typename SizeClassAllocator::CompactPtrT CompactPtrT; + + static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint; + void setFromArray(CompactPtrT *Array, u16 N) { + DCHECK_LE(N, MaxNumCached); + Count = N; + memcpy(Batch, Array, sizeof(Batch[0]) * Count); + } + void appendFromArray(CompactPtrT *Array, u16 N) { + DCHECK_LE(N, MaxNumCached - Count); + memcpy(Batch + Count, Array, sizeof(Batch[0]) * N); + // u16 will be promoted to int by arithmetic type conversion. + Count = static_cast(Count + N); + } + void appendFromTransferBatch(TransferBatch *B, u16 N) { + DCHECK_LE(N, MaxNumCached - Count); + DCHECK_GE(B->Count, N); + // Append from the back of `B`. + memcpy(Batch + Count, B->Batch + (B->Count - N), sizeof(Batch[0]) * N); + // u16 will be promoted to int by arithmetic type conversion. + Count = static_cast(Count + N); + B->Count = static_cast(B->Count - N); + } + void clear() { Count = 0; } + void add(CompactPtrT P) { + DCHECK_LT(Count, MaxNumCached); + Batch[Count++] = P; + } + void moveToArray(CompactPtrT *Array) { + memcpy(Array, Batch, sizeof(Batch[0]) * Count); + clear(); + } + u16 getCount() const { return Count; } + bool isEmpty() const { return Count == 0U; } + CompactPtrT get(u16 I) const { + DCHECK_LE(I, Count); + return Batch[I]; + } + TransferBatch *Next; + +private: + CompactPtrT Batch[MaxNumCached]; + u16 Count; +}; + +// A BatchGroup is used to collect blocks. Each group has a group id to +// identify the group kind of contained blocks. +template struct BatchGroup { + // `Next` is used by IntrusiveList. + BatchGroup *Next; + // The compact base address of each group + uptr CompactPtrGroupBase; + // Cache value of SizeClassAllocatorLocalCache::getMaxCached() + u16 MaxCachedPerBatch; + // Number of blocks pushed into this group. This is an increment-only + // counter. + uptr PushedBlocks; + // This is used to track how many bytes are not in-use since last time we + // tried to release pages. + uptr BytesInBGAtLastCheckpoint; + // Blocks are managed by TransferBatch in a list. + SinglyLinkedList> Batches; +}; + +} // namespace scudo + +#endif // SCUDO_ALLOCATOR_COMMON_H_ diff --git a/Telegram/ThirdParty/scudo/allocator_config.h b/Telegram/ThirdParty/scudo/allocator_config.h index 315a04f76..3c6aa3acb 100644 --- a/Telegram/ThirdParty/scudo/allocator_config.h +++ b/Telegram/ThirdParty/scudo/allocator_config.h @@ -11,6 +11,7 @@ #include "combined.h" #include "common.h" +#include "condition_variable.h" #include "flags.h" #include "primary32.h" #include "primary64.h" @@ -82,6 +83,14 @@ namespace scudo { // // Defines the minimal & maximal release interval that can be set. // static const s32 MinReleaseToOsIntervalMs = INT32_MIN; // static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; +// +// // Use condition variable to shorten the waiting time of refillment of +// // freelist. Note that this depends on the implementation of condition +// // variable on each platform and the performance may vary so that it +// // doesn't guarantee a performance benefit. +// // Note that both variables have to be defined to enable it. +// static const bool UseConditionVariable = true; +// using ConditionVariableT = ConditionVariableLinux; // }; // // Defines the type of Primary allocator to use. // template using PrimaryT = SizeClassAllocator64; @@ -195,50 +204,6 @@ struct AndroidConfig { template using SecondaryT = MapAllocator; }; -struct AndroidSvelteConfig { - static const bool MaySupportMemoryTagging = false; - template - using TSDRegistryT = TSDRegistrySharedT; // Shared, max 2 TSDs. - - struct Primary { - using SizeClassMap = SvelteSizeClassMap; -#if SCUDO_CAN_USE_PRIMARY64 - static const uptr RegionSizeLog = 27U; - typedef u32 CompactPtrT; - static const uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; - static const uptr GroupSizeLog = 18U; - static const bool EnableRandomOffset = true; - static const uptr MapSizeIncrement = 1UL << 18; -#else - static const uptr RegionSizeLog = 16U; - static const uptr GroupSizeLog = 16U; - typedef uptr CompactPtrT; -#endif - static const s32 MinReleaseToOsIntervalMs = 1000; - static const s32 MaxReleaseToOsIntervalMs = 1000; - }; - -#if SCUDO_CAN_USE_PRIMARY64 - template using PrimaryT = SizeClassAllocator64; -#else - template using PrimaryT = SizeClassAllocator32; -#endif - - struct Secondary { - struct Cache { - static const u32 EntriesArraySize = 16U; - static const u32 QuarantineSize = 32U; - static const u32 DefaultMaxEntriesCount = 4U; - static const uptr DefaultMaxEntrySize = 1UL << 18; - static const s32 MinReleaseToOsIntervalMs = 0; - static const s32 MaxReleaseToOsIntervalMs = 0; - }; - template using CacheT = MapAllocatorCache; - }; - - template using SecondaryT = MapAllocator; -}; - #if SCUDO_CAN_USE_PRIMARY64 struct FuchsiaConfig { static const bool MaySupportMemoryTagging = false; diff --git a/Telegram/ThirdParty/scudo/atomic_helpers.h b/Telegram/ThirdParty/scudo/atomic_helpers.h index d88f5d7be..a68ffd162 100644 --- a/Telegram/ThirdParty/scudo/atomic_helpers.h +++ b/Telegram/ThirdParty/scudo/atomic_helpers.h @@ -133,10 +133,10 @@ inline void atomic_store_relaxed(volatile T *A, typename T::Type V) { } template -inline typename T::Type atomic_compare_exchange(volatile T *A, - typename T::Type Cmp, - typename T::Type Xchg) { - atomic_compare_exchange_strong(A, &Cmp, Xchg, memory_order_acquire); +inline typename T::Type +atomic_compare_exchange_strong(volatile T *A, typename T::Type Cmp, + typename T::Type Xchg, memory_order MO) { + atomic_compare_exchange_strong(A, &Cmp, Xchg, MO); return Cmp; } diff --git a/Telegram/ThirdParty/scudo/benchmarks/malloc_benchmark.cpp b/Telegram/ThirdParty/scudo/benchmarks/malloc_benchmark.cpp index 2adec88da..4fb05b761 100644 --- a/Telegram/ThirdParty/scudo/benchmarks/malloc_benchmark.cpp +++ b/Telegram/ThirdParty/scudo/benchmarks/malloc_benchmark.cpp @@ -52,8 +52,6 @@ static const size_t MaxSize = 128 * 1024; // cleanly. BENCHMARK_TEMPLATE(BM_malloc_free, scudo::AndroidConfig) ->Range(MinSize, MaxSize); -BENCHMARK_TEMPLATE(BM_malloc_free, scudo::AndroidSvelteConfig) - ->Range(MinSize, MaxSize); #if SCUDO_CAN_USE_PRIMARY64 BENCHMARK_TEMPLATE(BM_malloc_free, scudo::FuchsiaConfig) ->Range(MinSize, MaxSize); @@ -99,8 +97,6 @@ static const size_t MaxIters = 32 * 1024; // cleanly. BENCHMARK_TEMPLATE(BM_malloc_free_loop, scudo::AndroidConfig) ->Range(MinIters, MaxIters); -BENCHMARK_TEMPLATE(BM_malloc_free_loop, scudo::AndroidSvelteConfig) - ->Range(MinIters, MaxIters); #if SCUDO_CAN_USE_PRIMARY64 BENCHMARK_TEMPLATE(BM_malloc_free_loop, scudo::FuchsiaConfig) ->Range(MinIters, MaxIters); diff --git a/Telegram/ThirdParty/scudo/chunk.h b/Telegram/ThirdParty/scudo/chunk.h index 32874a8df..9228df047 100644 --- a/Telegram/ThirdParty/scudo/chunk.h +++ b/Telegram/ThirdParty/scudo/chunk.h @@ -128,19 +128,6 @@ inline void loadHeader(u32 Cookie, const void *Ptr, reportHeaderCorruption(const_cast(Ptr)); } -inline void compareExchangeHeader(u32 Cookie, void *Ptr, - UnpackedHeader *NewUnpackedHeader, - UnpackedHeader *OldUnpackedHeader) { - NewUnpackedHeader->Checksum = - computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader); - PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); - PackedHeader OldPackedHeader = bit_cast(*OldUnpackedHeader); - if (UNLIKELY(!atomic_compare_exchange_strong( - getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader, - memory_order_relaxed))) - reportHeaderRace(Ptr); -} - inline bool isValid(u32 Cookie, const void *Ptr, UnpackedHeader *NewUnpackedHeader) { PackedHeader NewPackedHeader = atomic_load_relaxed(getConstAtomicHeader(Ptr)); diff --git a/Telegram/ThirdParty/scudo/combined.h b/Telegram/ThirdParty/scudo/combined.h index b17acc71f..4624f83d1 100644 --- a/Telegram/ThirdParty/scudo/combined.h +++ b/Telegram/ThirdParty/scudo/combined.h @@ -14,11 +14,11 @@ #include "flags.h" #include "flags_parser.h" #include "local_cache.h" +#include "mem_map.h" #include "memtag.h" #include "options.h" #include "quarantine.h" #include "report.h" -#include "rss_limit_checker.h" #include "secondary.h" #include "stack_depot.h" #include "string_utils.h" @@ -68,14 +68,13 @@ public: if (UNLIKELY(Header.State != Chunk::State::Quarantined)) reportInvalidChunkState(AllocatorAction::Recycling, Ptr); - Chunk::UnpackedHeader NewHeader = Header; - NewHeader.State = Chunk::State::Available; - Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); + Header.State = Chunk::State::Available; + Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(Ptr); - void *BlockBegin = Allocator::getBlockBegin(Ptr, &NewHeader); - Cache.deallocate(NewHeader.ClassId, BlockBegin); + void *BlockBegin = Allocator::getBlockBegin(Ptr, &Header); + Cache.deallocate(Header.ClassId, BlockBegin); } // We take a shortcut when allocating a quarantine batch by working with the @@ -118,9 +117,8 @@ public: DCHECK_EQ(Header.Offset, 0); DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch)); - Chunk::UnpackedHeader NewHeader = Header; - NewHeader.State = Chunk::State::Available; - Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); + Header.State = Chunk::State::Available; + Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); Cache.deallocate(QuarantineClassId, reinterpret_cast(reinterpret_cast(Ptr) - Chunk::getHeaderSize())); @@ -149,9 +147,6 @@ public: initFlags(); reportUnrecognizedFlags(); - RssChecker.init(scudo::getFlags()->soft_rss_limit_mb, - scudo::getFlags()->hard_rss_limit_mb); - // Store some flags locally. if (getFlags()->may_return_null) Primary.Options.set(OptionBit::MayReturnNull); @@ -251,12 +246,14 @@ public: // - unlinking the local stats from the global ones (destroying the cache does // the last two items). void commitBack(TSD *TSD) { + TSD->assertLocked(/*BypassCheck=*/true); Quarantine.drain(&TSD->getQuarantineCache(), QuarantineCallback(*this, TSD->getCache())); TSD->getCache().destroy(&Stats); } void drainCache(TSD *TSD) { + TSD->assertLocked(/*BypassCheck=*/true); Quarantine.drainAndRecycle(&TSD->getQuarantineCache(), QuarantineCallback(*this, TSD->getCache())); TSD->getCache().drain(); @@ -299,7 +296,7 @@ public: #endif } - uptr computeOddEvenMaskForPointerMaybe(Options Options, uptr Ptr, + uptr computeOddEvenMaskForPointerMaybe(const Options &Options, uptr Ptr, uptr ClassId) { if (!Options.get(OptionBit::UseOddEvenTags)) return 0; @@ -329,8 +326,6 @@ public: #ifdef GWP_ASAN_HOOKS if (UNLIKELY(GuardedAlloc.shouldSample())) { if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) { - if (UNLIKELY(&__scudo_allocate_hook)) - __scudo_allocate_hook(Ptr, Size); Stats.lock(); Stats.add(StatAllocated, GuardedAllocSlotSize); Stats.sub(StatFree, GuardedAllocSlotSize); @@ -363,19 +358,6 @@ public: } DCHECK_LE(Size, NeededSize); - switch (RssChecker.getRssLimitExceeded()) { - case RssLimitChecker::Neither: - break; - case RssLimitChecker::Soft: - if (Options.get(OptionBit::MayReturnNull)) - return nullptr; - reportSoftRSSLimit(RssChecker.getSoftRssLimit()); - break; - case RssLimitChecker::Hard: - reportHardRSSLimit(RssChecker.getHardRssLimit()); - break; - } - void *Block = nullptr; uptr ClassId = 0; uptr SecondaryBlockEnd = 0; @@ -384,11 +366,11 @@ public: DCHECK_NE(ClassId, 0U); bool UnlockRequired; auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); Block = TSD->getCache().allocate(ClassId); - // If the allocation failed, the most likely reason with a 32-bit primary - // is the region being full. In that event, retry in each successively - // larger class until it fits. If it fails to fit in the largest class, - // fallback to the Secondary. + // If the allocation failed, retry in each successively larger class until + // it fits. If it fails to fit in the largest class, fallback to the + // Secondary. if (UNLIKELY(!Block)) { while (ClassId < SizeClassMap::LargestClassId && !Block) Block = TSD->getCache().allocate(++ClassId); @@ -406,6 +388,7 @@ public: if (UNLIKELY(!Block)) { if (Options.get(OptionBit::MayReturnNull)) return nullptr; + printStats(); reportOutOfMemory(NeededSize); } @@ -535,14 +518,14 @@ public: Chunk::SizeOrUnusedBytesMask; Chunk::storeHeader(Cookie, Ptr, &Header); - if (UNLIKELY(&__scudo_allocate_hook)) - __scudo_allocate_hook(TaggedPtr, Size); - return TaggedPtr; } NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0, UNUSED uptr Alignment = MinAlignment) { + if (UNLIKELY(!Ptr)) + return; + // For a deallocation, we only ensure minimal initialization, meaning thread // local data will be left uninitialized for now (when using ELF TLS). The // fallback cache will be used instead. This is a workaround for a situation @@ -551,12 +534,6 @@ public: // being destroyed properly. Any other heap operation will do a full init. initThreadMaybe(/*MinimalInit=*/true); - if (UNLIKELY(&__scudo_deallocate_hook)) - __scudo_deallocate_hook(Ptr); - - if (UNLIKELY(!Ptr)) - return; - #ifdef GWP_ASAN_HOOKS if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) { GuardedAlloc.deallocate(Ptr); @@ -635,47 +612,46 @@ public: if (UNLIKELY(!isAligned(reinterpret_cast(OldPtr), MinAlignment))) reportMisalignedPointer(AllocatorAction::Reallocating, OldPtr); - Chunk::UnpackedHeader OldHeader; - Chunk::loadHeader(Cookie, OldPtr, &OldHeader); + Chunk::UnpackedHeader Header; + Chunk::loadHeader(Cookie, OldPtr, &Header); - if (UNLIKELY(OldHeader.State != Chunk::State::Allocated)) + if (UNLIKELY(Header.State != Chunk::State::Allocated)) reportInvalidChunkState(AllocatorAction::Reallocating, OldPtr); // Pointer has to be allocated with a malloc-type function. Some // applications think that it is OK to realloc a memalign'ed pointer, which // will trigger this check. It really isn't. if (Options.get(OptionBit::DeallocTypeMismatch)) { - if (UNLIKELY(OldHeader.OriginOrWasZeroed != Chunk::Origin::Malloc)) + if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Malloc)) reportDeallocTypeMismatch(AllocatorAction::Reallocating, OldPtr, - OldHeader.OriginOrWasZeroed, + Header.OriginOrWasZeroed, Chunk::Origin::Malloc); } - void *BlockBegin = getBlockBegin(OldTaggedPtr, &OldHeader); + void *BlockBegin = getBlockBegin(OldTaggedPtr, &Header); uptr BlockEnd; uptr OldSize; - const uptr ClassId = OldHeader.ClassId; + const uptr ClassId = Header.ClassId; if (LIKELY(ClassId)) { BlockEnd = reinterpret_cast(BlockBegin) + SizeClassMap::getSizeByClassId(ClassId); - OldSize = OldHeader.SizeOrUnusedBytes; + OldSize = Header.SizeOrUnusedBytes; } else { BlockEnd = SecondaryT::getBlockEnd(BlockBegin); OldSize = BlockEnd - (reinterpret_cast(OldTaggedPtr) + - OldHeader.SizeOrUnusedBytes); + Header.SizeOrUnusedBytes); } // If the new chunk still fits in the previously allocated block (with a // reasonable delta), we just keep the old block, and update the chunk // header to reflect the size change. if (reinterpret_cast(OldTaggedPtr) + NewSize <= BlockEnd) { if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) { - Chunk::UnpackedHeader NewHeader = OldHeader; - NewHeader.SizeOrUnusedBytes = + Header.SizeOrUnusedBytes = (ClassId ? NewSize : BlockEnd - (reinterpret_cast(OldTaggedPtr) + NewSize)) & Chunk::SizeOrUnusedBytesMask; - Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader); + Chunk::storeHeader(Cookie, OldPtr, &Header); if (UNLIKELY(useMemoryTagging(Options))) { if (ClassId) { resizeTaggedChunk(reinterpret_cast(OldTaggedPtr) + OldSize, @@ -697,9 +673,7 @@ public: void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment); if (LIKELY(NewPtr)) { memcpy(NewPtr, OldTaggedPtr, Min(NewSize, OldSize)); - if (UNLIKELY(&__scudo_deallocate_hook)) - __scudo_deallocate_hook(OldTaggedPtr); - quarantineOrDeallocateChunk(Options, OldTaggedPtr, &OldHeader, OldSize); + quarantineOrDeallocateChunk(Options, OldTaggedPtr, &Header, OldSize); } return NewPtr; } @@ -754,6 +728,13 @@ public: Str.output(); } + void printFragmentationInfo() { + ScopedString Str; + Primary.getFragmentationInfo(&Str); + // Secondary allocator dumps the fragmentation data in getStats(). + Str.output(); + } + void releaseToOS(ReleaseToOS ReleaseType) { initThreadMaybe(); if (ReleaseType == ReleaseToOS::ForceAll) @@ -847,10 +828,15 @@ public: // for it, which then forces realloc to copy the usable size of a chunk as // opposed to its actual size. uptr getUsableSize(const void *Ptr) { - initThreadMaybe(); if (UNLIKELY(!Ptr)) return 0; + return getAllocSize(Ptr); + } + + uptr getAllocSize(const void *Ptr) { + initThreadMaybe(); + #ifdef GWP_ASAN_HOOKS if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) return GuardedAlloc.getSize(Ptr); @@ -859,9 +845,11 @@ public: Ptr = getHeaderTaggedPointer(const_cast(Ptr)); Chunk::UnpackedHeader Header; Chunk::loadHeader(Cookie, Ptr, &Header); - // Getting the usable size of a chunk only makes sense if it's allocated. + + // Getting the alloc size of a chunk only makes sense if it's allocated. if (UNLIKELY(Header.State != Chunk::State::Allocated)) reportInvalidChunkState(AllocatorAction::Sizing, const_cast(Ptr)); + return getSize(Ptr, &Header); } @@ -887,13 +875,6 @@ public: Header.State == Chunk::State::Allocated; } - void setRssLimitsTestOnly(int SoftRssLimitMb, int HardRssLimitMb, - bool MayReturnNull) { - RssChecker.init(SoftRssLimitMb, HardRssLimitMb); - if (MayReturnNull) - Primary.Options.set(OptionBit::MayReturnNull); - } - bool useMemoryTaggingTestOnly() const { return useMemoryTagging(Primary.Options.load()); } @@ -913,7 +894,7 @@ public: void setTrackAllocationStacks(bool Track) { initThreadMaybe(); - if (getFlags()->allocation_ring_buffer_size == 0) { + if (getFlags()->allocation_ring_buffer_size <= 0) { DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks)); return; } @@ -955,21 +936,7 @@ public: uptr getRingBufferSize() { initThreadMaybe(); - auto *RingBuffer = getRingBuffer(); - return RingBuffer ? ringBufferSizeInBytes(RingBuffer->Size) : 0; - } - - static bool setRingBufferSizeForBuffer(char *Buffer, size_t Size) { - // Need at least one entry. - if (Size < sizeof(AllocationRingBuffer) + - sizeof(typename AllocationRingBuffer::Entry)) { - return false; - } - AllocationRingBuffer *RingBuffer = - reinterpret_cast(Buffer); - RingBuffer->Size = (Size - sizeof(AllocationRingBuffer)) / - sizeof(typename AllocationRingBuffer::Entry); - return true; + return RingBufferElements ? ringBufferSizeInBytes(RingBufferElements) : 0; } static const uptr MaxTraceSize = 64; @@ -986,8 +953,9 @@ public: static void getErrorInfo(struct scudo_error_info *ErrorInfo, uintptr_t FaultAddr, const char *DepotPtr, const char *RegionInfoPtr, const char *RingBufferPtr, - const char *Memory, const char *MemoryTags, - uintptr_t MemoryAddr, size_t MemorySize) { + size_t RingBufferSize, const char *Memory, + const char *MemoryTags, uintptr_t MemoryAddr, + size_t MemorySize) { *ErrorInfo = {}; if (!allocatorSupportsMemoryTagging() || MemoryAddr + MemorySize < MemoryAddr) @@ -1006,7 +974,7 @@ public: // Check the ring buffer. For primary allocations this will only find UAF; // for secondary allocations we can find either UAF or OOB. getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot, - RingBufferPtr); + RingBufferPtr, RingBufferSize); // Check for OOB in the 28 blocks surrounding the 3 we checked earlier. // Beyond that we are likely to hit false positives. @@ -1053,7 +1021,6 @@ private: QuarantineT Quarantine; TSDRegistryT TSDRegistry; pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT; - RssLimitChecker RssChecker; #ifdef GWP_ASAN_HOOKS gwp_asan::GuardedPoolAllocator GuardedAlloc; @@ -1073,13 +1040,14 @@ private: }; atomic_uptr Pos; - u32 Size; // An array of Size (at least one) elements of type Entry is immediately // following to this struct. }; // Pointer to memory mapped area starting with AllocationRingBuffer struct, // and immediately followed by Size elements of type Entry. char *RawRingBuffer = {}; + u32 RingBufferElements = 0; + MemMapT RawRingBufferMap; // The following might get optimized out by the compiler. NOINLINE void performSanityChecks() { @@ -1134,35 +1102,34 @@ private: reinterpret_cast(Ptr) - SizeOrUnusedBytes; } - void quarantineOrDeallocateChunk(Options Options, void *TaggedPtr, + void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr, Chunk::UnpackedHeader *Header, uptr Size) NO_THREAD_SAFETY_ANALYSIS { void *Ptr = getHeaderTaggedPointer(TaggedPtr); - Chunk::UnpackedHeader NewHeader = *Header; // If the quarantine is disabled, the actual size of a chunk is 0 or larger // than the maximum allowed, we return a chunk directly to the backend. // This purposefully underflows for Size == 0. const bool BypassQuarantine = !Quarantine.getCacheSize() || ((Size - 1) >= QuarantineMaxChunkSize) || - !NewHeader.ClassId; + !Header->ClassId; if (BypassQuarantine) - NewHeader.State = Chunk::State::Available; + Header->State = Chunk::State::Available; else - NewHeader.State = Chunk::State::Quarantined; - NewHeader.OriginOrWasZeroed = useMemoryTagging(Options) && - NewHeader.ClassId && - !TSDRegistry.getDisableMemInit(); - Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header); + Header->State = Chunk::State::Quarantined; + Header->OriginOrWasZeroed = useMemoryTagging(Options) && + Header->ClassId && + !TSDRegistry.getDisableMemInit(); + Chunk::storeHeader(Cookie, Ptr, Header); if (UNLIKELY(useMemoryTagging(Options))) { u8 PrevTag = extractTag(reinterpret_cast(TaggedPtr)); storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size); - if (NewHeader.ClassId) { + if (Header->ClassId) { if (!TSDRegistry.getDisableMemInit()) { uptr TaggedBegin, TaggedEnd; const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe( - Options, reinterpret_cast(getBlockBegin(Ptr, &NewHeader)), - NewHeader.ClassId); + Options, reinterpret_cast(getBlockBegin(Ptr, Header)), + Header->ClassId); // Exclude the previous tag so that immediate use after free is // detected 100% of the time. setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin, @@ -1173,11 +1140,12 @@ private: if (BypassQuarantine) { if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(Ptr); - void *BlockBegin = getBlockBegin(Ptr, &NewHeader); - const uptr ClassId = NewHeader.ClassId; + void *BlockBegin = getBlockBegin(Ptr, Header); + const uptr ClassId = Header->ClassId; if (LIKELY(ClassId)) { bool UnlockRequired; auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); const bool CacheDrained = TSD->getCache().deallocate(ClassId, BlockBegin); if (UnlockRequired) @@ -1197,6 +1165,7 @@ private: } else { bool UnlockRequired; auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); Quarantine.put(&TSD->getQuarantineCache(), QuarantineCallback(*this, TSD->getCache()), Ptr, Size); if (UnlockRequired) @@ -1273,7 +1242,7 @@ private: storeEndMarker(RoundNewPtr, NewSize, BlockEnd); } - void storePrimaryAllocationStackMaybe(Options Options, void *Ptr) { + void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) { if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks))) return; auto *Ptr32 = reinterpret_cast(Ptr); @@ -1286,7 +1255,7 @@ private: u32 DeallocationTid) { uptr Pos = atomic_fetch_add(&getRingBuffer()->Pos, 1, memory_order_relaxed); typename AllocationRingBuffer::Entry *Entry = - getRingBufferEntry(RawRingBuffer, Pos % getRingBuffer()->Size); + getRingBufferEntry(RawRingBuffer, Pos % RingBufferElements); // First invalidate our entry so that we don't attempt to interpret a // partially written state in getSecondaryErrorInfo(). The fences below @@ -1305,7 +1274,7 @@ private: atomic_store_relaxed(&Entry->Ptr, reinterpret_cast(Ptr)); } - void storeSecondaryAllocationStackMaybe(Options Options, void *Ptr, + void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr, uptr Size) { if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks))) return; @@ -1320,8 +1289,8 @@ private: storeRingBufferEntry(untagPointer(Ptr), Trace, Tid, Size, 0, 0); } - void storeDeallocationStackMaybe(Options Options, void *Ptr, u8 PrevTag, - uptr Size) { + void storeDeallocationStackMaybe(const Options &Options, void *Ptr, + u8 PrevTag, uptr Size) { if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks))) return; @@ -1427,17 +1396,19 @@ private: size_t &NextErrorReport, uintptr_t FaultAddr, const StackDepot *Depot, - const char *RingBufferPtr) { + const char *RingBufferPtr, + size_t RingBufferSize) { auto *RingBuffer = reinterpret_cast(RingBufferPtr); - if (!RingBuffer || RingBuffer->Size == 0) + size_t RingBufferElements = ringBufferElementsFromBytes(RingBufferSize); + if (!RingBuffer || RingBufferElements == 0) return; uptr Pos = atomic_load_relaxed(&RingBuffer->Pos); - for (uptr I = Pos - 1; - I != Pos - 1 - RingBuffer->Size && NextErrorReport != NumErrorReports; + for (uptr I = Pos - 1; I != Pos - 1 - RingBufferElements && + NextErrorReport != NumErrorReports; --I) { - auto *Entry = getRingBufferEntry(RingBufferPtr, I % RingBuffer->Size); + auto *Entry = getRingBufferEntry(RingBufferPtr, I % RingBufferElements); uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr); if (!EntryPtr) continue; @@ -1516,17 +1487,19 @@ private: } void mapAndInitializeRingBuffer() { + if (getFlags()->allocation_ring_buffer_size <= 0) + return; u32 AllocationRingBufferSize = static_cast(getFlags()->allocation_ring_buffer_size); - if (AllocationRingBufferSize < 1) - return; - RawRingBuffer = static_cast( - map(/*Addr=*/nullptr, - roundUp(ringBufferSizeInBytes(AllocationRingBufferSize), - getPageSizeCached()), - "AllocatorRingBuffer")); - auto *RingBuffer = reinterpret_cast(RawRingBuffer); - RingBuffer->Size = AllocationRingBufferSize; + MemMapT MemMap; + MemMap.map( + /*Addr=*/0U, + roundUp(ringBufferSizeInBytes(AllocationRingBufferSize), + getPageSizeCached()), + "scudo:ring_buffer"); + RawRingBuffer = reinterpret_cast(MemMap.getBase()); + RawRingBufferMap = MemMap; + RingBufferElements = AllocationRingBufferSize; static_assert(sizeof(AllocationRingBuffer) % alignof(typename AllocationRingBuffer::Entry) == 0, @@ -1534,14 +1507,25 @@ private: } void unmapRingBuffer() { - unmap(RawRingBuffer, roundUp(getRingBufferSize(), getPageSizeCached())); + auto *RingBuffer = getRingBuffer(); + if (RingBuffer != nullptr) { + RawRingBufferMap.unmap(RawRingBufferMap.getBase(), + RawRingBufferMap.getCapacity()); + } RawRingBuffer = nullptr; } - static constexpr size_t ringBufferSizeInBytes(u32 AllocationRingBufferSize) { + static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) { return sizeof(AllocationRingBuffer) + - AllocationRingBufferSize * - sizeof(typename AllocationRingBuffer::Entry); + RingBufferElements * sizeof(typename AllocationRingBuffer::Entry); + } + + static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) { + if (Bytes < sizeof(AllocationRingBuffer)) { + return 0; + } + return (Bytes - sizeof(AllocationRingBuffer)) / + sizeof(typename AllocationRingBuffer::Entry); } inline AllocationRingBuffer *getRingBuffer() { diff --git a/Telegram/ThirdParty/scudo/common.cpp b/Telegram/ThirdParty/scudo/common.cpp index 9f14faeef..06e930638 100644 --- a/Telegram/ThirdParty/scudo/common.cpp +++ b/Telegram/ThirdParty/scudo/common.cpp @@ -21,22 +21,4 @@ uptr getPageSizeSlow() { return PageSizeCached; } -// Fatal internal map() or unmap() error (potentially OOM related). -void NORETURN dieOnMapUnmapError(uptr SizeIfOOM) { - char Error[128] = "Scudo ERROR: internal map or unmap failure\n"; - if (SizeIfOOM) { - formatString( - Error, sizeof(Error), - "Scudo ERROR: internal map failure (NO MEMORY) requesting %zuKB\n", - SizeIfOOM >> 10); - } - outputRaw(Error); - setAbortMessage(Error); - die(); -} - -#if !SCUDO_LINUX -uptr GetRSS() { return 0; } -#endif - } // namespace scudo diff --git a/Telegram/ThirdParty/scudo/common.h b/Telegram/ThirdParty/scudo/common.h index 82e6cf4ae..ae45683f1 100644 --- a/Telegram/ThirdParty/scudo/common.h +++ b/Telegram/ThirdParty/scudo/common.h @@ -17,6 +17,7 @@ #include #include +#include namespace scudo { @@ -111,19 +112,19 @@ template inline void shuffle(T *A, u32 N, u32 *RandState) { *RandState = State; } -// Hardware specific inlinable functions. +inline void computePercentage(uptr Numerator, uptr Denominator, uptr *Integral, + uptr *Fractional) { + constexpr uptr Digits = 100; + if (Denominator == 0) { + *Integral = 100; + *Fractional = 0; + return; + } -inline void yieldProcessor(UNUSED u8 Count) { -#if defined(__i386__) || defined(__x86_64__) - __asm__ __volatile__("" ::: "memory"); - for (u8 I = 0; I < Count; I++) - __asm__ __volatile__("pause"); -#elif defined(__aarch64__) || defined(__arm__) - __asm__ __volatile__("" ::: "memory"); - for (u8 I = 0; I < Count; I++) - __asm__ __volatile__("yield"); -#endif - __asm__ __volatile__("" ::: "memory"); + *Integral = Numerator * Digits / Denominator; + *Fractional = + (((Numerator * Digits) % Denominator) * Digits + Denominator / 2) / + Denominator; } // Platform specific functions. @@ -131,9 +132,10 @@ inline void yieldProcessor(UNUSED u8 Count) { extern uptr PageSizeCached; uptr getPageSizeSlow(); inline uptr getPageSizeCached() { - // Bionic uses a hardcoded value. - if (SCUDO_ANDROID) - return 4096U; +#if SCUDO_ANDROID && defined(PAGE_SIZE) + // Most Android builds have a build-time constant page size. + return PAGE_SIZE; +#endif if (LIKELY(PageSizeCached)) return PageSizeCached; return getPageSizeSlow(); @@ -144,8 +146,6 @@ u32 getNumberOfCPUs(); const char *getEnv(const char *Name); -uptr GetRSS(); - u64 getMonotonicTime(); // Gets the time faster but with less accuracy. Can call getMonotonicTime // if no fast version is available. @@ -190,10 +190,6 @@ void setMemoryPermission(uptr Addr, uptr Size, uptr Flags, void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, MapPlatformData *Data = nullptr); -// Internal map & unmap fatal error. This must not call map(). SizeIfOOM shall -// hold the requested size on an out-of-memory error, 0 otherwise. -void NORETURN dieOnMapUnmapError(uptr SizeIfOOM = 0); - // Logging related functions. void setAbortMessage(const char *Message); diff --git a/Telegram/ThirdParty/scudo/condition_variable.h b/Telegram/ThirdParty/scudo/condition_variable.h new file mode 100644 index 000000000..4afebdc9d --- /dev/null +++ b/Telegram/ThirdParty/scudo/condition_variable.h @@ -0,0 +1,60 @@ +//===-- condition_variable.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CONDITION_VARIABLE_H_ +#define SCUDO_CONDITION_VARIABLE_H_ + +#include "condition_variable_base.h" + +#include "common.h" +#include "platform.h" + +#include "condition_variable_linux.h" + +namespace scudo { + +// A default implementation of default condition variable. It doesn't do a real +// `wait`, instead it spins a short amount of time only. +class ConditionVariableDummy + : public ConditionVariableBase { +public: + void notifyAllImpl(UNUSED HybridMutex &M) REQUIRES(M) {} + + void waitImpl(UNUSED HybridMutex &M) REQUIRES(M) { + M.unlock(); + + constexpr u32 SpinTimes = 64; + volatile u32 V = 0; + for (u32 I = 0; I < SpinTimes; ++I) { + u32 Tmp = V + 1; + V = Tmp; + } + + M.lock(); + } +}; + +template +struct ConditionVariableState { + static constexpr bool enabled() { return false; } + // This is only used for compilation purpose so that we won't end up having + // many conditional compilations. If you want to use `ConditionVariableDummy`, + // define `ConditionVariableT` in your allocator configuration. See + // allocator_config.h for more details. + using ConditionVariableT = ConditionVariableDummy; +}; + +template +struct ConditionVariableState { + static constexpr bool enabled() { return Config::UseConditionVariable; } + using ConditionVariableT = typename Config::ConditionVariableT; +}; + +} // namespace scudo + +#endif // SCUDO_CONDITION_VARIABLE_H_ diff --git a/Telegram/ThirdParty/scudo/condition_variable_base.h b/Telegram/ThirdParty/scudo/condition_variable_base.h new file mode 100644 index 000000000..416c327fe --- /dev/null +++ b/Telegram/ThirdParty/scudo/condition_variable_base.h @@ -0,0 +1,56 @@ +//===-- condition_variable_base.h -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CONDITION_VARIABLE_BASE_H_ +#define SCUDO_CONDITION_VARIABLE_BASE_H_ + +#include "mutex.h" +#include "thread_annotations.h" + +namespace scudo { + +template class ConditionVariableBase { +public: + constexpr ConditionVariableBase() = default; + + void bindTestOnly(HybridMutex &Mutex) { +#if SCUDO_DEBUG + boundMutex = &Mutex; +#else + (void)Mutex; +#endif + } + + void notifyAll(HybridMutex &M) REQUIRES(M) { +#if SCUDO_DEBUG + CHECK_EQ(&M, boundMutex); +#endif + getDerived()->notifyAllImpl(M); + } + + void wait(HybridMutex &M) REQUIRES(M) { +#if SCUDO_DEBUG + CHECK_EQ(&M, boundMutex); +#endif + getDerived()->waitImpl(M); + } + +protected: + Derived *getDerived() { return static_cast(this); } + +#if SCUDO_DEBUG + // Because thread-safety analysis doesn't support pointer aliasing, we are not + // able to mark the proper annotations without false positive. Instead, we + // pass the lock and do the same-lock check separately. + HybridMutex *boundMutex = nullptr; +#endif +}; + +} // namespace scudo + +#endif // SCUDO_CONDITION_VARIABLE_BASE_H_ diff --git a/Telegram/ThirdParty/scudo/condition_variable_linux.cpp b/Telegram/ThirdParty/scudo/condition_variable_linux.cpp new file mode 100644 index 000000000..e6d9bd177 --- /dev/null +++ b/Telegram/ThirdParty/scudo/condition_variable_linux.cpp @@ -0,0 +1,52 @@ +//===-- condition_variable_linux.cpp ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_LINUX + +#include "condition_variable_linux.h" + +#include "atomic_helpers.h" + +#include +#include +#include +#include + +namespace scudo { + +void ConditionVariableLinux::notifyAllImpl(UNUSED HybridMutex &M) { + const u32 V = atomic_load_relaxed(&Counter); + atomic_store_relaxed(&Counter, V + 1); + + // TODO(chiahungduan): Move the waiters from the futex waiting queue + // `Counter` to futex waiting queue `M` so that the awoken threads won't be + // blocked again due to locked `M` by current thread. + if (LastNotifyAll != V) { + syscall(SYS_futex, reinterpret_cast(&Counter), FUTEX_WAKE_PRIVATE, + INT_MAX, nullptr, nullptr, 0); + } + + LastNotifyAll = V + 1; +} + +void ConditionVariableLinux::waitImpl(HybridMutex &M) { + const u32 V = atomic_load_relaxed(&Counter) + 1; + atomic_store_relaxed(&Counter, V); + + // TODO: Use ScopedUnlock when it's supported. + M.unlock(); + syscall(SYS_futex, reinterpret_cast(&Counter), FUTEX_WAIT_PRIVATE, V, + nullptr, nullptr, 0); + M.lock(); +} + +} // namespace scudo + +#endif // SCUDO_LINUX diff --git a/Telegram/ThirdParty/scudo/condition_variable_linux.h b/Telegram/ThirdParty/scudo/condition_variable_linux.h new file mode 100644 index 000000000..cd0732873 --- /dev/null +++ b/Telegram/ThirdParty/scudo/condition_variable_linux.h @@ -0,0 +1,38 @@ +//===-- condition_variable_linux.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CONDITION_VARIABLE_LINUX_H_ +#define SCUDO_CONDITION_VARIABLE_LINUX_H_ + +#include "platform.h" + +#if SCUDO_LINUX + +#include "atomic_helpers.h" +#include "condition_variable_base.h" +#include "thread_annotations.h" + +namespace scudo { + +class ConditionVariableLinux + : public ConditionVariableBase { +public: + void notifyAllImpl(HybridMutex &M) REQUIRES(M); + + void waitImpl(HybridMutex &M) REQUIRES(M); + +private: + u32 LastNotifyAll = 0; + atomic_u32 Counter = {}; +}; + +} // namespace scudo + +#endif // SCUDO_LINUX + +#endif // SCUDO_CONDITION_VARIABLE_LINUX_H_ diff --git a/Telegram/ThirdParty/scudo/flags.cpp b/Telegram/ThirdParty/scudo/flags.cpp index de5153b28..f498edfbd 100644 --- a/Telegram/ThirdParty/scudo/flags.cpp +++ b/Telegram/ThirdParty/scudo/flags.cpp @@ -68,6 +68,9 @@ void initFlags() { Parser.parseString(getCompileDefinitionScudoDefaultOptions()); Parser.parseString(getScudoDefaultOptions()); Parser.parseString(getEnv("SCUDO_OPTIONS")); + if (const char *V = getEnv("SCUDO_ALLOCATION_RING_BUFFER_SIZE")) { + Parser.parseStringPair("allocation_ring_buffer_size", V); + } } } // namespace scudo diff --git a/Telegram/ThirdParty/scudo/flags.inc b/Telegram/ThirdParty/scudo/flags.inc index c1f153baf..f5a2bab50 100644 --- a/Telegram/ThirdParty/scudo/flags.inc +++ b/Telegram/ThirdParty/scudo/flags.inc @@ -46,14 +46,6 @@ SCUDO_FLAG(int, release_to_os_interval_ms, SCUDO_ANDROID ? INT32_MIN : 5000, "Interval (in milliseconds) at which to attempt release of unused " "memory to the OS. Negative values disable the feature.") -SCUDO_FLAG(int, hard_rss_limit_mb, 0, - "Hard RSS Limit in Mb. If non-zero, once the limit is achieved, " - "abort the process") - -SCUDO_FLAG(int, soft_rss_limit_mb, 0, - "Soft RSS Limit in Mb. If non-zero, once the limit is reached, all " - "subsequent calls will fail or return NULL until the RSS goes below " - "the soft limit") - SCUDO_FLAG(int, allocation_ring_buffer_size, 32768, - "Entries to keep in the allocation ring buffer for scudo.") + "Entries to keep in the allocation ring buffer for scudo. " + "Values less or equal to zero disable the buffer.") diff --git a/Telegram/ThirdParty/scudo/flags_parser.cpp b/Telegram/ThirdParty/scudo/flags_parser.cpp index be39fcd4f..3d8c6f378 100644 --- a/Telegram/ThirdParty/scudo/flags_parser.cpp +++ b/Telegram/ThirdParty/scudo/flags_parser.cpp @@ -10,6 +10,8 @@ #include "common.h" #include "report.h" +#include +#include #include #include @@ -80,7 +82,7 @@ void FlagParser::parseFlag() { ++Pos; Value = Buffer + ValueStart; } - if (!runHandler(Name, Value)) + if (!runHandler(Name, Value, '=')) reportError("flag parsing failed."); } @@ -122,10 +124,16 @@ inline bool parseBool(const char *Value, bool *b) { return false; } -bool FlagParser::runHandler(const char *Name, const char *Value) { +void FlagParser::parseStringPair(const char *Name, const char *Value) { + if (!runHandler(Name, Value, '\0')) + reportError("flag parsing failed."); +} + +bool FlagParser::runHandler(const char *Name, const char *Value, + const char Sep) { for (u32 I = 0; I < NumberOfFlags; ++I) { const uptr Len = strlen(Flags[I].Name); - if (strncmp(Name, Flags[I].Name, Len) != 0 || Name[Len] != '=') + if (strncmp(Name, Flags[I].Name, Len) != 0 || Name[Len] != Sep) continue; bool Ok = false; switch (Flags[I].Type) { @@ -136,12 +144,18 @@ bool FlagParser::runHandler(const char *Name, const char *Value) { break; case FlagType::FT_int: char *ValueEnd; - *reinterpret_cast(Flags[I].Var) = - static_cast(strtol(Value, &ValueEnd, 10)); - Ok = - *ValueEnd == '"' || *ValueEnd == '\'' || isSeparatorOrNull(*ValueEnd); - if (!Ok) + errno = 0; + long V = strtol(Value, &ValueEnd, 10); + if (errno != 0 || // strtol failed (over or underflow) + V > INT_MAX || V < INT_MIN || // overflows integer + // contains unexpected characters + (*ValueEnd != '"' && *ValueEnd != '\'' && + !isSeparatorOrNull(*ValueEnd))) { reportInvalidFlag("int", Value); + break; + } + *reinterpret_cast(Flags[I].Var) = static_cast(V); + Ok = true; break; } return Ok; diff --git a/Telegram/ThirdParty/scudo/flags_parser.h b/Telegram/ThirdParty/scudo/flags_parser.h index ba832adbd..ded496fda 100644 --- a/Telegram/ThirdParty/scudo/flags_parser.h +++ b/Telegram/ThirdParty/scudo/flags_parser.h @@ -27,6 +27,7 @@ public: void *Var); void parseString(const char *S); void printFlagDescriptions(); + void parseStringPair(const char *Name, const char *Value); private: static const u32 MaxFlags = 20; @@ -45,7 +46,7 @@ private: void skipWhitespace(); void parseFlags(); void parseFlag(); - bool runHandler(const char *Name, const char *Value); + bool runHandler(const char *Name, const char *Value, char Sep); }; void reportUnrecognizedFlags(); diff --git a/Telegram/ThirdParty/scudo/fuzz/get_error_info_fuzzer.cpp b/Telegram/ThirdParty/scudo/fuzz/get_error_info_fuzzer.cpp index 74456450a..5b01ebe11 100644 --- a/Telegram/ThirdParty/scudo/fuzz/get_error_info_fuzzer.cpp +++ b/Telegram/ThirdParty/scudo/fuzz/get_error_info_fuzzer.cpp @@ -46,14 +46,11 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *Data, size_t Size) { } std::string RingBufferBytes = FDP.ConsumeRemainingBytesAsString(); - // RingBuffer is too short. - if (!AllocatorT::setRingBufferSizeForBuffer(RingBufferBytes.data(), - RingBufferBytes.size())) - return 0; scudo_error_info ErrorInfo; AllocatorT::getErrorInfo(&ErrorInfo, FaultAddr, StackDepot.data(), - RegionInfo.data(), RingBufferBytes.data(), Memory, - MemoryTags, MemoryAddr, MemorySize); + RegionInfo.data(), RingBufferBytes.data(), + RingBufferBytes.size(), Memory, MemoryTags, + MemoryAddr, MemorySize); return 0; } diff --git a/Telegram/ThirdParty/scudo/include/scudo/interface.h b/Telegram/ThirdParty/scudo/include/scudo/interface.h index 6c0c521f8..a2dedea91 100644 --- a/Telegram/ThirdParty/scudo/include/scudo/interface.h +++ b/Telegram/ThirdParty/scudo/include/scudo/interface.h @@ -17,10 +17,22 @@ extern "C" { __attribute__((weak)) const char *__scudo_default_options(void); // Post-allocation & pre-deallocation hooks. -// They must be thread-safe and not use heap related functions. __attribute__((weak)) void __scudo_allocate_hook(void *ptr, size_t size); __attribute__((weak)) void __scudo_deallocate_hook(void *ptr); +// `realloc` involves both deallocation and allocation but they are not reported +// atomically. In one specific case which may keep taking a snapshot right in +// the middle of `realloc` reporting the deallocation and allocation, it may +// confuse the user by missing memory from `realloc`. To alleviate that case, +// define the two `realloc` hooks to get the knowledge of the bundled hook +// calls. These hooks are optional and should only be used when a hooks user +// wants to track reallocs more closely. +// +// See more details in the comment of `realloc` in wrapper_c.inc. +__attribute__((weak)) void +__scudo_realloc_allocate_hook(void *old_ptr, void *new_ptr, size_t size); +__attribute__((weak)) void __scudo_realloc_deallocate_hook(void *old_ptr); + void __scudo_print_stats(void); typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg); @@ -73,7 +85,8 @@ typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg); // pointer. void __scudo_get_error_info(struct scudo_error_info *error_info, uintptr_t fault_addr, const char *stack_depot, - const char *region_info, const char *ring_buffer, + size_t stack_depot_size, const char *region_info, + const char *ring_buffer, size_t ring_buffer_size, const char *memory, const char *memory_tags, uintptr_t memory_addr, size_t memory_size); diff --git a/Telegram/ThirdParty/scudo/linux.cpp b/Telegram/ThirdParty/scudo/linux.cpp index e285d8a3d..274695108 100644 --- a/Telegram/ThirdParty/scudo/linux.cpp +++ b/Telegram/ThirdParty/scudo/linux.cpp @@ -14,6 +14,7 @@ #include "internal_defs.h" #include "linux.h" #include "mutex.h" +#include "report_linux.h" #include "string_utils.h" #include @@ -43,6 +44,7 @@ uptr getPageSize() { return static_cast(sysconf(_SC_PAGESIZE)); } void NORETURN die() { abort(); } +// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead. void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, UNUSED MapPlatformData *Data) { int MmapFlags = MAP_PRIVATE | MAP_ANONYMOUS; @@ -65,7 +67,7 @@ void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, void *P = mmap(Addr, Size, MmapProt, MmapFlags, -1, 0); if (P == MAP_FAILED) { if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM) - dieOnMapUnmapError(errno == ENOMEM ? Size : 0); + reportMapError(errno == ENOMEM ? Size : 0); return nullptr; } #if SCUDO_ANDROID @@ -75,19 +77,22 @@ void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, return P; } +// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead. void unmap(void *Addr, uptr Size, UNUSED uptr Flags, UNUSED MapPlatformData *Data) { if (munmap(Addr, Size) != 0) - dieOnMapUnmapError(); + reportUnmapError(reinterpret_cast(Addr), Size); } +// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead. void setMemoryPermission(uptr Addr, uptr Size, uptr Flags, UNUSED MapPlatformData *Data) { int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE); if (mprotect(reinterpret_cast(Addr), Size, Prot) != 0) - dieOnMapUnmapError(); + reportProtectError(Addr, Size, Prot); } +// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead. void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, UNUSED MapPlatformData *Data) { void *Addr = reinterpret_cast(BaseAddress + Offset); @@ -104,12 +109,14 @@ enum State : u32 { Unlocked = 0, Locked = 1, Sleeping = 2 }; } bool HybridMutex::tryLock() { - return atomic_compare_exchange(&M, Unlocked, Locked) == Unlocked; + return atomic_compare_exchange_strong(&M, Unlocked, Locked, + memory_order_acquire) == Unlocked; } // The following is based on https://akkadia.org/drepper/futex.pdf. void HybridMutex::lockSlow() { - u32 V = atomic_compare_exchange(&M, Unlocked, Locked); + u32 V = atomic_compare_exchange_strong(&M, Unlocked, Locked, + memory_order_acquire); if (V == Unlocked) return; if (V != Sleeping) @@ -197,39 +204,6 @@ bool getRandom(void *Buffer, uptr Length, UNUSED bool Blocking) { extern "C" WEAK int async_safe_write_log(int pri, const char *tag, const char *msg); -static uptr GetRSSFromBuffer(const char *Buf) { - // The format of the file is: - // 1084 89 69 11 0 79 0 - // We need the second number which is RSS in pages. - const char *Pos = Buf; - // Skip the first number. - while (*Pos >= '0' && *Pos <= '9') - Pos++; - // Skip whitespaces. - while (!(*Pos >= '0' && *Pos <= '9') && *Pos != 0) - Pos++; - // Read the number. - u64 Rss = 0; - for (; *Pos >= '0' && *Pos <= '9'; Pos++) - Rss = Rss * 10 + static_cast(*Pos) - '0'; - return static_cast(Rss * getPageSizeCached()); -} - -uptr GetRSS() { - // TODO: We currently use sanitizer_common's GetRSS which reads the - // RSS from /proc/self/statm by default. We might want to - // call getrusage directly, even if it's less accurate. - auto Fd = open("/proc/self/statm", O_RDONLY); - char Buf[64]; - s64 Len = read(Fd, Buf, sizeof(Buf) - 1); - close(Fd); - if (Len <= 0) - return 0; - Buf[Len] = 0; - - return GetRSSFromBuffer(Buf); -} - void outputRaw(const char *Buffer) { if (&async_safe_write_log) { constexpr s32 AndroidLogInfo = 4; diff --git a/Telegram/ThirdParty/scudo/local_cache.h b/Telegram/ThirdParty/scudo/local_cache.h index 1095eb5f1..46d6affdc 100644 --- a/Telegram/ThirdParty/scudo/local_cache.h +++ b/Telegram/ThirdParty/scudo/local_cache.h @@ -22,80 +22,13 @@ template struct SizeClassAllocatorLocalCache { typedef typename SizeClassAllocator::SizeClassMap SizeClassMap; typedef typename SizeClassAllocator::CompactPtrT CompactPtrT; - struct TransferBatch { - static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint; - void setFromArray(CompactPtrT *Array, u16 N) { - DCHECK_LE(N, MaxNumCached); - Count = N; - memcpy(Batch, Array, sizeof(Batch[0]) * Count); - } - void appendFromArray(CompactPtrT *Array, u16 N) { - DCHECK_LE(N, MaxNumCached - Count); - memcpy(Batch + Count, Array, sizeof(Batch[0]) * N); - // u16 will be promoted to int by arithmetic type conversion. - Count = static_cast(Count + N); - } - void appendFromTransferBatch(TransferBatch *B, u16 N) { - DCHECK_LE(N, MaxNumCached - Count); - DCHECK_GE(B->Count, N); - // Append from the back of `B`. - memcpy(Batch + Count, B->Batch + (B->Count - N), sizeof(Batch[0]) * N); - // u16 will be promoted to int by arithmetic type conversion. - Count = static_cast(Count + N); - B->Count = static_cast(B->Count - N); - } - void clear() { Count = 0; } - void add(CompactPtrT P) { - DCHECK_LT(Count, MaxNumCached); - Batch[Count++] = P; - } - void copyToArray(CompactPtrT *Array) const { - memcpy(Array, Batch, sizeof(Batch[0]) * Count); - } - u16 getCount() const { return Count; } - bool isEmpty() const { return Count == 0U; } - CompactPtrT get(u16 I) const { - DCHECK_LE(I, Count); - return Batch[I]; - } - static u16 getMaxCached(uptr Size) { - return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size)); - } - TransferBatch *Next; - - private: - CompactPtrT Batch[MaxNumCached]; - u16 Count; - }; - - // A BatchGroup is used to collect blocks. Each group has a group id to - // identify the group kind of contained blocks. - struct BatchGroup { - // `Next` is used by IntrusiveList. - BatchGroup *Next; - // The compact base address of each group - uptr CompactPtrGroupBase; - // Cache value of TransferBatch::getMaxCached() - u16 MaxCachedPerBatch; - // Number of blocks pushed into this group. This is an increment-only - // counter. - uptr PushedBlocks; - // This is used to track how many bytes are not in-use since last time we - // tried to release pages. - uptr BytesInBGAtLastCheckpoint; - // Blocks are managed by TransferBatch in a list. - SinglyLinkedList Batches; - }; - - static_assert(sizeof(BatchGroup) <= sizeof(TransferBatch), - "BatchGroup uses the same class size as TransferBatch"); - void init(GlobalStats *S, SizeClassAllocator *A) { DCHECK(isEmpty()); Stats.init(); if (LIKELY(S)) S->link(&Stats); Allocator = A; + initCache(); } void destroy(GlobalStats *S) { @@ -108,7 +41,9 @@ template struct SizeClassAllocatorLocalCache { DCHECK_LT(ClassId, NumClasses); PerClass *C = &PerClassArray[ClassId]; if (C->Count == 0) { - if (UNLIKELY(!refill(C, ClassId))) + // Refill half of the number of max cached. + DCHECK_GT(C->MaxCount / 2, 0U); + if (UNLIKELY(!refill(C, ClassId, C->MaxCount / 2))) return nullptr; DCHECK_GT(C->Count, 0); } @@ -125,9 +60,6 @@ template struct SizeClassAllocatorLocalCache { bool deallocate(uptr ClassId, void *P) { CHECK_LT(ClassId, NumClasses); PerClass *C = &PerClassArray[ClassId]; - // We still have to initialize the cache in the event that the first heap - // operation in a thread is a deallocation. - initCacheMaybe(C); // If the cache is full, drain half of blocks back to the main allocator. const bool NeedToDrainCache = C->Count == C->MaxCount; @@ -151,7 +83,7 @@ template struct SizeClassAllocatorLocalCache { } void drain() { - // Drain BatchClassId last as createBatch can refill it. + // Drain BatchClassId last as it may be needed while draining normal blocks. for (uptr I = 0; I < NumClasses; ++I) { if (I == BatchClassId) continue; @@ -163,19 +95,11 @@ template struct SizeClassAllocatorLocalCache { DCHECK(isEmpty()); } - TransferBatch *createBatch(uptr ClassId, void *B) { - if (ClassId != BatchClassId) - B = allocate(BatchClassId); + void *getBatchClassBlock() { + void *B = allocate(BatchClassId); if (UNLIKELY(!B)) reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId)); - return reinterpret_cast(B); - } - - BatchGroup *createGroup() { - void *Ptr = allocate(BatchClassId); - if (UNLIKELY(!Ptr)) - reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId)); - return reinterpret_cast(Ptr); + return B; } LocalStats &getStats() { return Stats; } @@ -203,6 +127,11 @@ template struct SizeClassAllocatorLocalCache { Str->append(" No block is cached.\n"); } + static u16 getMaxCached(uptr Size) { + return Min(SizeClassMap::MaxNumCachedHint, + SizeClassMap::getMaxCachedHint(Size)); + } + private: static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr BatchClassId = SizeClassMap::BatchClassId; @@ -211,24 +140,17 @@ private: u16 MaxCount; // Note: ClassSize is zero for the transfer batch. uptr ClassSize; - CompactPtrT Chunks[2 * TransferBatch::MaxNumCached]; + CompactPtrT Chunks[2 * SizeClassMap::MaxNumCachedHint]; }; PerClass PerClassArray[NumClasses] = {}; LocalStats Stats; SizeClassAllocator *Allocator = nullptr; - ALWAYS_INLINE void initCacheMaybe(PerClass *C) { - if (LIKELY(C->MaxCount)) - return; - initCache(); - DCHECK_NE(C->MaxCount, 0U); - } - NOINLINE void initCache() { for (uptr I = 0; I < NumClasses; I++) { PerClass *P = &PerClassArray[I]; const uptr Size = SizeClassAllocator::getSizeByClassId(I); - P->MaxCount = static_cast(2 * TransferBatch::getMaxCached(Size)); + P->MaxCount = static_cast(2 * getMaxCached(Size)); if (I != BatchClassId) { P->ClassSize = Size; } else { @@ -244,17 +166,12 @@ private: deallocate(BatchClassId, B); } - NOINLINE bool refill(PerClass *C, uptr ClassId) { - initCacheMaybe(C); - TransferBatch *B = Allocator->popBatch(this, ClassId); - if (UNLIKELY(!B)) - return false; - DCHECK_GT(B->getCount(), 0); - C->Count = B->getCount(); - B->copyToArray(C->Chunks); - B->clear(); - destroyBatch(ClassId, B); - return true; + NOINLINE bool refill(PerClass *C, uptr ClassId, u16 MaxRefill) { + const u16 NumBlocksRefilled = + Allocator->popBlocks(this, ClassId, C->Chunks, MaxRefill); + DCHECK_LE(NumBlocksRefilled, MaxRefill); + C->Count = static_cast(C->Count + NumBlocksRefilled); + return NumBlocksRefilled != 0; } NOINLINE void drain(PerClass *C, uptr ClassId) { diff --git a/Telegram/ThirdParty/scudo/mem_map.h b/Telegram/ThirdParty/scudo/mem_map.h index 409e4dbbe..b92216cf2 100644 --- a/Telegram/ThirdParty/scudo/mem_map.h +++ b/Telegram/ThirdParty/scudo/mem_map.h @@ -22,6 +22,7 @@ #include "trusty.h" #include "mem_map_fuchsia.h" +#include "mem_map_linux.h" namespace scudo { @@ -73,10 +74,10 @@ private: }; #if SCUDO_LINUX -using ReservedMemoryT = ReservedMemoryDefault; +using ReservedMemoryT = ReservedMemoryLinux; using MemMapT = ReservedMemoryT::MemMapT; #elif SCUDO_FUCHSIA -using ReservedMemoryT = ReservedMemoryDefault; +using ReservedMemoryT = ReservedMemoryFuchsia; using MemMapT = ReservedMemoryT::MemMapT; #elif SCUDO_TRUSTY using ReservedMemoryT = ReservedMemoryDefault; diff --git a/Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp b/Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp index 9ace1fef7..0566ab065 100644 --- a/Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp +++ b/Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp @@ -41,7 +41,7 @@ static void setVmoName(zx_handle_t Vmo, const char *Name) { static uptr getRootVmarBase() { static atomic_uptr CachedResult = {0}; - uptr Result = atomic_load_relaxed(&CachedResult); + uptr Result = atomic_load(&CachedResult, memory_order_acquire); if (UNLIKELY(!Result)) { zx_info_vmar_t VmarInfo; zx_status_t Status = @@ -50,7 +50,7 @@ static uptr getRootVmarBase() { CHECK_EQ(Status, ZX_OK); CHECK_NE(VmarInfo.base, 0); - atomic_store_relaxed(&CachedResult, VmarInfo.base); + atomic_store(&CachedResult, VmarInfo.base, memory_order_release); Result = VmarInfo.base; } @@ -61,7 +61,7 @@ static uptr getRootVmarBase() { static zx_handle_t getPlaceholderVmo() { static atomic_u32 StoredVmo = {ZX_HANDLE_INVALID}; - zx_handle_t Vmo = atomic_load_relaxed(&StoredVmo); + zx_handle_t Vmo = atomic_load(&StoredVmo, memory_order_acquire); if (UNLIKELY(Vmo == ZX_HANDLE_INVALID)) { // Create a zero-sized placeholder VMO. zx_status_t Status = _zx_vmo_create(0, 0, &Vmo); @@ -72,9 +72,9 @@ static zx_handle_t getPlaceholderVmo() { // Atomically store its handle. If some other thread wins the race, use its // handle and discard ours. - zx_handle_t OldValue = - atomic_compare_exchange(&StoredVmo, ZX_HANDLE_INVALID, Vmo); - if (OldValue != ZX_HANDLE_INVALID) { + zx_handle_t OldValue = atomic_compare_exchange_strong( + &StoredVmo, ZX_HANDLE_INVALID, Vmo, memory_order_acq_rel); + if (UNLIKELY(OldValue != ZX_HANDLE_INVALID)) { Status = _zx_handle_close(Vmo); CHECK_EQ(Status, ZX_OK); diff --git a/Telegram/ThirdParty/scudo/mem_map_linux.cpp b/Telegram/ThirdParty/scudo/mem_map_linux.cpp new file mode 100644 index 000000000..783c4f0d9 --- /dev/null +++ b/Telegram/ThirdParty/scudo/mem_map_linux.cpp @@ -0,0 +1,153 @@ +//===-- mem_map_linux.cpp ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_LINUX + +#include "mem_map_linux.h" + +#include "common.h" +#include "internal_defs.h" +#include "linux.h" +#include "mutex.h" +#include "report_linux.h" +#include "string_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if SCUDO_ANDROID +// TODO(chiahungduan): Review if we still need the followings macros. +#include +// Definitions of prctl arguments to set a vma name in Android kernels. +#define ANDROID_PR_SET_VMA 0x53564d41 +#define ANDROID_PR_SET_VMA_ANON_NAME 0 +#endif + +namespace scudo { + +static void *mmapWrapper(uptr Addr, uptr Size, const char *Name, uptr Flags) { + int MmapFlags = MAP_PRIVATE | MAP_ANONYMOUS; + int MmapProt; + if (Flags & MAP_NOACCESS) { + MmapFlags |= MAP_NORESERVE; + MmapProt = PROT_NONE; + } else { + MmapProt = PROT_READ | PROT_WRITE; + } +#if defined(__aarch64__) +#ifndef PROT_MTE +#define PROT_MTE 0x20 +#endif + if (Flags & MAP_MEMTAG) + MmapProt |= PROT_MTE; +#endif + if (Addr) + MmapFlags |= MAP_FIXED; + void *P = + mmap(reinterpret_cast(Addr), Size, MmapProt, MmapFlags, -1, 0); + if (P == MAP_FAILED) { + if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM) + reportMapError(errno == ENOMEM ? Size : 0); + return nullptr; + } +#if SCUDO_ANDROID + if (Name) + prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, P, Size, Name); +#else + (void)Name; +#endif + + return P; +} + +bool MemMapLinux::mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags) { + void *P = mmapWrapper(Addr, Size, Name, Flags); + if (P == nullptr) + return false; + + MapBase = reinterpret_cast(P); + MapCapacity = Size; + return true; +} + +void MemMapLinux::unmapImpl(uptr Addr, uptr Size) { + // If we unmap all the pages, also mark `MapBase` to 0 to indicate invalid + // status. + if (Size == MapCapacity) { + MapBase = MapCapacity = 0; + } else { + // This is partial unmap and is unmapping the pages from the beginning, + // shift `MapBase` to the new base. + if (MapBase == Addr) + MapBase = Addr + Size; + MapCapacity -= Size; + } + + if (munmap(reinterpret_cast(Addr), Size) != 0) + reportUnmapError(Addr, Size); +} + +bool MemMapLinux::remapImpl(uptr Addr, uptr Size, const char *Name, + uptr Flags) { + void *P = mmapWrapper(Addr, Size, Name, Flags); + if (reinterpret_cast(P) != Addr) + reportMapError(); + return true; +} + +void MemMapLinux::setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags) { + int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE); + if (mprotect(reinterpret_cast(Addr), Size, Prot) != 0) + reportProtectError(Addr, Size, Prot); +} + +void MemMapLinux::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) { + void *Addr = reinterpret_cast(From); + + while (madvise(Addr, Size, MADV_DONTNEED) == -1 && errno == EAGAIN) { + } +} + +bool ReservedMemoryLinux::createImpl(uptr Addr, uptr Size, const char *Name, + uptr Flags) { + ReservedMemoryLinux::MemMapT MemMap; + if (!MemMap.map(Addr, Size, Name, Flags | MAP_NOACCESS)) + return false; + + MapBase = MemMap.getBase(); + MapCapacity = MemMap.getCapacity(); + + return true; +} + +void ReservedMemoryLinux::releaseImpl() { + if (munmap(reinterpret_cast(getBase()), getCapacity()) != 0) + reportUnmapError(getBase(), getCapacity()); +} + +ReservedMemoryLinux::MemMapT ReservedMemoryLinux::dispatchImpl(uptr Addr, + uptr Size) { + return ReservedMemoryLinux::MemMapT(Addr, Size); +} + +} // namespace scudo + +#endif // SCUDO_LINUX diff --git a/Telegram/ThirdParty/scudo/mem_map_linux.h b/Telegram/ThirdParty/scudo/mem_map_linux.h new file mode 100644 index 000000000..7a89b3bff --- /dev/null +++ b/Telegram/ThirdParty/scudo/mem_map_linux.h @@ -0,0 +1,67 @@ +//===-- mem_map_linux.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_MEM_MAP_LINUX_H_ +#define SCUDO_MEM_MAP_LINUX_H_ + +#include "platform.h" + +#if SCUDO_LINUX + +#include "common.h" +#include "mem_map_base.h" + +namespace scudo { + +class MemMapLinux final : public MemMapBase { +public: + constexpr MemMapLinux() = default; + MemMapLinux(uptr Base, uptr Capacity) + : MapBase(Base), MapCapacity(Capacity) {} + + // Impls for base functions. + bool mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags = 0); + void unmapImpl(uptr Addr, uptr Size); + bool remapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags = 0); + void setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags); + void releasePagesToOSImpl(uptr From, uptr Size) { + return releaseAndZeroPagesToOSImpl(From, Size); + } + void releaseAndZeroPagesToOSImpl(uptr From, uptr Size); + uptr getBaseImpl() { return MapBase; } + uptr getCapacityImpl() { return MapCapacity; } + +private: + uptr MapBase = 0; + uptr MapCapacity = 0; +}; + +// This will be deprecated when every allocator has been supported by each +// platform's `MemMap` implementation. +class ReservedMemoryLinux final + : public ReservedMemory { +public: + // The following two are the Impls for function in `MemMapBase`. + uptr getBaseImpl() { return MapBase; } + uptr getCapacityImpl() { return MapCapacity; } + + // These threes are specific to `ReservedMemory`. + bool createImpl(uptr Addr, uptr Size, const char *Name, uptr Flags); + void releaseImpl(); + MemMapT dispatchImpl(uptr Addr, uptr Size); + +private: + uptr MapBase = 0; + uptr MapCapacity = 0; +}; + +} // namespace scudo + +#endif // SCUDO_LINUX + +#endif // SCUDO_MEM_MAP_LINUX_H_ diff --git a/Telegram/ThirdParty/scudo/mutex.h b/Telegram/ThirdParty/scudo/mutex.h index 05340de3e..4caa94521 100644 --- a/Telegram/ThirdParty/scudo/mutex.h +++ b/Telegram/ThirdParty/scudo/mutex.h @@ -35,7 +35,7 @@ public: #pragma nounroll #endif for (u8 I = 0U; I < NumberOfTries; I++) { - yieldProcessor(NumberOfYields); + delayLoop(); if (tryLock()) return; } @@ -53,10 +53,23 @@ public: } private: + void delayLoop() { + // The value comes from the average time spent in accessing caches (which + // are the fastest operations) so that we are unlikely to wait too long for + // fast operations. + constexpr u32 SpinTimes = 16; + volatile u32 V = 0; + for (u32 I = 0; I < SpinTimes; ++I) { + u32 Tmp = V + 1; + V = Tmp; + } + } + void assertHeldImpl(); - static constexpr u8 NumberOfTries = 8U; - static constexpr u8 NumberOfYields = 8U; + // TODO(chiahungduan): Adapt this value based on scenarios. E.g., primary and + // secondary allocator have different allocation times. + static constexpr u8 NumberOfTries = 32U; #if SCUDO_LINUX atomic_u32 M = {}; diff --git a/Telegram/ThirdParty/scudo/options.h b/Telegram/ThirdParty/scudo/options.h index 4e6786513..b20142a41 100644 --- a/Telegram/ThirdParty/scudo/options.h +++ b/Telegram/ThirdParty/scudo/options.h @@ -38,7 +38,7 @@ struct Options { } }; -template bool useMemoryTagging(Options Options) { +template bool useMemoryTagging(const Options &Options) { return allocatorSupportsMemoryTagging() && Options.get(OptionBit::UseMemoryTagging); } diff --git a/Telegram/ThirdParty/scudo/platform.h b/Telegram/ThirdParty/scudo/platform.h index 7c7024ff5..b71a86be7 100644 --- a/Telegram/ThirdParty/scudo/platform.h +++ b/Telegram/ThirdParty/scudo/platform.h @@ -63,6 +63,20 @@ #define SCUDO_CAN_USE_MTE (SCUDO_LINUX || SCUDO_TRUSTY) #endif +// Use smaller table sizes for fuzzing in order to reduce input size. +// Trusty just has less available memory. +#ifndef SCUDO_SMALL_STACK_DEPOT +#if defined(SCUDO_FUZZ) || SCUDO_TRUSTY +#define SCUDO_SMALL_STACK_DEPOT 1 +#else +#define SCUDO_SMALL_STACK_DEPOT 0 +#endif +#endif + +#ifndef SCUDO_ENABLE_HOOKS +#define SCUDO_ENABLE_HOOKS 0 +#endif + #ifndef SCUDO_MIN_ALIGNMENT_LOG // We force malloc-type functions to be aligned to std::max_align_t, but there // is no reason why the minimum alignment for all other functions can't be 8 diff --git a/Telegram/ThirdParty/scudo/primary32.h b/Telegram/ThirdParty/scudo/primary32.h index 1d8a34ec6..4d03b282d 100644 --- a/Telegram/ThirdParty/scudo/primary32.h +++ b/Telegram/ThirdParty/scudo/primary32.h @@ -9,6 +9,7 @@ #ifndef SCUDO_PRIMARY32_H_ #define SCUDO_PRIMARY32_H_ +#include "allocator_common.h" #include "bytemap.h" #include "common.h" #include "list.h" @@ -53,12 +54,15 @@ public: ""); typedef SizeClassAllocator32 ThisT; typedef SizeClassAllocatorLocalCache CacheT; - typedef typename CacheT::TransferBatch TransferBatch; - typedef typename CacheT::BatchGroup BatchGroup; + typedef TransferBatch TransferBatchT; + typedef BatchGroup BatchGroupT; + + static_assert(sizeof(BatchGroupT) <= sizeof(TransferBatchT), + "BatchGroupT uses the same class size as TransferBatchT"); static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) - ? sizeof(TransferBatch) + ? sizeof(TransferBatchT) : SizeClassMap::getSizeByClassId(ClassId); } @@ -126,7 +130,7 @@ public: SizeClassInfo *Sci = getSizeClassInfo(I); ScopedLock L1(Sci->Mutex); uptr TotalBlocks = 0; - for (BatchGroup &BG : Sci->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Sci->FreeListInfo.BlockList) { // `BG::Batches` are `TransferBatches`. +1 for `BatchGroup`. BatchClassUsedInFreeLists += BG.Batches.size() + 1; for (const auto &It : BG.Batches) @@ -141,7 +145,7 @@ public: SizeClassInfo *Sci = getSizeClassInfo(SizeClassMap::BatchClassId); ScopedLock L1(Sci->Mutex); uptr TotalBlocks = 0; - for (BatchGroup &BG : Sci->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Sci->FreeListInfo.BlockList) { if (LIKELY(!BG.Batches.empty())) { for (const auto &It : BG.Batches) TotalBlocks += It.getCount(); @@ -187,11 +191,30 @@ public: return BlockSize > PageSize; } - TransferBatch *popBatch(CacheT *C, uptr ClassId) { + // Note that the `MaxBlockCount` will be used when we support arbitrary blocks + // count. Now it's the same as the number of blocks stored in the + // `TransferBatch`. + u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray, + UNUSED const u16 MaxBlockCount) { + TransferBatchT *B = popBatch(C, ClassId); + if (!B) + return 0; + + const u16 Count = B->getCount(); + DCHECK_GT(Count, 0U); + B->moveToArray(ToArray); + + if (ClassId != SizeClassMap::BatchClassId) + C->deallocate(SizeClassMap::BatchClassId, B); + + return Count; + } + + TransferBatchT *popBatch(CacheT *C, uptr ClassId) { DCHECK_LT(ClassId, NumClasses); SizeClassInfo *Sci = getSizeClassInfo(ClassId); ScopedLock L(Sci->Mutex); - TransferBatch *B = popBatchImpl(C, ClassId, Sci); + TransferBatchT *B = popBatchImpl(C, ClassId, Sci); if (UNLIKELY(!B)) { if (UNLIKELY(!populateFreeList(C, ClassId, Sci))) return nullptr; @@ -311,6 +334,18 @@ public: } } + void getFragmentationInfo(ScopedString *Str) { + Str->append( + "Fragmentation Stats: SizeClassAllocator32: page size = %zu bytes\n", + getPageSizeCached()); + + for (uptr I = 1; I < NumClasses; I++) { + SizeClassInfo *Sci = getSizeClassInfo(I); + ScopedLock L(Sci->Mutex); + getSizeClassFragmentationInfo(Sci, I, Str); + } + } + bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { const s32 Interval = Max(Min(static_cast(Value), @@ -369,7 +404,7 @@ private: }; struct BlocksInfo { - SinglyLinkedList BlockList = {}; + SinglyLinkedList BlockList = {}; uptr PoppedBlocks = 0; uptr PushedBlocks = 0; }; @@ -493,11 +528,11 @@ private: // reusable and don't need additional space for them. Sci->FreeListInfo.PushedBlocks += Size; - BatchGroup *BG = Sci->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Sci->FreeListInfo.BlockList.front(); if (BG == nullptr) { // Construct `BatchGroup` on the last element. - BG = reinterpret_cast( + BG = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1])); --Size; BG->Batches.clear(); @@ -508,8 +543,8 @@ private: // from `CreateGroup` in `pushBlocksImpl` BG->PushedBlocks = 1; BG->BytesInBGAtLastCheckpoint = 0; - BG->MaxCachedPerBatch = TransferBatch::getMaxCached( - getSizeByClassId(SizeClassMap::BatchClassId)); + BG->MaxCachedPerBatch = + CacheT::getMaxCached(getSizeByClassId(SizeClassMap::BatchClassId)); Sci->FreeListInfo.BlockList.push_front(BG); } @@ -522,7 +557,7 @@ private: // 2. Only 1 block is pushed when the freelist is empty. if (BG->Batches.empty()) { // Construct the `TransferBatch` on the last element. - TransferBatch *TB = reinterpret_cast( + TransferBatchT *TB = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1])); TB->clear(); // As mentioned above, addresses of `TransferBatch` and `BatchGroup` are @@ -537,14 +572,14 @@ private: BG->Batches.push_front(TB); } - TransferBatch *CurBatch = BG->Batches.front(); + TransferBatchT *CurBatch = BG->Batches.front(); DCHECK_NE(CurBatch, nullptr); for (u32 I = 0; I < Size;) { u16 UnusedSlots = static_cast(BG->MaxCachedPerBatch - CurBatch->getCount()); if (UnusedSlots == 0) { - CurBatch = reinterpret_cast( + CurBatch = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[I])); CurBatch->clear(); // Self-contained @@ -588,24 +623,25 @@ private: DCHECK_GT(Size, 0U); auto CreateGroup = [&](uptr CompactPtrGroupBase) { - BatchGroup *BG = C->createGroup(); + BatchGroupT *BG = + reinterpret_cast(C->getBatchClassBlock()); BG->Batches.clear(); - TransferBatch *TB = C->createBatch(ClassId, nullptr); + TransferBatchT *TB = + reinterpret_cast(C->getBatchClassBlock()); TB->clear(); BG->CompactPtrGroupBase = CompactPtrGroupBase; BG->Batches.push_front(TB); BG->PushedBlocks = 0; BG->BytesInBGAtLastCheckpoint = 0; - BG->MaxCachedPerBatch = - TransferBatch::getMaxCached(getSizeByClassId(ClassId)); + BG->MaxCachedPerBatch = CacheT::getMaxCached(getSizeByClassId(ClassId)); return BG; }; - auto InsertBlocks = [&](BatchGroup *BG, CompactPtrT *Array, u32 Size) { - SinglyLinkedList &Batches = BG->Batches; - TransferBatch *CurBatch = Batches.front(); + auto InsertBlocks = [&](BatchGroupT *BG, CompactPtrT *Array, u32 Size) { + SinglyLinkedList &Batches = BG->Batches; + TransferBatchT *CurBatch = Batches.front(); DCHECK_NE(CurBatch, nullptr); for (u32 I = 0; I < Size;) { @@ -613,9 +649,8 @@ private: u16 UnusedSlots = static_cast(BG->MaxCachedPerBatch - CurBatch->getCount()); if (UnusedSlots == 0) { - CurBatch = C->createBatch( - ClassId, - reinterpret_cast(decompactPtr(ClassId, Array[I]))); + CurBatch = + reinterpret_cast(C->getBatchClassBlock()); CurBatch->clear(); Batches.push_front(CurBatch); UnusedSlots = BG->MaxCachedPerBatch; @@ -630,11 +665,11 @@ private: }; Sci->FreeListInfo.PushedBlocks += Size; - BatchGroup *Cur = Sci->FreeListInfo.BlockList.front(); + BatchGroupT *Cur = Sci->FreeListInfo.BlockList.front(); // In the following, `Cur` always points to the BatchGroup for blocks that // will be pushed next. `Prev` is the element right before `Cur`. - BatchGroup *Prev = nullptr; + BatchGroupT *Prev = nullptr; while (Cur != nullptr && compactPtrGroupBase(Array[0]) > Cur->CompactPtrGroupBase) { @@ -695,22 +730,22 @@ private: // group id will be considered first. // // The region mutex needs to be held while calling this method. - TransferBatch *popBatchImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci) + TransferBatchT *popBatchImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci) REQUIRES(Sci->Mutex) { if (Sci->FreeListInfo.BlockList.empty()) return nullptr; - SinglyLinkedList &Batches = + SinglyLinkedList &Batches = Sci->FreeListInfo.BlockList.front()->Batches; if (Batches.empty()) { DCHECK_EQ(ClassId, SizeClassMap::BatchClassId); - BatchGroup *BG = Sci->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Sci->FreeListInfo.BlockList.front(); Sci->FreeListInfo.BlockList.pop_front(); // Block used by `BatchGroup` is from BatchClassId. Turn the block into // `TransferBatch` with single block. - TransferBatch *TB = reinterpret_cast(BG); + TransferBatchT *TB = reinterpret_cast(BG); TB->clear(); TB->add( compactPtr(SizeClassMap::BatchClassId, reinterpret_cast(TB))); @@ -718,13 +753,13 @@ private: return TB; } - TransferBatch *B = Batches.front(); + TransferBatchT *B = Batches.front(); Batches.pop_front(); DCHECK_NE(B, nullptr); DCHECK_GT(B->getCount(), 0U); if (Batches.empty()) { - BatchGroup *BG = Sci->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Sci->FreeListInfo.BlockList.front(); Sci->FreeListInfo.BlockList.pop_front(); // We don't keep BatchGroup with zero blocks to avoid empty-checking while @@ -763,7 +798,7 @@ private: } const uptr Size = getSizeByClassId(ClassId); - const u16 MaxCount = TransferBatch::getMaxCached(Size); + const u16 MaxCount = CacheT::getMaxCached(Size); DCHECK_GT(MaxCount, 0U); // The maximum number of blocks we should carve in the region is dictated // by the maximum number of batches we want to fill, and the amount of @@ -776,7 +811,7 @@ private: DCHECK_GT(NumberOfBlocks, 0U); constexpr u32 ShuffleArraySize = - MaxNumBatches * TransferBatch::MaxNumCached; + MaxNumBatches * TransferBatchT::MaxNumCached; // Fill the transfer batches and put them in the size-class freelist. We // need to randomize the blocks for security purposes, so we first fill a // local array that we then shuffle before populating the batches. @@ -856,11 +891,60 @@ private: PushedBytesDelta >> 10); } + void getSizeClassFragmentationInfo(SizeClassInfo *Sci, uptr ClassId, + ScopedString *Str) REQUIRES(Sci->Mutex) { + const uptr BlockSize = getSizeByClassId(ClassId); + const uptr First = Sci->MinRegionIndex; + const uptr Last = Sci->MaxRegionIndex; + const uptr Base = First * RegionSize; + const uptr NumberOfRegions = Last - First + 1U; + auto SkipRegion = [this, First, ClassId](uptr RegionIndex) { + ScopedLock L(ByteMapMutex); + return (PossibleRegions[First + RegionIndex] - 1U) != ClassId; + }; + + FragmentationRecorder Recorder; + if (!Sci->FreeListInfo.BlockList.empty()) { + PageReleaseContext Context = + markFreeBlocks(Sci, ClassId, BlockSize, Base, NumberOfRegions, + ReleaseToOS::ForceAll); + releaseFreeMemoryToOS(Context, Recorder, SkipRegion); + } + + const uptr PageSize = getPageSizeCached(); + const uptr TotalBlocks = Sci->AllocatedUser / BlockSize; + const uptr InUseBlocks = + Sci->FreeListInfo.PoppedBlocks - Sci->FreeListInfo.PushedBlocks; + uptr AllocatedPagesCount = 0; + if (TotalBlocks != 0U) { + for (uptr I = 0; I < NumberOfRegions; ++I) { + if (SkipRegion(I)) + continue; + AllocatedPagesCount += RegionSize / PageSize; + } + + DCHECK_NE(AllocatedPagesCount, 0U); + } + + DCHECK_GE(AllocatedPagesCount, Recorder.getReleasedPagesCount()); + const uptr InUsePages = + AllocatedPagesCount - Recorder.getReleasedPagesCount(); + const uptr InUseBytes = InUsePages * PageSize; + + uptr Integral; + uptr Fractional; + computePercentage(BlockSize * InUseBlocks, InUsePages * PageSize, &Integral, + &Fractional); + Str->append(" %02zu (%6zu): inuse/total blocks: %6zu/%6zu inuse/total " + "pages: %6zu/%6zu inuse bytes: %6zuK util: %3zu.%02zu%%\n", + ClassId, BlockSize, InUseBlocks, TotalBlocks, InUsePages, + AllocatedPagesCount, InUseBytes >> 10, Integral, Fractional); + } + NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId, ReleaseToOS ReleaseType = ReleaseToOS::Normal) REQUIRES(Sci->Mutex) { const uptr BlockSize = getSizeByClassId(ClassId); - const uptr PageSize = getPageSizeCached(); DCHECK_GE(Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks); const uptr BytesInFreeList = @@ -871,6 +955,60 @@ private: if (UNLIKELY(BytesInFreeList == 0)) return 0; + // ====================================================================== // + // 1. Check if we have enough free blocks and if it's worth doing a page + // release. + // ====================================================================== // + if (ReleaseType != ReleaseToOS::ForceAll && + !hasChanceToReleasePages(Sci, BlockSize, BytesInFreeList, + ReleaseType)) { + return 0; + } + + const uptr First = Sci->MinRegionIndex; + const uptr Last = Sci->MaxRegionIndex; + DCHECK_NE(Last, 0U); + DCHECK_LE(First, Last); + uptr TotalReleasedBytes = 0; + const uptr Base = First * RegionSize; + const uptr NumberOfRegions = Last - First + 1U; + + // ==================================================================== // + // 2. Mark the free blocks and we can tell which pages are in-use by + // querying `PageReleaseContext`. + // ==================================================================== // + PageReleaseContext Context = markFreeBlocks(Sci, ClassId, BlockSize, Base, + NumberOfRegions, ReleaseType); + if (!Context.hasBlockMarked()) + return 0; + + // ==================================================================== // + // 3. Release the unused physical pages back to the OS. + // ==================================================================== // + ReleaseRecorder Recorder(Base); + auto SkipRegion = [this, First, ClassId](uptr RegionIndex) { + ScopedLock L(ByteMapMutex); + return (PossibleRegions[First + RegionIndex] - 1U) != ClassId; + }; + releaseFreeMemoryToOS(Context, Recorder, SkipRegion); + + if (Recorder.getReleasedRangesCount() > 0) { + Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; + Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); + Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); + TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes; + } + Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast(); + + return TotalReleasedBytes; + } + + bool hasChanceToReleasePages(SizeClassInfo *Sci, uptr BlockSize, + uptr BytesInFreeList, ReleaseToOS ReleaseType) + REQUIRES(Sci->Mutex) { + DCHECK_GE(Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks); + const uptr PageSize = getPageSizeCached(); + if (BytesInFreeList <= Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint) Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; @@ -892,22 +1030,20 @@ private: // (BytesInFreeListAtLastCheckpoint - BytesInFreeList). const uptr PushedBytesDelta = BytesInFreeList - Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint; - if (PushedBytesDelta < PageSize && ReleaseType != ReleaseToOS::ForceAll) - return 0; + if (PushedBytesDelta < PageSize) + return false; - const bool CheckDensity = - isSmallBlock(BlockSize) && ReleaseType != ReleaseToOS::ForceAll; // Releasing smaller blocks is expensive, so we want to make sure that a // significant amount of bytes are free, and that there has been a good // amount of batches pushed to the freelist before attempting to release. - if (CheckDensity && ReleaseType == ReleaseToOS::Normal) + if (isSmallBlock(BlockSize) && ReleaseType == ReleaseToOS::Normal) if (PushedBytesDelta < Sci->AllocatedUser / 16U) - return 0; + return false; if (ReleaseType == ReleaseToOS::Normal) { const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs); if (IntervalMs < 0) - return 0; + return false; // The constant 8 here is selected from profiling some apps and the number // of unreleased pages in the large size classes is around 16 pages or @@ -920,30 +1056,31 @@ private: static_cast(IntervalMs) * 1000000 > getMonotonicTimeFast()) { // Memory was returned recently. - return 0; + return false; } } } // if (ReleaseType == ReleaseToOS::Normal) - const uptr First = Sci->MinRegionIndex; - const uptr Last = Sci->MaxRegionIndex; - DCHECK_NE(Last, 0U); - DCHECK_LE(First, Last); - uptr TotalReleasedBytes = 0; - const uptr Base = First * RegionSize; - const uptr NumberOfRegions = Last - First + 1U; - const uptr GroupSize = (1U << GroupSizeLog); + return true; + } + + PageReleaseContext markFreeBlocks(SizeClassInfo *Sci, const uptr ClassId, + const uptr BlockSize, const uptr Base, + const uptr NumberOfRegions, + ReleaseToOS ReleaseType) + REQUIRES(Sci->Mutex) { + const uptr PageSize = getPageSizeCached(); + const uptr GroupSize = (1UL << GroupSizeLog); const uptr CurGroupBase = compactPtrGroupBase(compactPtr(ClassId, Sci->CurrentRegion)); - ReleaseRecorder Recorder(Base); PageReleaseContext Context(BlockSize, NumberOfRegions, /*ReleaseSize=*/RegionSize); auto DecompactPtr = [](CompactPtrT CompactPtr) { return reinterpret_cast(CompactPtr); }; - for (BatchGroup &BG : Sci->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Sci->FreeListInfo.BlockList) { const uptr GroupBase = decompactGroupBase(BG.CompactPtrGroupBase); // The `GroupSize` may not be divided by `BlockSize`, which means there is // an unused space at the end of Region. Exclude that space to avoid @@ -960,25 +1097,27 @@ private: BG.Batches.front()->getCount(); const uptr BytesInBG = NumBlocks * BlockSize; - if (ReleaseType != ReleaseToOS::ForceAll && - BytesInBG <= BG.BytesInBGAtLastCheckpoint) { - BG.BytesInBGAtLastCheckpoint = BytesInBG; - continue; - } - const uptr PushedBytesDelta = BytesInBG - BG.BytesInBGAtLastCheckpoint; - if (ReleaseType != ReleaseToOS::ForceAll && PushedBytesDelta < PageSize) - continue; + if (ReleaseType != ReleaseToOS::ForceAll) { + if (BytesInBG <= BG.BytesInBGAtLastCheckpoint) { + BG.BytesInBGAtLastCheckpoint = BytesInBG; + continue; + } - // Given the randomness property, we try to release the pages only if the - // bytes used by free blocks exceed certain proportion of allocated - // spaces. - if (CheckDensity && (BytesInBG * 100U) / AllocatedGroupSize < - (100U - 1U - BlockSize / 16U)) { - continue; + const uptr PushedBytesDelta = BytesInBG - BG.BytesInBGAtLastCheckpoint; + if (PushedBytesDelta < PageSize) + continue; + + // Given the randomness property, we try to release the pages only if + // the bytes used by free blocks exceed certain proportion of allocated + // spaces. + if (isSmallBlock(BlockSize) && (BytesInBG * 100U) / AllocatedGroupSize < + (100U - 1U - BlockSize / 16U)) { + continue; + } } // TODO: Consider updating this after page release if `ReleaseRecorder` - // can tell the releasd bytes in each group. + // can tell the released bytes in each group. BG.BytesInBGAtLastCheckpoint = BytesInBG; const uptr MaxContainedBlocks = AllocatedGroupSize / BlockSize; @@ -1006,27 +1145,10 @@ private: // We may not be able to do the page release In a rare case that we may // fail on PageMap allocation. if (UNLIKELY(!Context.hasBlockMarked())) - return 0; + break; } - if (!Context.hasBlockMarked()) - return 0; - - auto SkipRegion = [this, First, ClassId](uptr RegionIndex) { - ScopedLock L(ByteMapMutex); - return (PossibleRegions[First + RegionIndex] - 1U) != ClassId; - }; - releaseFreeMemoryToOS(Context, Recorder, SkipRegion); - - if (Recorder.getReleasedRangesCount() > 0) { - Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; - Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); - Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); - TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes; - } - Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast(); - - return TotalReleasedBytes; + return Context; } SizeClassInfo SizeClassInfoArray[NumClasses] = {}; diff --git a/Telegram/ThirdParty/scudo/primary64.h b/Telegram/ThirdParty/scudo/primary64.h index fd7a1f9e8..9a642d236 100644 --- a/Telegram/ThirdParty/scudo/primary64.h +++ b/Telegram/ThirdParty/scudo/primary64.h @@ -9,6 +9,7 @@ #ifndef SCUDO_PRIMARY64_H_ #define SCUDO_PRIMARY64_H_ +#include "allocator_common.h" #include "bytemap.h" #include "common.h" #include "list.h" @@ -21,6 +22,8 @@ #include "string_utils.h" #include "thread_annotations.h" +#include "condition_variable.h" + namespace scudo { // SizeClassAllocator64 is an allocator tuned for 64-bit address space. @@ -47,27 +50,39 @@ template class SizeClassAllocator64 { public: typedef typename Config::Primary::CompactPtrT CompactPtrT; typedef typename Config::Primary::SizeClassMap SizeClassMap; + typedef typename ConditionVariableState< + typename Config::Primary>::ConditionVariableT ConditionVariableT; static const uptr CompactPtrScale = Config::Primary::CompactPtrScale; + static const uptr RegionSizeLog = Config::Primary::RegionSizeLog; static const uptr GroupSizeLog = Config::Primary::GroupSizeLog; + static_assert(RegionSizeLog >= GroupSizeLog, + "Group size shouldn't be greater than the region size"); static const uptr GroupScale = GroupSizeLog - CompactPtrScale; typedef SizeClassAllocator64 ThisT; typedef SizeClassAllocatorLocalCache CacheT; - typedef typename CacheT::TransferBatch TransferBatch; - typedef typename CacheT::BatchGroup BatchGroup; + typedef TransferBatch TransferBatchT; + typedef BatchGroup BatchGroupT; + + static_assert(sizeof(BatchGroupT) <= sizeof(TransferBatchT), + "BatchGroupT uses the same class size as TransferBatchT"); static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) - ? roundUp(sizeof(TransferBatch), 1U << CompactPtrScale) + ? roundUp(sizeof(TransferBatchT), 1U << CompactPtrScale) : SizeClassMap::getSizeByClassId(ClassId); } static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; } + static bool conditionVariableEnabled() { + return ConditionVariableState::enabled(); + } + void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS { DCHECK(isAligned(reinterpret_cast(this), alignof(ThisT))); const uptr PageSize = getPageSizeCached(); - const uptr GroupSize = (1U << GroupSizeLog); + const uptr GroupSize = (1UL << GroupSizeLog); const uptr PagesInGroup = GroupSize / PageSize; const uptr MinSizeClass = getSizeByClassId(1); // When trying to release pages back to memory, visiting smaller size @@ -117,13 +132,13 @@ public: for (uptr I = 0; I < NumClasses; I++) { RegionInfo *Region = getRegionInfo(I); + // The actual start of a region is offset by a random number of pages // when PrimaryEnableRandomOffset is set. - Region->RegionBeg = - (PrimaryBase + (I << Config::Primary::RegionSizeLog)) + - (Config::Primary::EnableRandomOffset - ? ((getRandomModN(&Seed, 16) + 1) * PageSize) - : 0); + Region->RegionBeg = (PrimaryBase + (I << RegionSizeLog)) + + (Config::Primary::EnableRandomOffset + ? ((getRandomModN(&Seed, 16) + 1) * PageSize) + : 0); Region->RandState = getRandomU32(&Seed); // Releasing small blocks is expensive, set a higher threshold to avoid // frequent page releases. @@ -134,11 +149,16 @@ public: Region->ReleaseInfo.LastReleaseAtNs = Time; Region->MemMapInfo.MemMap = ReservedMemory.dispatch( - PrimaryBase + (I << Config::Primary::RegionSizeLog), RegionSize); + PrimaryBase + (I << RegionSizeLog), RegionSize); CHECK(Region->MemMapInfo.MemMap.isAllocated()); } shuffle(RegionInfoArray, NumClasses, &Seed); + // The binding should be done after region shuffling so that it won't bind + // the FLLock from the wrong region. + for (uptr I = 0; I < NumClasses; I++) + getRegionInfo(I)->FLLockCV.bindTestOnly(getRegionInfo(I)->FLLock); + setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); } @@ -165,7 +185,7 @@ public: ScopedLock FL(Region->FLLock); const uptr BlockSize = getSizeByClassId(I); uptr TotalBlocks = 0; - for (BatchGroup &BG : Region->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Region->FreeListInfo.BlockList) { // `BG::Batches` are `TransferBatches`. +1 for `BatchGroup`. BatchClassUsedInFreeLists += BG.Batches.size() + 1; for (const auto &It : BG.Batches) @@ -182,7 +202,7 @@ public: ScopedLock FL(Region->FLLock); const uptr BlockSize = getSizeByClassId(SizeClassMap::BatchClassId); uptr TotalBlocks = 0; - for (BatchGroup &BG : Region->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Region->FreeListInfo.BlockList) { if (LIKELY(!BG.Batches.empty())) { for (const auto &It : BG.Batches) TotalBlocks += It.getCount(); @@ -201,51 +221,64 @@ public: DCHECK_EQ(BlocksInUse, BatchClassUsedInFreeLists); } - TransferBatch *popBatch(CacheT *C, uptr ClassId) { + // Note that the `MaxBlockCount` will be used when we support arbitrary blocks + // count. Now it's the same as the number of blocks stored in the + // `TransferBatch`. + u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray, + UNUSED const u16 MaxBlockCount) { + TransferBatchT *B = popBatch(C, ClassId); + if (!B) + return 0; + + const u16 Count = B->getCount(); + DCHECK_GT(Count, 0U); + B->moveToArray(ToArray); + + if (ClassId != SizeClassMap::BatchClassId) + C->deallocate(SizeClassMap::BatchClassId, B); + + return Count; + } + + TransferBatchT *popBatch(CacheT *C, uptr ClassId) { DCHECK_LT(ClassId, NumClasses); RegionInfo *Region = getRegionInfo(ClassId); { ScopedLock L(Region->FLLock); - TransferBatch *B = popBatchImpl(C, ClassId, Region); + TransferBatchT *B = popBatchImpl(C, ClassId, Region); if (LIKELY(B)) return B; } - bool PrintStats = false; - TransferBatch *B = nullptr; + bool ReportRegionExhausted = false; + TransferBatchT *B = nullptr; - while (true) { - // When two threads compete for `Region->MMLock`, we only want one of them - // does the populateFreeListAndPopBatch(). To avoid both of them doing - // that, always check the freelist before mapping new pages. - // - // TODO(chiahungduan): Use a condition variable so that we don't need to - // hold `Region->MMLock` here. - ScopedLock ML(Region->MMLock); - { - ScopedLock FL(Region->FLLock); - B = popBatchImpl(C, ClassId, Region); - if (LIKELY(B)) - return B; + if (conditionVariableEnabled()) { + B = popBatchWithCV(C, ClassId, Region, ReportRegionExhausted); + } else { + while (true) { + // When two threads compete for `Region->MMLock`, we only want one of + // them to call populateFreeListAndPopBatch(). To avoid both of them + // doing that, always check the freelist before mapping new pages. + ScopedLock ML(Region->MMLock); + { + ScopedLock FL(Region->FLLock); + if ((B = popBatchImpl(C, ClassId, Region))) + break; + } + + const bool RegionIsExhausted = Region->Exhausted; + if (!RegionIsExhausted) + B = populateFreeListAndPopBatch(C, ClassId, Region); + ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted; + break; } - - const bool RegionIsExhausted = Region->Exhausted; - if (!RegionIsExhausted) - B = populateFreeListAndPopBatch(C, ClassId, Region); - PrintStats = !RegionIsExhausted && Region->Exhausted; - break; } - // Note that `getStats()` requires locking each region so we can't call it - // while locking the Region->Mutex in the above. - if (UNLIKELY(PrintStats)) { - ScopedString Str; - getStats(&Str); - Str.append( - "Scudo OOM: The process has exhausted %zuM for size class %zu.\n", - RegionSize >> 20, getSizeByClassId(ClassId)); - Str.output(); + if (UNLIKELY(ReportRegionExhausted)) { + Printf("Can't populate more pages for size class %zu.\n", + getSizeByClassId(ClassId)); // Theoretically, BatchClass shouldn't be used up. Abort immediately when // it happens. @@ -265,30 +298,36 @@ public: if (ClassId == SizeClassMap::BatchClassId) { ScopedLock L(Region->FLLock); pushBatchClassBlocks(Region, Array, Size); + if (conditionVariableEnabled()) + Region->FLLockCV.notifyAll(Region->FLLock); return; } // TODO(chiahungduan): Consider not doing grouping if the group size is not // greater than the block size with a certain scale. - // Sort the blocks so that blocks belonging to the same group can be pushed - // together. bool SameGroup = true; - for (u32 I = 1; I < Size; ++I) { - if (compactPtrGroup(Array[I - 1]) != compactPtrGroup(Array[I])) - SameGroup = false; - CompactPtrT Cur = Array[I]; - u32 J = I; - while (J > 0 && compactPtrGroup(Cur) < compactPtrGroup(Array[J - 1])) { - Array[J] = Array[J - 1]; - --J; + if (GroupSizeLog < RegionSizeLog) { + // Sort the blocks so that blocks belonging to the same group can be + // pushed together. + for (u32 I = 1; I < Size; ++I) { + if (compactPtrGroup(Array[I - 1]) != compactPtrGroup(Array[I])) + SameGroup = false; + CompactPtrT Cur = Array[I]; + u32 J = I; + while (J > 0 && compactPtrGroup(Cur) < compactPtrGroup(Array[J - 1])) { + Array[J] = Array[J - 1]; + --J; + } + Array[J] = Cur; } - Array[J] = Cur; } { ScopedLock L(Region->FLLock); pushBlocksImpl(C, ClassId, Region, Array, Size, SameGroup); + if (conditionVariableEnabled()) + Region->FLLockCV.notifyAll(Region->FLLock); } } @@ -363,6 +402,18 @@ public: } } + void getFragmentationInfo(ScopedString *Str) { + Str->append( + "Fragmentation Stats: SizeClassAllocator64: page size = %zu bytes\n", + getPageSizeCached()); + + for (uptr I = 1; I < NumClasses; I++) { + RegionInfo *Region = getRegionInfo(I); + ScopedLock L(Region->MMLock); + getRegionFragmentationInfo(Region, I, Str); + } + } + bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { const s32 Interval = Max(Min(static_cast(Value), @@ -477,7 +528,7 @@ public: AtomicOptions Options; private: - static const uptr RegionSize = 1UL << Config::Primary::RegionSizeLog; + static const uptr RegionSize = 1UL << RegionSizeLog; static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr PrimarySize = RegionSize * NumClasses; @@ -493,7 +544,7 @@ private: }; struct BlocksInfo { - SinglyLinkedList BlockList = {}; + SinglyLinkedList BlockList = {}; uptr PoppedBlocks = 0; uptr PushedBlocks = 0; }; @@ -509,6 +560,7 @@ private: struct UnpaddedRegionInfo { // Mutex for operations on freelist HybridMutex FLLock; + ConditionVariableT FLLockCV GUARDED_BY(FLLock); // Mutex for memmap operations HybridMutex MMLock ACQUIRED_BEFORE(FLLock); // `RegionBeg` is initialized before thread creation and won't be changed. @@ -520,6 +572,7 @@ private: uptr TryReleaseThreshold GUARDED_BY(MMLock) = 0; ReleaseToOsInfo ReleaseInfo GUARDED_BY(MMLock) = {}; bool Exhausted GUARDED_BY(MMLock) = false; + bool isPopulatingFreeList GUARDED_BY(FLLock) = false; }; struct RegionInfo : UnpaddedRegionInfo { char Padding[SCUDO_CACHE_LINE_SIZE - @@ -605,11 +658,11 @@ private: // reusable and don't need additional space for them. Region->FreeListInfo.PushedBlocks += Size; - BatchGroup *BG = Region->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Region->FreeListInfo.BlockList.front(); if (BG == nullptr) { // Construct `BatchGroup` on the last element. - BG = reinterpret_cast( + BG = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1])); --Size; BG->Batches.clear(); @@ -620,8 +673,8 @@ private: // from `CreateGroup` in `pushBlocksImpl` BG->PushedBlocks = 1; BG->BytesInBGAtLastCheckpoint = 0; - BG->MaxCachedPerBatch = TransferBatch::getMaxCached( - getSizeByClassId(SizeClassMap::BatchClassId)); + BG->MaxCachedPerBatch = + CacheT::getMaxCached(getSizeByClassId(SizeClassMap::BatchClassId)); Region->FreeListInfo.BlockList.push_front(BG); } @@ -634,7 +687,7 @@ private: // 2. Only 1 block is pushed when the freelist is empty. if (BG->Batches.empty()) { // Construct the `TransferBatch` on the last element. - TransferBatch *TB = reinterpret_cast( + TransferBatchT *TB = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1])); TB->clear(); // As mentioned above, addresses of `TransferBatch` and `BatchGroup` are @@ -649,14 +702,14 @@ private: BG->Batches.push_front(TB); } - TransferBatch *CurBatch = BG->Batches.front(); + TransferBatchT *CurBatch = BG->Batches.front(); DCHECK_NE(CurBatch, nullptr); for (u32 I = 0; I < Size;) { u16 UnusedSlots = static_cast(BG->MaxCachedPerBatch - CurBatch->getCount()); if (UnusedSlots == 0) { - CurBatch = reinterpret_cast( + CurBatch = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[I])); CurBatch->clear(); // Self-contained @@ -699,24 +752,25 @@ private: DCHECK_GT(Size, 0U); auto CreateGroup = [&](uptr CompactPtrGroupBase) { - BatchGroup *BG = C->createGroup(); + BatchGroupT *BG = + reinterpret_cast(C->getBatchClassBlock()); BG->Batches.clear(); - TransferBatch *TB = C->createBatch(ClassId, nullptr); + TransferBatchT *TB = + reinterpret_cast(C->getBatchClassBlock()); TB->clear(); BG->CompactPtrGroupBase = CompactPtrGroupBase; BG->Batches.push_front(TB); BG->PushedBlocks = 0; BG->BytesInBGAtLastCheckpoint = 0; - BG->MaxCachedPerBatch = - TransferBatch::getMaxCached(getSizeByClassId(ClassId)); + BG->MaxCachedPerBatch = CacheT::getMaxCached(getSizeByClassId(ClassId)); return BG; }; - auto InsertBlocks = [&](BatchGroup *BG, CompactPtrT *Array, u32 Size) { - SinglyLinkedList &Batches = BG->Batches; - TransferBatch *CurBatch = Batches.front(); + auto InsertBlocks = [&](BatchGroupT *BG, CompactPtrT *Array, u32 Size) { + SinglyLinkedList &Batches = BG->Batches; + TransferBatchT *CurBatch = Batches.front(); DCHECK_NE(CurBatch, nullptr); for (u32 I = 0; I < Size;) { @@ -724,9 +778,8 @@ private: u16 UnusedSlots = static_cast(BG->MaxCachedPerBatch - CurBatch->getCount()); if (UnusedSlots == 0) { - CurBatch = C->createBatch( - ClassId, - reinterpret_cast(decompactPtr(ClassId, Array[I]))); + CurBatch = + reinterpret_cast(C->getBatchClassBlock()); CurBatch->clear(); Batches.push_front(CurBatch); UnusedSlots = BG->MaxCachedPerBatch; @@ -741,21 +794,11 @@ private: }; Region->FreeListInfo.PushedBlocks += Size; - BatchGroup *Cur = Region->FreeListInfo.BlockList.front(); - - if (ClassId == SizeClassMap::BatchClassId) { - if (Cur == nullptr) { - // Don't need to classify BatchClassId. - Cur = CreateGroup(/*CompactPtrGroupBase=*/0); - Region->FreeListInfo.BlockList.push_front(Cur); - } - InsertBlocks(Cur, Array, Size); - return; - } + BatchGroupT *Cur = Region->FreeListInfo.BlockList.front(); // In the following, `Cur` always points to the BatchGroup for blocks that // will be pushed next. `Prev` is the element right before `Cur`. - BatchGroup *Prev = nullptr; + BatchGroupT *Prev = nullptr; while (Cur != nullptr && compactPtrGroup(Array[0]) > Cur->CompactPtrGroupBase) { @@ -812,26 +855,96 @@ private: InsertBlocks(Cur, Array + Size - Count, Count); } + TransferBatchT *popBatchWithCV(CacheT *C, uptr ClassId, RegionInfo *Region, + bool &ReportRegionExhausted) { + TransferBatchT *B = nullptr; + + while (true) { + // We only expect one thread doing the freelist refillment and other + // threads will be waiting for either the completion of the + // `populateFreeListAndPopBatch()` or `pushBlocks()` called by other + // threads. + bool PopulateFreeList = false; + { + ScopedLock FL(Region->FLLock); + if (!Region->isPopulatingFreeList) { + Region->isPopulatingFreeList = true; + PopulateFreeList = true; + } + } + + if (PopulateFreeList) { + ScopedLock ML(Region->MMLock); + + const bool RegionIsExhausted = Region->Exhausted; + if (!RegionIsExhausted) + B = populateFreeListAndPopBatch(C, ClassId, Region); + ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted; + + { + // Before reacquiring the `FLLock`, the freelist may be used up again + // and some threads are waiting for the freelist refillment by the + // current thread. It's important to set + // `Region->isPopulatingFreeList` to false so the threads about to + // sleep will notice the status change. + ScopedLock FL(Region->FLLock); + Region->isPopulatingFreeList = false; + Region->FLLockCV.notifyAll(Region->FLLock); + } + + break; + } + + // At here, there are two preconditions to be met before waiting, + // 1. The freelist is empty. + // 2. Region->isPopulatingFreeList == true, i.e, someone is still doing + // `populateFreeListAndPopBatch()`. + // + // Note that it has the chance that freelist is empty but + // Region->isPopulatingFreeList == false because all the new populated + // blocks were used up right after the refillment. Therefore, we have to + // check if someone is still populating the freelist. + ScopedLock FL(Region->FLLock); + if (LIKELY(B = popBatchImpl(C, ClassId, Region))) + break; + + if (!Region->isPopulatingFreeList) + continue; + + // Now the freelist is empty and someone's doing the refillment. We will + // wait until anyone refills the freelist or someone finishes doing + // `populateFreeListAndPopBatch()`. The refillment can be done by + // `populateFreeListAndPopBatch()`, `pushBlocks()`, + // `pushBatchClassBlocks()` and `mergeGroupsToReleaseBack()`. + Region->FLLockCV.wait(Region->FLLock); + + if (LIKELY(B = popBatchImpl(C, ClassId, Region))) + break; + } + + return B; + } + // Pop one TransferBatch from a BatchGroup. The BatchGroup with the smallest // group id will be considered first. // // The region mutex needs to be held while calling this method. - TransferBatch *popBatchImpl(CacheT *C, uptr ClassId, RegionInfo *Region) + TransferBatchT *popBatchImpl(CacheT *C, uptr ClassId, RegionInfo *Region) REQUIRES(Region->FLLock) { if (Region->FreeListInfo.BlockList.empty()) return nullptr; - SinglyLinkedList &Batches = + SinglyLinkedList &Batches = Region->FreeListInfo.BlockList.front()->Batches; if (Batches.empty()) { DCHECK_EQ(ClassId, SizeClassMap::BatchClassId); - BatchGroup *BG = Region->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Region->FreeListInfo.BlockList.front(); Region->FreeListInfo.BlockList.pop_front(); // Block used by `BatchGroup` is from BatchClassId. Turn the block into // `TransferBatch` with single block. - TransferBatch *TB = reinterpret_cast(BG); + TransferBatchT *TB = reinterpret_cast(BG); TB->clear(); TB->add( compactPtr(SizeClassMap::BatchClassId, reinterpret_cast(TB))); @@ -839,13 +952,13 @@ private: return TB; } - TransferBatch *B = Batches.front(); + TransferBatchT *B = Batches.front(); Batches.pop_front(); DCHECK_NE(B, nullptr); DCHECK_GT(B->getCount(), 0U); if (Batches.empty()) { - BatchGroup *BG = Region->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Region->FreeListInfo.BlockList.front(); Region->FreeListInfo.BlockList.pop_front(); // We don't keep BatchGroup with zero blocks to avoid empty-checking while @@ -863,11 +976,11 @@ private: } // Refill the freelist and return one batch. - NOINLINE TransferBatch *populateFreeListAndPopBatch(CacheT *C, uptr ClassId, - RegionInfo *Region) + NOINLINE TransferBatchT *populateFreeListAndPopBatch(CacheT *C, uptr ClassId, + RegionInfo *Region) REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { const uptr Size = getSizeByClassId(ClassId); - const u16 MaxCount = TransferBatch::getMaxCached(Size); + const u16 MaxCount = CacheT::getMaxCached(Size); const uptr RegionBeg = Region->RegionBeg; const uptr MappedUser = Region->MemMapInfo.MappedUser; @@ -903,7 +1016,7 @@ private: DCHECK_GT(NumberOfBlocks, 0); constexpr u32 ShuffleArraySize = - MaxNumBatches * TransferBatch::MaxNumCached; + MaxNumBatches * TransferBatchT::MaxNumCached; CompactPtrT ShuffleArray[ShuffleArraySize]; DCHECK_LE(NumberOfBlocks, ShuffleArraySize); @@ -936,7 +1049,7 @@ private: pushBatchClassBlocks(Region, ShuffleArray, NumberOfBlocks); } - TransferBatch *B = popBatchImpl(C, ClassId, Region); + TransferBatchT *B = popBatchImpl(C, ClassId, Region); DCHECK_NE(B, nullptr); // Note that `PushedBlocks` and `PoppedBlocks` are supposed to only record @@ -957,10 +1070,10 @@ private: if (Region->MemMapInfo.MappedUser == 0) return; const uptr BlockSize = getSizeByClassId(ClassId); - const uptr InUse = + const uptr InUseBlocks = Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks; const uptr BytesInFreeList = - Region->MemMapInfo.AllocatedUser - InUse * BlockSize; + Region->MemMapInfo.AllocatedUser - InUseBlocks * BlockSize; uptr RegionPushedBytesDelta = 0; if (BytesInFreeList >= Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint) { @@ -972,122 +1085,145 @@ private: "%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " "inuse: %6zu total: %6zu releases: %6zu last " "released: %6zuK latest pushed bytes: %6zuK region: 0x%zx (0x%zx)\n", - Region->Exhausted ? "F" : " ", ClassId, getSizeByClassId(ClassId), + Region->Exhausted ? "E" : " ", ClassId, getSizeByClassId(ClassId), Region->MemMapInfo.MappedUser >> 10, Region->FreeListInfo.PoppedBlocks, - Region->FreeListInfo.PushedBlocks, InUse, TotalChunks, + Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks, Region->ReleaseInfo.RangesReleased, Region->ReleaseInfo.LastReleasedBytes >> 10, RegionPushedBytesDelta >> 10, Region->RegionBeg, getRegionBaseByClassId(ClassId)); } + void getRegionFragmentationInfo(RegionInfo *Region, uptr ClassId, + ScopedString *Str) REQUIRES(Region->MMLock) { + const uptr BlockSize = getSizeByClassId(ClassId); + const uptr AllocatedUserEnd = + Region->MemMapInfo.AllocatedUser + Region->RegionBeg; + + SinglyLinkedList GroupsToRelease; + { + ScopedLock L(Region->FLLock); + GroupsToRelease = Region->FreeListInfo.BlockList; + Region->FreeListInfo.BlockList.clear(); + } + + FragmentationRecorder Recorder; + if (!GroupsToRelease.empty()) { + PageReleaseContext Context = + markFreeBlocks(Region, BlockSize, AllocatedUserEnd, + getCompactPtrBaseByClassId(ClassId), GroupsToRelease); + auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; }; + releaseFreeMemoryToOS(Context, Recorder, SkipRegion); + + mergeGroupsToReleaseBack(Region, GroupsToRelease); + } + + ScopedLock L(Region->FLLock); + const uptr PageSize = getPageSizeCached(); + const uptr TotalBlocks = Region->MemMapInfo.AllocatedUser / BlockSize; + const uptr InUseBlocks = + Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks; + const uptr AllocatedPagesCount = + roundUp(Region->MemMapInfo.AllocatedUser, PageSize) / PageSize; + DCHECK_GE(AllocatedPagesCount, Recorder.getReleasedPagesCount()); + const uptr InUsePages = + AllocatedPagesCount - Recorder.getReleasedPagesCount(); + const uptr InUseBytes = InUsePages * PageSize; + + uptr Integral; + uptr Fractional; + computePercentage(BlockSize * InUseBlocks, InUsePages * PageSize, &Integral, + &Fractional); + Str->append(" %02zu (%6zu): inuse/total blocks: %6zu/%6zu inuse/total " + "pages: %6zu/%6zu inuse bytes: %6zuK util: %3zu.%02zu%%\n", + ClassId, BlockSize, InUseBlocks, TotalBlocks, InUsePages, + AllocatedPagesCount, InUseBytes >> 10, Integral, Fractional); + } + NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId, ReleaseToOS ReleaseType = ReleaseToOS::Normal) REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { - ScopedLock L(Region->FLLock); - const uptr BlockSize = getSizeByClassId(ClassId); - const uptr BytesInFreeList = - Region->MemMapInfo.AllocatedUser - (Region->FreeListInfo.PoppedBlocks - - Region->FreeListInfo.PushedBlocks) * - BlockSize; - if (UNLIKELY(BytesInFreeList == 0)) - return false; - + uptr BytesInFreeList; const uptr AllocatedUserEnd = Region->MemMapInfo.AllocatedUser + Region->RegionBeg; - const uptr CompactPtrBase = getCompactPtrBaseByClassId(ClassId); + SinglyLinkedList GroupsToRelease; - // ====================================================================== // - // 1. Check if we have enough free blocks and if it's worth doing a page - // release. - // ====================================================================== // - if (ReleaseType != ReleaseToOS::ForceAll && - !hasChanceToReleasePages(Region, BlockSize, BytesInFreeList, - ReleaseType)) { - return 0; - } + { + ScopedLock L(Region->FLLock); - // ====================================================================== // - // 2. Determine which groups can release the pages. Use a heuristic to - // gather groups that are candidates for doing a release. - // ====================================================================== // - SinglyLinkedList GroupsToRelease; - if (ReleaseType == ReleaseToOS::ForceAll) { - GroupsToRelease = Region->FreeListInfo.BlockList; - Region->FreeListInfo.BlockList.clear(); - } else { - GroupsToRelease = collectGroupsToRelease( - Region, BlockSize, AllocatedUserEnd, CompactPtrBase); - } - if (GroupsToRelease.empty()) - return 0; + BytesInFreeList = Region->MemMapInfo.AllocatedUser - + (Region->FreeListInfo.PoppedBlocks - + Region->FreeListInfo.PushedBlocks) * + BlockSize; + if (UNLIKELY(BytesInFreeList == 0)) + return false; - // Ideally, we should use a class like `ScopedUnlock`. However, this form of - // unlocking is not supported by the thread-safety analysis. See - // https://clang.llvm.org/docs/ThreadSafetyAnalysis.html#no-alias-analysis - // for more details. - // Put it as local class so that we can mark the ctor/dtor with proper - // annotations associated to the target lock. Note that accessing the - // function variable in local class only works in thread-safety annotations. - // TODO: Implement general `ScopedUnlock` when it's supported. - class FLLockScopedUnlock { - public: - FLLockScopedUnlock(RegionInfo *Region) RELEASE(Region->FLLock) - : R(Region) { - R->FLLock.assertHeld(); - R->FLLock.unlock(); + // ==================================================================== // + // 1. Check if we have enough free blocks and if it's worth doing a page + // release. + // ==================================================================== // + if (ReleaseType != ReleaseToOS::ForceAll && + !hasChanceToReleasePages(Region, BlockSize, BytesInFreeList, + ReleaseType)) { + return 0; } - ~FLLockScopedUnlock() ACQUIRE(Region->FLLock) { R->FLLock.lock(); } - private: - RegionInfo *R; - }; + // ==================================================================== // + // 2. Determine which groups can release the pages. Use a heuristic to + // gather groups that are candidates for doing a release. + // ==================================================================== // + if (ReleaseType == ReleaseToOS::ForceAll) { + GroupsToRelease = Region->FreeListInfo.BlockList; + Region->FreeListInfo.BlockList.clear(); + } else { + GroupsToRelease = + collectGroupsToRelease(Region, BlockSize, AllocatedUserEnd, + getCompactPtrBaseByClassId(ClassId)); + } + if (GroupsToRelease.empty()) + return 0; + } // Note that we have extracted the `GroupsToRelease` from region freelist. // It's safe to let pushBlocks()/popBatches() access the remaining region // freelist. In the steps 3 and 4, we will temporarily release the FLLock // and lock it again before step 5. - uptr ReleasedBytes = 0; - { - FLLockScopedUnlock UL(Region); - // ==================================================================== // - // 3. Mark the free blocks in `GroupsToRelease` in the - // `PageReleaseContext`. Then we can tell which pages are in-use by - // querying `PageReleaseContext`. - // ==================================================================== // - PageReleaseContext Context = markFreeBlocks( - Region, BlockSize, AllocatedUserEnd, CompactPtrBase, GroupsToRelease); - if (UNLIKELY(!Context.hasBlockMarked())) { - ScopedLock L(Region->FLLock); - mergeGroupsToReleaseBack(Region, GroupsToRelease); - return 0; - } - - // ==================================================================== // - // 4. Release the unused physical pages back to the OS. - // ==================================================================== // - RegionReleaseRecorder Recorder(&Region->MemMapInfo.MemMap, - Region->RegionBeg, - Context.getReleaseOffset()); - auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; }; - releaseFreeMemoryToOS(Context, Recorder, SkipRegion); - if (Recorder.getReleasedRangesCount() > 0) { - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; - Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); - Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); - } - Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast(); - ReleasedBytes = Recorder.getReleasedBytes(); + // ==================================================================== // + // 3. Mark the free blocks in `GroupsToRelease` in the `PageReleaseContext`. + // Then we can tell which pages are in-use by querying + // `PageReleaseContext`. + // ==================================================================== // + PageReleaseContext Context = + markFreeBlocks(Region, BlockSize, AllocatedUserEnd, + getCompactPtrBaseByClassId(ClassId), GroupsToRelease); + if (UNLIKELY(!Context.hasBlockMarked())) { + mergeGroupsToReleaseBack(Region, GroupsToRelease); + return 0; } + // ==================================================================== // + // 4. Release the unused physical pages back to the OS. + // ==================================================================== // + RegionReleaseRecorder Recorder(&Region->MemMapInfo.MemMap, + Region->RegionBeg, + Context.getReleaseOffset()); + auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; }; + releaseFreeMemoryToOS(Context, Recorder, SkipRegion); + if (Recorder.getReleasedRangesCount() > 0) { + Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; + Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); + Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); + } + Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast(); + // ====================================================================== // // 5. Merge the `GroupsToRelease` back to the freelist. // ====================================================================== // mergeGroupsToReleaseBack(Region, GroupsToRelease); - return ReleasedBytes; + return Recorder.getReleasedBytes(); } bool hasChanceToReleasePages(RegionInfo *Region, uptr BlockSize, @@ -1154,13 +1290,13 @@ private: return true; } - SinglyLinkedList + SinglyLinkedList collectGroupsToRelease(RegionInfo *Region, const uptr BlockSize, const uptr AllocatedUserEnd, const uptr CompactPtrBase) REQUIRES(Region->MMLock, Region->FLLock) { - const uptr GroupSize = (1U << GroupSizeLog); + const uptr GroupSize = (1UL << GroupSizeLog); const uptr PageSize = getPageSizeCached(); - SinglyLinkedList GroupsToRelease; + SinglyLinkedList GroupsToRelease; // We are examining each group and will take the minimum distance to the // release threshold as the next Region::TryReleaseThreshold(). Note that if @@ -1169,8 +1305,8 @@ private: // the comment on `SmallerBlockReleasePageDelta` for more details. uptr MinDistToThreshold = GroupSize; - for (BatchGroup *BG = Region->FreeListInfo.BlockList.front(), - *Prev = nullptr; + for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(), + *Prev = nullptr; BG != nullptr;) { // Group boundary is always GroupSize-aligned from CompactPtr base. The // layout of memory groups is like, @@ -1254,7 +1390,7 @@ private: } } - // If `BG` is the first BatchGroup in the list, we only need to advance + // If `BG` is the first BatchGroupT in the list, we only need to advance // `BG` and call FreeListInfo.BlockList::pop_front(). No update is needed // for `Prev`. // @@ -1290,7 +1426,7 @@ private: // Note that we need to advance before pushing this BatchGroup to // GroupsToRelease because it's a destructive operation. - BatchGroup *Cur = BG; + BatchGroupT *Cur = BG; BG = BG->Next; // Ideally, we may want to update this only after successful release. @@ -1323,9 +1459,9 @@ private: PageReleaseContext markFreeBlocks(RegionInfo *Region, const uptr BlockSize, const uptr AllocatedUserEnd, const uptr CompactPtrBase, - SinglyLinkedList &GroupsToRelease) + SinglyLinkedList &GroupsToRelease) REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { - const uptr GroupSize = (1U << GroupSizeLog); + const uptr GroupSize = (1UL << GroupSizeLog); auto DecompactPtr = [CompactPtrBase](CompactPtrT CompactPtr) { return decompactPtrInternal(CompactPtrBase, CompactPtr); }; @@ -1352,7 +1488,7 @@ private: if (UNLIKELY(!Context.ensurePageMapAllocated())) return Context; - for (BatchGroup &BG : GroupsToRelease) { + for (BatchGroupT &BG : GroupsToRelease) { const uptr BatchGroupBase = decompactGroupBase(CompactPtrBase, BG.CompactPtrGroupBase); const uptr BatchGroupEnd = BatchGroupBase + GroupSize; @@ -1400,8 +1536,10 @@ private: } void mergeGroupsToReleaseBack(RegionInfo *Region, - SinglyLinkedList &GroupsToRelease) - REQUIRES(Region->MMLock, Region->FLLock) { + SinglyLinkedList &GroupsToRelease) + REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { + ScopedLock L(Region->FLLock); + // After merging two freelists, we may have redundant `BatchGroup`s that // need to be recycled. The number of unused `BatchGroup`s is expected to be // small. Pick a constant which is inferred from real programs. @@ -1419,8 +1557,8 @@ private: // Merge GroupsToRelease back to the Region::FreeListInfo.BlockList. Note // that both `Region->FreeListInfo.BlockList` and `GroupsToRelease` are // sorted. - for (BatchGroup *BG = Region->FreeListInfo.BlockList.front(), - *Prev = nullptr; + for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(), + *Prev = nullptr; ;) { if (BG == nullptr || GroupsToRelease.empty()) { if (!GroupsToRelease.empty()) @@ -1437,8 +1575,8 @@ private: continue; } - BatchGroup *Cur = GroupsToRelease.front(); - TransferBatch *UnusedTransferBatch = nullptr; + BatchGroupT *Cur = GroupsToRelease.front(); + TransferBatchT *UnusedTransferBatch = nullptr; GroupsToRelease.pop_front(); if (BG->CompactPtrGroupBase == Cur->CompactPtrGroupBase) { @@ -1454,7 +1592,7 @@ private: if (Cur->Batches.front()->getCount() == MaxCachedPerBatch) { BG->Batches.append_back(&Cur->Batches); } else { - TransferBatch *NonFullBatch = Cur->Batches.front(); + TransferBatchT *NonFullBatch = Cur->Batches.front(); Cur->Batches.pop_front(); const u16 NonFullBatchCount = NonFullBatch->getCount(); // The remaining Batches in `Cur` are full. @@ -1481,6 +1619,8 @@ private: if (UNLIKELY(Idx + NeededSlots > MaxUnusedSize)) { ScopedLock L(BatchClassRegion->FLLock); pushBatchClassBlocks(BatchClassRegion, Blocks, Idx); + if (conditionVariableEnabled()) + BatchClassRegion->FLLockCV.notifyAll(BatchClassRegion->FLLock); Idx = 0; } Blocks[Idx++] = @@ -1516,15 +1656,20 @@ private: if (Idx != 0) { ScopedLock L(BatchClassRegion->FLLock); pushBatchClassBlocks(BatchClassRegion, Blocks, Idx); + if (conditionVariableEnabled()) + BatchClassRegion->FLLockCV.notifyAll(BatchClassRegion->FLLock); } if (SCUDO_DEBUG) { - BatchGroup *Prev = Region->FreeListInfo.BlockList.front(); - for (BatchGroup *Cur = Prev->Next; Cur != nullptr; + BatchGroupT *Prev = Region->FreeListInfo.BlockList.front(); + for (BatchGroupT *Cur = Prev->Next; Cur != nullptr; Prev = Cur, Cur = Cur->Next) { CHECK_LT(Prev->CompactPtrGroupBase, Cur->CompactPtrGroupBase); } } + + if (conditionVariableEnabled()) + Region->FLLockCV.notifyAll(Region->FLLock); } // TODO: `PrimaryBase` can be obtained from ReservedMemory. This needs to be diff --git a/Telegram/ThirdParty/scudo/release.cpp b/Telegram/ThirdParty/scudo/release.cpp index 938bb41fa..875a2b0c1 100644 --- a/Telegram/ThirdParty/scudo/release.cpp +++ b/Telegram/ThirdParty/scudo/release.cpp @@ -10,7 +10,8 @@ namespace scudo { -BufferPool +BufferPool RegionPageMap::Buffers; } // namespace scudo diff --git a/Telegram/ThirdParty/scudo/release.h b/Telegram/ThirdParty/scudo/release.h index 5bf963d0f..b6f76a4d2 100644 --- a/Telegram/ThirdParty/scudo/release.h +++ b/Telegram/ThirdParty/scudo/release.h @@ -80,20 +80,53 @@ private: MapPlatformData *Data = nullptr; }; -// A buffer pool which holds a fixed number of static buffers for fast buffer -// allocation. If the request size is greater than `StaticBufferSize`, it'll +class FragmentationRecorder { +public: + FragmentationRecorder() = default; + + uptr getReleasedPagesCount() const { return ReleasedPagesCount; } + + void releasePageRangeToOS(uptr From, uptr To) { + DCHECK_EQ((To - From) % getPageSizeCached(), 0U); + ReleasedPagesCount += (To - From) / getPageSizeCached(); + } + +private: + uptr ReleasedPagesCount = 0; +}; + +// A buffer pool which holds a fixed number of static buffers of `uptr` elements +// for fast buffer allocation. If the request size is greater than +// `StaticBufferNumElements` or if all the static buffers are in use, it'll // delegate the allocation to map(). -template class BufferPool { +template +class BufferPool { public: // Preserve 1 bit in the `Mask` so that we don't need to do zero-check while // extracting the least significant bit from the `Mask`. static_assert(StaticBufferCount < SCUDO_WORDSIZE, ""); - static_assert(isAligned(StaticBufferSize, SCUDO_CACHE_LINE_SIZE), ""); + static_assert(isAligned(StaticBufferNumElements * sizeof(uptr), + SCUDO_CACHE_LINE_SIZE), + ""); - // Return a buffer which is at least `BufferSize`. - uptr *getBuffer(const uptr BufferSize) { - if (UNLIKELY(BufferSize > StaticBufferSize)) - return getDynamicBuffer(BufferSize); + struct Buffer { + // Pointer to the buffer's memory, or nullptr if no buffer was allocated. + uptr *Data = nullptr; + + // The index of the underlying static buffer, or StaticBufferCount if this + // buffer was dynamically allocated. This value is initially set to a poison + // value to aid debugging. + uptr BufferIndex = ~static_cast(0); + + // Only valid if BufferIndex == StaticBufferCount. + MemMapT MemMap = {}; + }; + + // Return a zero-initialized buffer which can contain at least the given + // number of elements, or nullptr on failure. + Buffer getBuffer(const uptr NumElements) { + if (UNLIKELY(NumElements > StaticBufferNumElements)) + return getDynamicBuffer(NumElements); uptr index; { @@ -108,69 +141,55 @@ public: } if (index >= StaticBufferCount) - return getDynamicBuffer(BufferSize); + return getDynamicBuffer(NumElements); - const uptr Offset = index * StaticBufferSize; - memset(&RawBuffer[Offset], 0, StaticBufferSize); - return &RawBuffer[Offset]; + Buffer Buf; + Buf.Data = &RawBuffer[index * StaticBufferNumElements]; + Buf.BufferIndex = index; + memset(Buf.Data, 0, StaticBufferNumElements * sizeof(uptr)); + return Buf; } - void releaseBuffer(uptr *Buffer, const uptr BufferSize) { - const uptr index = getStaticBufferIndex(Buffer, BufferSize); - if (index < StaticBufferCount) { + void releaseBuffer(Buffer Buf) { + DCHECK_NE(Buf.Data, nullptr); + DCHECK_LE(Buf.BufferIndex, StaticBufferCount); + if (Buf.BufferIndex != StaticBufferCount) { ScopedLock L(Mutex); - DCHECK_EQ((Mask & (static_cast(1) << index)), 0U); - Mask |= static_cast(1) << index; + DCHECK_EQ((Mask & (static_cast(1) << Buf.BufferIndex)), 0U); + Mask |= static_cast(1) << Buf.BufferIndex; } else { - unmap(reinterpret_cast(Buffer), - roundUp(BufferSize, getPageSizeCached())); + Buf.MemMap.unmap(Buf.MemMap.getBase(), Buf.MemMap.getCapacity()); } } - bool isStaticBufferTestOnly(uptr *Buffer, uptr BufferSize) { - return getStaticBufferIndex(Buffer, BufferSize) < StaticBufferCount; + bool isStaticBufferTestOnly(const Buffer &Buf) { + DCHECK_NE(Buf.Data, nullptr); + DCHECK_LE(Buf.BufferIndex, StaticBufferCount); + return Buf.BufferIndex != StaticBufferCount; } private: - uptr getStaticBufferIndex(uptr *Buffer, uptr BufferSize) { - if (UNLIKELY(BufferSize > StaticBufferSize)) - return StaticBufferCount; - - const uptr BufferBase = reinterpret_cast(Buffer); - const uptr RawBufferBase = reinterpret_cast(RawBuffer); - - if (BufferBase < RawBufferBase || - BufferBase >= RawBufferBase + sizeof(RawBuffer)) { - return StaticBufferCount; - } - - DCHECK_LE(BufferSize, StaticBufferSize); - DCHECK_LE(BufferBase + BufferSize, RawBufferBase + sizeof(RawBuffer)); - DCHECK_EQ((BufferBase - RawBufferBase) % StaticBufferSize, 0U); - - const uptr index = - (BufferBase - RawBufferBase) / (StaticBufferSize * sizeof(uptr)); - DCHECK_LT(index, StaticBufferCount); - return index; - } - - uptr *getDynamicBuffer(const uptr BufferSize) { + Buffer getDynamicBuffer(const uptr NumElements) { // When using a heap-based buffer, precommit the pages backing the // Vmar by passing |MAP_PRECOMMIT| flag. This allows an optimization // where page fault exceptions are skipped as the allocated memory // is accessed. So far, this is only enabled on Fuchsia. It hasn't proven a // performance benefit on other platforms. const uptr MmapFlags = MAP_ALLOWNOMEM | (SCUDO_FUCHSIA ? MAP_PRECOMMIT : 0); - return reinterpret_cast( - map(nullptr, roundUp(BufferSize, getPageSizeCached()), "scudo:counters", - MmapFlags, &MapData)); + const uptr MappedSize = + roundUp(NumElements * sizeof(uptr), getPageSizeCached()); + Buffer Buf; + if (Buf.MemMap.map(/*Addr=*/0, MappedSize, "scudo:counters", MmapFlags)) { + Buf.Data = reinterpret_cast(Buf.MemMap.getBase()); + Buf.BufferIndex = StaticBufferCount; + } + return Buf; } HybridMutex Mutex; // '1' means that buffer index is not used. '0' means the buffer is in use. uptr Mask GUARDED_BY(Mutex) = ~static_cast(0); - uptr RawBuffer[StaticBufferCount * StaticBufferSize] GUARDED_BY(Mutex); - [[no_unique_address]] MapPlatformData MapData = {}; + uptr RawBuffer[StaticBufferCount * StaticBufferNumElements] GUARDED_BY(Mutex); }; // A Region page map is used to record the usage of pages in the regions. It @@ -185,23 +204,17 @@ private: class RegionPageMap { public: RegionPageMap() - : Regions(0), - NumCounters(0), - CounterSizeBitsLog(0), - CounterMask(0), - PackingRatioLog(0), - BitOffsetMask(0), - SizePerRegion(0), - BufferSize(0), - Buffer(nullptr) {} + : Regions(0), NumCounters(0), CounterSizeBitsLog(0), CounterMask(0), + PackingRatioLog(0), BitOffsetMask(0), SizePerRegion(0), + BufferNumElements(0) {} RegionPageMap(uptr NumberOfRegions, uptr CountersPerRegion, uptr MaxValue) { reset(NumberOfRegions, CountersPerRegion, MaxValue); } ~RegionPageMap() { if (!isAllocated()) return; - Buffers.releaseBuffer(Buffer, BufferSize); - Buffer = nullptr; + Buffers.releaseBuffer(Buffer); + Buffer = {}; } // Lock of `StaticBuffer` is acquired conditionally and there's no easy way to @@ -216,7 +229,7 @@ public: Regions = NumberOfRegion; NumCounters = CountersPerRegion; - constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL; + constexpr uptr MaxCounterBits = sizeof(*Buffer.Data) * 8UL; // Rounding counter storage size up to the power of two allows for using // bit shifts calculating particular counter's Index and offset. const uptr CounterSizeBits = @@ -233,11 +246,11 @@ public: SizePerRegion = roundUp(NumCounters, static_cast(1U) << PackingRatioLog) >> PackingRatioLog; - BufferSize = SizePerRegion * sizeof(*Buffer) * Regions; - Buffer = Buffers.getBuffer(BufferSize); + BufferNumElements = SizePerRegion * Regions; + Buffer = Buffers.getBuffer(BufferNumElements); } - bool isAllocated() const { return !!Buffer; } + bool isAllocated() const { return Buffer.Data != nullptr; } uptr getCount() const { return NumCounters; } @@ -246,7 +259,8 @@ public: DCHECK_LT(I, NumCounters); const uptr Index = I >> PackingRatioLog; const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; - return (Buffer[Region * SizePerRegion + Index] >> BitOffset) & CounterMask; + return (Buffer.Data[Region * SizePerRegion + Index] >> BitOffset) & + CounterMask; } void inc(uptr Region, uptr I) const { @@ -255,8 +269,8 @@ public: const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; DCHECK_LT(BitOffset, SCUDO_WORDSIZE); DCHECK_EQ(isAllCounted(Region, I), false); - Buffer[Region * SizePerRegion + Index] += static_cast(1U) - << BitOffset; + Buffer.Data[Region * SizePerRegion + Index] += static_cast(1U) + << BitOffset; } void incN(uptr Region, uptr I, uptr N) const { @@ -267,7 +281,7 @@ public: const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; DCHECK_LT(BitOffset, SCUDO_WORDSIZE); DCHECK_EQ(isAllCounted(Region, I), false); - Buffer[Region * SizePerRegion + Index] += N << BitOffset; + Buffer.Data[Region * SizePerRegion + Index] += N << BitOffset; } void incRange(uptr Region, uptr From, uptr To) const { @@ -286,7 +300,7 @@ public: const uptr Index = I >> PackingRatioLog; const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; DCHECK_LT(BitOffset, SCUDO_WORDSIZE); - Buffer[Region * SizePerRegion + Index] |= CounterMask << BitOffset; + Buffer.Data[Region * SizePerRegion + Index] |= CounterMask << BitOffset; } void setAsAllCountedRange(uptr Region, uptr From, uptr To) const { DCHECK_LE(From, To); @@ -309,9 +323,16 @@ public: return get(Region, I) == CounterMask; } - uptr getBufferSize() const { return BufferSize; } + uptr getBufferNumElements() const { return BufferNumElements; } private: + // We may consider making this configurable if there are cases which may + // benefit from this. + static const uptr StaticBufferCount = 2U; + static const uptr StaticBufferNumElements = 512U; + using BufferPoolT = BufferPool; + static BufferPoolT Buffers; + uptr Regions; uptr NumCounters; uptr CounterSizeBitsLog; @@ -320,14 +341,8 @@ private: uptr BitOffsetMask; uptr SizePerRegion; - uptr BufferSize; - uptr *Buffer; - - // We may consider making this configurable if there are cases which may - // benefit from this. - static const uptr StaticBufferCount = 2U; - static const uptr StaticBufferSize = 512U; - static BufferPool Buffers; + uptr BufferNumElements; + BufferPoolT::Buffer Buffer; }; template class FreePagesRangeTracker { diff --git a/Telegram/ThirdParty/scudo/report.cpp b/Telegram/ThirdParty/scudo/report.cpp index 81b3dce4e..9cef0adc0 100644 --- a/Telegram/ThirdParty/scudo/report.cpp +++ b/Telegram/ThirdParty/scudo/report.cpp @@ -24,11 +24,7 @@ public: Message.vappend(Format, Args); va_end(Args); } - NORETURN ~ScopedErrorReport() { - outputRaw(Message.data()); - setAbortMessage(Message.data()); - die(); - } + NORETURN ~ScopedErrorReport() { reportRawError(Message.data()); } private: ScopedString Message; @@ -36,18 +32,6 @@ private: inline void NORETURN trap() { __builtin_trap(); } -void NORETURN reportSoftRSSLimit(uptr RssLimitMb) { - ScopedErrorReport Report; - Report.append("Soft RSS limit of %zu MB exhausted, current RSS is %zu MB\n", - RssLimitMb, GetRSS() >> 20); -} - -void NORETURN reportHardRSSLimit(uptr RssLimitMb) { - ScopedErrorReport Report; - Report.append("Hard RSS limit of %zu MB exhausted, current RSS is %zu MB\n", - RssLimitMb, GetRSS() >> 20); -} - // This could potentially be called recursively if a CHECK fails in the reports. void NORETURN reportCheckFailed(const char *File, int Line, const char *Condition, u64 Value1, u64 Value2) { @@ -67,6 +51,13 @@ void NORETURN reportError(const char *Message) { Report.append("%s\n", Message); } +// Generic fatal error message without ScopedString. +void NORETURN reportRawError(const char *Message) { + outputRaw(Message); + setAbortMessage(Message); + die(); +} + void NORETURN reportInvalidFlag(const char *FlagType, const char *Value) { ScopedErrorReport Report; Report.append("invalid value for %s option: '%s'\n", FlagType, Value); @@ -79,14 +70,6 @@ void NORETURN reportHeaderCorruption(void *Ptr) { Report.append("corrupted chunk header at address %p\n", Ptr); } -// Two threads have attempted to modify a chunk header at the same time. This is -// symptomatic of a race-condition in the application code, or general lack of -// proper locking. -void NORETURN reportHeaderRace(void *Ptr) { - ScopedErrorReport Report; - Report.append("race on chunk header at address %p\n", Ptr); -} - // The allocator was compiled with parameters that conflict with field size // requirements. void NORETURN reportSanityCheckError(const char *Field) { diff --git a/Telegram/ThirdParty/scudo/report.h b/Telegram/ThirdParty/scudo/report.h index 3a78ab64b..a510fdaeb 100644 --- a/Telegram/ThirdParty/scudo/report.h +++ b/Telegram/ThirdParty/scudo/report.h @@ -15,15 +15,17 @@ namespace scudo { // Reports are *fatal* unless stated otherwise. -// Generic error. +// Generic error, adds newline to end of message. void NORETURN reportError(const char *Message); +// Generic error, but the message is not modified. +void NORETURN reportRawError(const char *Message); + // Flags related errors. void NORETURN reportInvalidFlag(const char *FlagType, const char *Value); // Chunk header related errors. void NORETURN reportHeaderCorruption(void *Ptr); -void NORETURN reportHeaderRace(void *Ptr); // Sanity checks related error. void NORETURN reportSanityCheckError(const char *Field); @@ -34,8 +36,6 @@ void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, uptr MaxSize); void NORETURN reportOutOfBatchClass(); void NORETURN reportOutOfMemory(uptr RequestedSize); -void NORETURN reportSoftRSSLimit(uptr RssLimitMb); -void NORETURN reportHardRSSLimit(uptr RssLimitMb); enum class AllocatorAction : u8 { Recycling, Deallocating, diff --git a/Telegram/ThirdParty/scudo/report_linux.cpp b/Telegram/ThirdParty/scudo/report_linux.cpp new file mode 100644 index 000000000..6a983036e --- /dev/null +++ b/Telegram/ThirdParty/scudo/report_linux.cpp @@ -0,0 +1,58 @@ +//===-- report_linux.cpp ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_LINUX || SCUDO_TRUSTY + +#include "common.h" +#include "internal_defs.h" +#include "report.h" +#include "report_linux.h" +#include "string_utils.h" + +#include +#include +#include + +namespace scudo { + +// Fatal internal map() error (potentially OOM related). +void NORETURN reportMapError(uptr SizeIfOOM) { + char Error[128] = "Scudo ERROR: internal map failure\n"; + if (SizeIfOOM) { + formatString( + Error, sizeof(Error), + "Scudo ERROR: internal map failure (NO MEMORY) requesting %zuKB\n", + SizeIfOOM >> 10); + } + reportRawError(Error); +} + +void NORETURN reportUnmapError(uptr Addr, uptr Size) { + char Error[128]; + formatString(Error, sizeof(Error), + "Scudo ERROR: internal unmap failure (error desc=%s) Addr 0x%zx " + "Size %zu\n", + strerror(errno), Addr, Size); + reportRawError(Error); +} + +void NORETURN reportProtectError(uptr Addr, uptr Size, int Prot) { + char Error[128]; + formatString( + Error, sizeof(Error), + "Scudo ERROR: internal protect failure (error desc=%s) Addr 0x%zx " + "Size %zu Prot %x\n", + strerror(errno), Addr, Size, Prot); + reportRawError(Error); +} + +} // namespace scudo + +#endif // SCUDO_LINUX || SCUDO_TRUSTY diff --git a/Telegram/ThirdParty/scudo/report_linux.h b/Telegram/ThirdParty/scudo/report_linux.h new file mode 100644 index 000000000..aa0bb247e --- /dev/null +++ b/Telegram/ThirdParty/scudo/report_linux.h @@ -0,0 +1,34 @@ +//===-- report_linux.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_REPORT_LINUX_H_ +#define SCUDO_REPORT_LINUX_H_ + +#include "platform.h" + +#if SCUDO_LINUX || SCUDO_TRUSTY + +#include "internal_defs.h" + +namespace scudo { + +// Report a fatal error when a map call fails. SizeIfOOM shall +// hold the requested size on an out-of-memory error, 0 otherwise. +void NORETURN reportMapError(uptr SizeIfOOM = 0); + +// Report a fatal error when an unmap call fails. +void NORETURN reportUnmapError(uptr Addr, uptr Size); + +// Report a fatal error when a mprotect call fails. +void NORETURN reportProtectError(uptr Addr, uptr Size, int Prot); + +} // namespace scudo + +#endif // SCUDO_LINUX || SCUDO_TRUSTY + +#endif // SCUDO_REPORT_LINUX_H_ diff --git a/Telegram/ThirdParty/scudo/rss_limit_checker.cpp b/Telegram/ThirdParty/scudo/rss_limit_checker.cpp deleted file mode 100644 index f428386b7..000000000 --- a/Telegram/ThirdParty/scudo/rss_limit_checker.cpp +++ /dev/null @@ -1,37 +0,0 @@ -//===-- common.cpp ----------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "rss_limit_checker.h" -#include "atomic_helpers.h" -#include "string_utils.h" - -namespace scudo { - -void RssLimitChecker::check(u64 NextCheck) { - // The interval for the checks is 250ms. - static constexpr u64 CheckInterval = 250 * 1000000; - - // Early return in case another thread already did the calculation. - if (!atomic_compare_exchange_strong(&RssNextCheckAtNS, &NextCheck, - getMonotonicTime() + CheckInterval, - memory_order_relaxed)) { - return; - } - - const uptr CurrentRssMb = GetRSS() >> 20; - - RssLimitExceeded Result = RssLimitExceeded::Neither; - if (UNLIKELY(HardRssLimitMb && HardRssLimitMb < CurrentRssMb)) - Result = RssLimitExceeded::Hard; - else if (UNLIKELY(SoftRssLimitMb && SoftRssLimitMb < CurrentRssMb)) - Result = RssLimitExceeded::Soft; - - atomic_store_relaxed(&RssLimitStatus, static_cast(Result)); -} - -} // namespace scudo diff --git a/Telegram/ThirdParty/scudo/rss_limit_checker.h b/Telegram/ThirdParty/scudo/rss_limit_checker.h deleted file mode 100644 index 29dc063f3..000000000 --- a/Telegram/ThirdParty/scudo/rss_limit_checker.h +++ /dev/null @@ -1,63 +0,0 @@ -//===-- common.h ------------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_RSS_LIMIT_CHECKER_H_ -#define SCUDO_RSS_LIMIT_CHECKER_H_ - -#include "atomic_helpers.h" -#include "common.h" -#include "internal_defs.h" - -namespace scudo { - -class RssLimitChecker { -public: - enum RssLimitExceeded { - Neither, - Soft, - Hard, - }; - - void init(int SoftRssLimitMb, int HardRssLimitMb) { - CHECK_GE(SoftRssLimitMb, 0); - CHECK_GE(HardRssLimitMb, 0); - this->SoftRssLimitMb = static_cast(SoftRssLimitMb); - this->HardRssLimitMb = static_cast(HardRssLimitMb); - } - - // Opportunistic RSS limit check. This will update the RSS limit status, if - // it can, every 250ms, otherwise it will just return the current one. - RssLimitExceeded getRssLimitExceeded() { - if (!HardRssLimitMb && !SoftRssLimitMb) - return RssLimitExceeded::Neither; - - u64 NextCheck = atomic_load_relaxed(&RssNextCheckAtNS); - u64 Now = getMonotonicTime(); - - if (UNLIKELY(Now >= NextCheck)) - check(NextCheck); - - return static_cast(atomic_load_relaxed(&RssLimitStatus)); - } - - uptr getSoftRssLimit() const { return SoftRssLimitMb; } - uptr getHardRssLimit() const { return HardRssLimitMb; } - -private: - void check(u64 NextCheck); - - uptr SoftRssLimitMb = 0; - uptr HardRssLimitMb = 0; - - atomic_u64 RssNextCheckAtNS = {}; - atomic_u8 RssLimitStatus = {}; -}; - -} // namespace scudo - -#endif // SCUDO_RSS_LIMIT_CHECKER_H_ diff --git a/Telegram/ThirdParty/scudo/secondary.h b/Telegram/ThirdParty/scudo/secondary.h index 105b154b5..f52a4188b 100644 --- a/Telegram/ThirdParty/scudo/secondary.h +++ b/Telegram/ThirdParty/scudo/secondary.h @@ -72,11 +72,26 @@ static inline void unmap(LargeBlock::Header *H) { MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); } +namespace { +struct CachedBlock { + uptr CommitBase = 0; + uptr CommitSize = 0; + uptr BlockBegin = 0; + MemMapT MemMap = {}; + u64 Time = 0; + + bool isValid() { return CommitBase != 0; } + + void invalidate() { CommitBase = 0; } +}; +} // namespace + template class MapAllocatorNoCache { public: void init(UNUSED s32 ReleaseToOsInterval) {} bool retrieve(UNUSED Options Options, UNUSED uptr Size, UNUSED uptr Alignment, - UNUSED LargeBlock::Header **H, UNUSED bool *Zeroed) { + UNUSED uptr HeadersSize, UNUSED LargeBlock::Header **H, + UNUSED bool *Zeroed) { return false; } void store(UNUSED Options Options, LargeBlock::Header *H) { unmap(H); } @@ -102,20 +117,22 @@ public: static const uptr MaxUnusedCachePages = 4U; template -void mapSecondary(Options Options, uptr CommitBase, uptr CommitSize, +bool mapSecondary(const Options &Options, uptr CommitBase, uptr CommitSize, uptr AllocPos, uptr Flags, MemMapT &MemMap) { + Flags |= MAP_RESIZABLE; + Flags |= MAP_ALLOWNOMEM; + const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * getPageSizeCached(); if (useMemoryTagging(Options) && CommitSize > MaxUnusedCacheBytes) { const uptr UntaggedPos = Max(AllocPos, CommitBase + MaxUnusedCacheBytes); - MemMap.remap(CommitBase, UntaggedPos - CommitBase, "scudo:secondary", - MAP_RESIZABLE | MAP_MEMTAG | Flags); - MemMap.remap(UntaggedPos, CommitBase + CommitSize - UntaggedPos, - "scudo:secondary", MAP_RESIZABLE | Flags); + return MemMap.remap(CommitBase, UntaggedPos - CommitBase, "scudo:secondary", + MAP_MEMTAG | Flags) && + MemMap.remap(UntaggedPos, CommitBase + CommitSize - UntaggedPos, + "scudo:secondary", Flags); } else { const uptr RemapFlags = - MAP_RESIZABLE | (useMemoryTagging(Options) ? MAP_MEMTAG : 0) | - Flags; - MemMap.remap(CommitBase, CommitSize, "scudo:secondary", RemapFlags); + (useMemoryTagging(Options) ? MAP_MEMTAG : 0) | Flags; + return MemMap.remap(CommitBase, CommitSize, "scudo:secondary", RemapFlags); } } @@ -138,17 +155,24 @@ public: void getStats(ScopedString *Str) { ScopedLock L(Mutex); + uptr Integral; + uptr Fractional; + computePercentage(SuccessfulRetrieves, CallsToRetrieve, &Integral, + &Fractional); Str->append("Stats: MapAllocatorCache: EntriesCount: %d, " "MaxEntriesCount: %u, MaxEntrySize: %zu\n", EntriesCount, atomic_load_relaxed(&MaxEntriesCount), atomic_load_relaxed(&MaxEntrySize)); + Str->append("Stats: CacheRetrievalStats: SuccessRate: %u/%u " + "(%zu.%02zu%%)\n", + SuccessfulRetrieves, CallsToRetrieve, Integral, Fractional); for (CachedBlock Entry : Entries) { - if (!Entry.CommitBase) + if (!Entry.isValid()) continue; Str->append("StartBlockAddress: 0x%zx, EndBlockAddress: 0x%zx, " - "BlockSize: %zu\n", + "BlockSize: %zu %s\n", Entry.CommitBase, Entry.CommitBase + Entry.CommitSize, - Entry.CommitSize); + Entry.CommitSize, Entry.Time == 0 ? "[R]" : ""); } } @@ -166,7 +190,7 @@ public: setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); } - void store(Options Options, LargeBlock::Header *H) EXCLUDES(Mutex) { + void store(const Options &Options, LargeBlock::Header *H) EXCLUDES(Mutex) { if (!canCache(H->CommitSize)) return unmap(H); @@ -195,7 +219,7 @@ public: MAP_NOACCESS); } } else if (Interval == 0) { - Entry.MemMap.releasePagesToOS(Entry.CommitBase, Entry.CommitSize); + Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase, Entry.CommitSize); Entry.Time = 0; } do { @@ -210,7 +234,7 @@ public: if (CacheConfig::QuarantineSize && useMemoryTagging(Options)) { QuarantinePos = (QuarantinePos + 1) % Max(CacheConfig::QuarantineSize, 1u); - if (!Quarantine[QuarantinePos].CommitBase) { + if (!Quarantine[QuarantinePos].isValid()) { Quarantine[QuarantinePos] = Entry; return; } @@ -225,7 +249,7 @@ public: EmptyCache = true; } else { for (u32 I = 0; I < MaxCount; I++) { - if (Entries[I].CommitBase) + if (Entries[I].isValid()) continue; if (I != 0) Entries[I] = Entries[0]; @@ -246,26 +270,31 @@ public: Entry.MemMap.unmap(Entry.MemMap.getBase(), Entry.MemMap.getCapacity()); } - bool retrieve(Options Options, uptr Size, uptr Alignment, + bool retrieve(Options Options, uptr Size, uptr Alignment, uptr HeadersSize, LargeBlock::Header **H, bool *Zeroed) EXCLUDES(Mutex) { const uptr PageSize = getPageSizeCached(); const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); + // 10% of the requested size proved to be the optimal choice for + // retrieving cached blocks after testing several options. + constexpr u32 FragmentedBytesDivisor = 10; bool Found = false; CachedBlock Entry; - uptr HeaderPos = 0; + uptr EntryHeaderPos = 0; { ScopedLock L(Mutex); + CallsToRetrieve++; if (EntriesCount == 0) return false; + u32 OptimalFitIndex = 0; + uptr MinDiff = UINTPTR_MAX; for (u32 I = 0; I < MaxCount; I++) { - const uptr CommitBase = Entries[I].CommitBase; - if (!CommitBase) + if (!Entries[I].isValid()) continue; + const uptr CommitBase = Entries[I].CommitBase; const uptr CommitSize = Entries[I].CommitSize; const uptr AllocPos = roundDown(CommitBase + CommitSize - Size, Alignment); - HeaderPos = - AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize(); + const uptr HeaderPos = AllocPos - HeadersSize; if (HeaderPos > CommitBase + CommitSize) continue; if (HeaderPos < CommitBase || @@ -273,17 +302,36 @@ public: continue; } Found = true; - Entry = Entries[I]; - Entries[I].CommitBase = 0; + const uptr Diff = HeaderPos - CommitBase; + // immediately use a cached block if it's size is close enough to the + // requested size. + const uptr MaxAllowedFragmentedBytes = + (CommitBase + CommitSize - HeaderPos) / FragmentedBytesDivisor; + if (Diff <= MaxAllowedFragmentedBytes) { + OptimalFitIndex = I; + EntryHeaderPos = HeaderPos; + break; + } + // keep track of the smallest cached block + // that is greater than (AllocSize + HeaderSize) + if (Diff > MinDiff) + continue; + OptimalFitIndex = I; + MinDiff = Diff; + EntryHeaderPos = HeaderPos; + } + if (Found) { + Entry = Entries[OptimalFitIndex]; + Entries[OptimalFitIndex].invalidate(); EntriesCount--; - break; + SuccessfulRetrieves++; } } if (!Found) return false; *H = reinterpret_cast( - LargeBlock::addHeaderTag(HeaderPos)); + LargeBlock::addHeaderTag(EntryHeaderPos)); *Zeroed = Entry.Time == 0; if (useMemoryTagging(Options)) Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0); @@ -295,8 +343,7 @@ public: } else if (Entry.BlockBegin < NewBlockBegin) { storeTags(Entry.BlockBegin, NewBlockBegin); } else { - storeTags(untagPointer(NewBlockBegin), - untagPointer(Entry.BlockBegin)); + storeTags(untagPointer(NewBlockBegin), untagPointer(Entry.BlockBegin)); } } (*H)->CommitBase = Entry.CommitBase; @@ -338,15 +385,15 @@ public: void disableMemoryTagging() EXCLUDES(Mutex) { ScopedLock L(Mutex); for (u32 I = 0; I != CacheConfig::QuarantineSize; ++I) { - if (Quarantine[I].CommitBase) { + if (Quarantine[I].isValid()) { MemMapT &MemMap = Quarantine[I].MemMap; MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); - Quarantine[I].CommitBase = 0; + Quarantine[I].invalidate(); } } const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); for (u32 I = 0; I < MaxCount; I++) { - if (Entries[I].CommitBase) { + if (Entries[I].isValid()) { Entries[I].MemMap.setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, 0); } @@ -367,10 +414,10 @@ private: { ScopedLock L(Mutex); for (uptr I = 0; I < CacheConfig::EntriesArraySize; I++) { - if (!Entries[I].CommitBase) + if (!Entries[I].isValid()) continue; MapInfo[N] = Entries[I].MemMap; - Entries[I].CommitBase = 0; + Entries[I].invalidate(); N++; } EntriesCount = 0; @@ -382,23 +429,15 @@ private: } } - struct CachedBlock { - uptr CommitBase = 0; - uptr CommitSize = 0; - uptr BlockBegin = 0; - MemMapT MemMap = {}; - u64 Time = 0; - }; - void releaseIfOlderThan(CachedBlock &Entry, u64 Time) REQUIRES(Mutex) { - if (!Entry.CommitBase || !Entry.Time) + if (!Entry.isValid() || !Entry.Time) return; if (Entry.Time > Time) { if (OldestTime == 0 || Entry.Time < OldestTime) OldestTime = Entry.Time; return; } - Entry.MemMap.releasePagesToOS(Entry.CommitBase, Entry.CommitSize); + Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase, Entry.CommitSize); Entry.Time = 0; } @@ -421,6 +460,8 @@ private: u64 OldestTime GUARDED_BY(Mutex) = 0; u32 IsFullEvents GUARDED_BY(Mutex) = 0; atomic_s32 ReleaseToOsIntervalMs = {}; + u32 CallsToRetrieve GUARDED_BY(Mutex) = 0; + u32 SuccessfulRetrieves GUARDED_BY(Mutex) = 0; CachedBlock Entries[CacheConfig::EntriesArraySize] GUARDED_BY(Mutex) = {}; NonZeroLengthArray @@ -439,11 +480,11 @@ public: S->link(&Stats); } - void *allocate(Options Options, uptr Size, uptr AlignmentHint = 0, + void *allocate(const Options &Options, uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr, FillContentsMode FillContents = NoFill); - void deallocate(Options Options, void *Ptr); + void deallocate(const Options &Options, void *Ptr); static uptr getBlockEnd(void *Ptr) { auto *B = LargeBlock::getHeader(Ptr); @@ -454,6 +495,10 @@ public: return getBlockEnd(Ptr) - reinterpret_cast(Ptr); } + static constexpr uptr getHeadersSize() { + return Chunk::getHeaderSize() + LargeBlock::getHeaderSize(); + } + void disable() NO_THREAD_SAFETY_ANALYSIS { Mutex.lock(); Cache.disable(); @@ -494,6 +539,7 @@ private: DoublyLinkedList InUseBlocks GUARDED_BY(Mutex); uptr AllocatedBytes GUARDED_BY(Mutex) = 0; uptr FreedBytes GUARDED_BY(Mutex) = 0; + uptr FragmentedBytes GUARDED_BY(Mutex) = 0; uptr LargestSize GUARDED_BY(Mutex) = 0; u32 NumberOfAllocs GUARDED_BY(Mutex) = 0; u32 NumberOfFrees GUARDED_BY(Mutex) = 0; @@ -512,24 +558,23 @@ private: // the committed memory will amount to something close to Size - AlignmentHint // (pending rounding and headers). template -void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, - uptr *BlockEndPtr, +void *MapAllocator::allocate(const Options &Options, uptr Size, + uptr Alignment, uptr *BlockEndPtr, FillContentsMode FillContents) { if (Options.get(OptionBit::AddLargeAllocationSlack)) Size += 1UL << SCUDO_MIN_ALIGNMENT_LOG; Alignment = Max(Alignment, uptr(1U) << SCUDO_MIN_ALIGNMENT_LOG); const uptr PageSize = getPageSizeCached(); - uptr RoundedSize = - roundUp(roundUp(Size, Alignment) + LargeBlock::getHeaderSize() + - Chunk::getHeaderSize(), - PageSize); - if (Alignment > PageSize) - RoundedSize += Alignment - PageSize; - if (Alignment < PageSize && Cache.canCache(RoundedSize)) { + // Note that cached blocks may have aligned address already. Thus we simply + // pass the required size (`Size` + `getHeadersSize()`) to do cache look up. + const uptr MinNeededSizeForCache = roundUp(Size + getHeadersSize(), PageSize); + + if (Alignment < PageSize && Cache.canCache(MinNeededSizeForCache)) { LargeBlock::Header *H; bool Zeroed; - if (Cache.retrieve(Options, Size, Alignment, &H, &Zeroed)) { + if (Cache.retrieve(Options, Size, Alignment, getHeadersSize(), &H, + &Zeroed)) { const uptr BlockEnd = H->CommitBase + H->CommitSize; if (BlockEndPtr) *BlockEndPtr = BlockEnd; @@ -545,6 +590,7 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, ScopedLock L(Mutex); InUseBlocks.push_back(H); AllocatedBytes += H->CommitSize; + FragmentedBytes += H->MemMap.getCapacity() - H->CommitSize; NumberOfAllocs++; Stats.add(StatAllocated, H->CommitSize); Stats.add(StatMapped, H->MemMap.getCapacity()); @@ -553,16 +599,22 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, } } + uptr RoundedSize = + roundUp(roundUp(Size, Alignment) + getHeadersSize(), PageSize); + if (Alignment > PageSize) + RoundedSize += Alignment - PageSize; + ReservedMemoryT ReservedMemory; const uptr MapSize = RoundedSize + 2 * PageSize; - ReservedMemory.create(/*Addr=*/0U, MapSize, nullptr, MAP_ALLOWNOMEM); + if (UNLIKELY(!ReservedMemory.create(/*Addr=*/0U, MapSize, nullptr, + MAP_ALLOWNOMEM))) { + return nullptr; + } // Take the entire ownership of reserved region. MemMapT MemMap = ReservedMemory.dispatch(ReservedMemory.getBase(), ReservedMemory.getCapacity()); uptr MapBase = MemMap.getBase(); - if (UNLIKELY(!MapBase)) - return nullptr; uptr CommitBase = MapBase + PageSize; uptr MapEnd = MapBase + MapSize; @@ -592,9 +644,12 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, const uptr CommitSize = MapEnd - PageSize - CommitBase; const uptr AllocPos = roundDown(CommitBase + CommitSize - Size, Alignment); - mapSecondary(Options, CommitBase, CommitSize, AllocPos, 0, MemMap); - const uptr HeaderPos = - AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize(); + if (!mapSecondary(Options, CommitBase, CommitSize, AllocPos, 0, + MemMap)) { + MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); + return nullptr; + } + const uptr HeaderPos = AllocPos - getHeadersSize(); LargeBlock::Header *H = reinterpret_cast( LargeBlock::addHeaderTag(HeaderPos)); if (useMemoryTagging(Options)) @@ -609,6 +664,7 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, ScopedLock L(Mutex); InUseBlocks.push_back(H); AllocatedBytes += CommitSize; + FragmentedBytes += H->MemMap.getCapacity() - CommitSize; if (LargestSize < CommitSize) LargestSize = CommitSize; NumberOfAllocs++; @@ -619,7 +675,7 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, } template -void MapAllocator::deallocate(Options Options, void *Ptr) +void MapAllocator::deallocate(const Options &Options, void *Ptr) EXCLUDES(Mutex) { LargeBlock::Header *H = LargeBlock::getHeader(Ptr); const uptr CommitSize = H->CommitSize; @@ -627,6 +683,7 @@ void MapAllocator::deallocate(Options Options, void *Ptr) ScopedLock L(Mutex); InUseBlocks.remove(H); FreedBytes += CommitSize; + FragmentedBytes -= H->MemMap.getCapacity() - CommitSize; NumberOfFrees++; Stats.sub(StatAllocated, CommitSize); Stats.sub(StatMapped, H->MemMap.getCapacity()); @@ -638,10 +695,11 @@ template void MapAllocator::getStats(ScopedString *Str) EXCLUDES(Mutex) { ScopedLock L(Mutex); Str->append("Stats: MapAllocator: allocated %u times (%zuK), freed %u times " - "(%zuK), remains %u (%zuK) max %zuM\n", + "(%zuK), remains %u (%zuK) max %zuM, Fragmented %zuK\n", NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10, NumberOfAllocs - NumberOfFrees, - (AllocatedBytes - FreedBytes) >> 10, LargestSize >> 20); + (AllocatedBytes - FreedBytes) >> 10, LargestSize >> 20, + FragmentedBytes >> 10); Cache.getStats(Str); } diff --git a/Telegram/ThirdParty/scudo/size_class_map.h b/Telegram/ThirdParty/scudo/size_class_map.h index 2a6e298f9..4138885de 100644 --- a/Telegram/ThirdParty/scudo/size_class_map.h +++ b/Telegram/ThirdParty/scudo/size_class_map.h @@ -254,7 +254,7 @@ struct AndroidSizeClassConfig { static const u16 MaxNumCachedHint = 13; static const uptr MaxBytesCachedLog = 13; - static constexpr u32 Classes[] = { + static constexpr uptr Classes[] = { 0x00020, 0x00030, 0x00040, 0x00050, 0x00060, 0x00070, 0x00090, 0x000b0, 0x000c0, 0x000e0, 0x00120, 0x00160, 0x001c0, 0x00250, 0x00320, 0x00450, 0x00670, 0x00830, 0x00a10, 0x00c30, 0x01010, 0x01210, 0x01bd0, 0x02210, @@ -269,7 +269,7 @@ struct AndroidSizeClassConfig { static const u16 MaxNumCachedHint = 14; static const uptr MaxBytesCachedLog = 13; - static constexpr u32 Classes[] = { + static constexpr uptr Classes[] = { 0x00020, 0x00030, 0x00040, 0x00050, 0x00060, 0x00070, 0x00080, 0x00090, 0x000a0, 0x000b0, 0x000c0, 0x000e0, 0x000f0, 0x00110, 0x00120, 0x00130, 0x00150, 0x00160, 0x00170, 0x00190, 0x001d0, 0x00210, 0x00240, 0x002a0, @@ -289,28 +289,6 @@ typedef TableSizeClassMap AndroidSizeClassMap; static_assert(AndroidSizeClassMap::usesCompressedLSBFormat(), ""); #endif -struct SvelteSizeClassConfig { -#if SCUDO_WORDSIZE == 64U - static const uptr NumBits = 4; - static const uptr MinSizeLog = 4; - static const uptr MidSizeLog = 8; - static const uptr MaxSizeLog = 14; - static const u16 MaxNumCachedHint = 13; - static const uptr MaxBytesCachedLog = 10; - static const uptr SizeDelta = Chunk::getHeaderSize(); -#else - static const uptr NumBits = 4; - static const uptr MinSizeLog = 3; - static const uptr MidSizeLog = 7; - static const uptr MaxSizeLog = 14; - static const u16 MaxNumCachedHint = 14; - static const uptr MaxBytesCachedLog = 10; - static const uptr SizeDelta = Chunk::getHeaderSize(); -#endif -}; - -typedef FixedSizeClassMap SvelteSizeClassMap; - struct TrustySizeClassConfig { static const uptr NumBits = 1; static const uptr MinSizeLog = 5; diff --git a/Telegram/ThirdParty/scudo/stack_depot.h b/Telegram/ThirdParty/scudo/stack_depot.h index 458198fcb..12c35eb2a 100644 --- a/Telegram/ThirdParty/scudo/stack_depot.h +++ b/Telegram/ThirdParty/scudo/stack_depot.h @@ -62,8 +62,7 @@ class StackDepot { // This is achieved by re-checking the hash of the stack trace before // returning the trace. -#ifdef SCUDO_FUZZ - // Use smaller table sizes for fuzzing in order to reduce input size. +#if SCUDO_SMALL_STACK_DEPOT static const uptr TabBits = 4; #else static const uptr TabBits = 16; @@ -72,7 +71,7 @@ class StackDepot { static const uptr TabMask = TabSize - 1; atomic_u32 Tab[TabSize] = {}; -#ifdef SCUDO_FUZZ +#if SCUDO_SMALL_STACK_DEPOT static const uptr RingBits = 4; #else static const uptr RingBits = 19; diff --git a/Telegram/ThirdParty/scudo/tests/CMakeLists.txt b/Telegram/ThirdParty/scudo/tests/CMakeLists.txt index cbb85ce3e..c6b6a1cb5 100644 --- a/Telegram/ThirdParty/scudo/tests/CMakeLists.txt +++ b/Telegram/ThirdParty/scudo/tests/CMakeLists.txt @@ -15,11 +15,15 @@ set(SCUDO_UNITTEST_CFLAGS -DGTEST_HAS_RTTI=0 -g # Extra flags for the C++ tests + -Wconversion # TODO(kostyak): find a way to make -fsized-deallocation work -Wno-mismatched-new-delete) if(COMPILER_RT_DEBUG) - list(APPEND SCUDO_UNITTEST_CFLAGS -DSCUDO_DEBUG=1) + list(APPEND SCUDO_UNITTEST_CFLAGS -DSCUDO_DEBUG=1 -DSCUDO_ENABLE_HOOKS=1) + if (NOT FUCHSIA) + list(APPEND SCUDO_UNITTEST_CFLAGS -DSCUDO_ENABLE_HOOKS_TESTS=1) + endif() endif() if(ANDROID) @@ -92,6 +96,7 @@ set(SCUDO_UNIT_TEST_SOURCES chunk_test.cpp combined_test.cpp common_test.cpp + condition_variable_test.cpp flags_test.cpp list_test.cpp map_test.cpp @@ -137,11 +142,3 @@ set(SCUDO_CXX_UNIT_TEST_SOURCES add_scudo_unittest(ScudoCxxUnitTest SOURCES ${SCUDO_CXX_UNIT_TEST_SOURCES} ADDITIONAL_RTOBJECTS RTScudoStandaloneCWrappers RTScudoStandaloneCxxWrappers) - -set(SCUDO_HOOKS_UNIT_TEST_SOURCES - scudo_hooks_test.cpp - scudo_unit_test_main.cpp - ) - -add_scudo_unittest(ScudoHooksUnitTest - SOURCES ${SCUDO_HOOKS_UNIT_TEST_SOURCES}) diff --git a/Telegram/ThirdParty/scudo/tests/chunk_test.cpp b/Telegram/ThirdParty/scudo/tests/chunk_test.cpp index 7a29f3c11..1b2c1eb5a 100644 --- a/Telegram/ThirdParty/scudo/tests/chunk_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/chunk_test.cpp @@ -37,29 +37,6 @@ TEST(ScudoChunkDeathTest, ChunkBasic) { free(Block); } -TEST(ScudoChunkTest, ChunkCmpXchg) { - initChecksum(); - const scudo::uptr Size = 0x100U; - scudo::Chunk::UnpackedHeader OldHeader = {}; - OldHeader.OriginOrWasZeroed = scudo::Chunk::Origin::Malloc; - OldHeader.ClassId = 0x42U; - OldHeader.SizeOrUnusedBytes = Size; - OldHeader.State = scudo::Chunk::State::Allocated; - void *Block = malloc(HeaderSize + Size); - void *P = reinterpret_cast(reinterpret_cast(Block) + - HeaderSize); - scudo::Chunk::storeHeader(Cookie, P, &OldHeader); - memset(P, 'A', Size); - scudo::Chunk::UnpackedHeader NewHeader = OldHeader; - NewHeader.State = scudo::Chunk::State::Quarantined; - scudo::Chunk::compareExchangeHeader(Cookie, P, &NewHeader, &OldHeader); - NewHeader = {}; - EXPECT_TRUE(scudo::Chunk::isValid(Cookie, P, &NewHeader)); - EXPECT_EQ(NewHeader.State, scudo::Chunk::State::Quarantined); - EXPECT_FALSE(scudo::Chunk::isValid(InvalidCookie, P, &NewHeader)); - free(Block); -} - TEST(ScudoChunkDeathTest, CorruptHeader) { initChecksum(); const scudo::uptr Size = 0x100U; diff --git a/Telegram/ThirdParty/scudo/tests/combined_test.cpp b/Telegram/ThirdParty/scudo/tests/combined_test.cpp index 79d327849..3dbd93cac 100644 --- a/Telegram/ThirdParty/scudo/tests/combined_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/combined_test.cpp @@ -12,8 +12,11 @@ #include "allocator_config.h" #include "chunk.h" #include "combined.h" +#include "condition_variable.h" #include "mem_map.h" +#include "size_class_map.h" +#include #include #include #include @@ -54,7 +57,7 @@ void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size, EXPECT_DEATH( { disableDebuggerdMaybe(); - reinterpret_cast(P)[-1] = 0xaa; + reinterpret_cast(P)[-1] = 'A'; }, ""); if (isPrimaryAllocation(Size, Alignment) @@ -63,7 +66,7 @@ void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size, EXPECT_DEATH( { disableDebuggerdMaybe(); - reinterpret_cast(P)[Size] = 0xaa; + reinterpret_cast(P)[Size] = 'A'; }, ""); } @@ -78,14 +81,70 @@ template struct TestAllocator : scudo::Allocator { } ~TestAllocator() { this->unmapTestOnly(); } - void *operator new(size_t size) { - void *p = nullptr; - EXPECT_EQ(0, posix_memalign(&p, alignof(TestAllocator), size)); - return p; + void *operator new(size_t size); + void operator delete(void *ptr); +}; + +constexpr size_t kMaxAlign = std::max({ + alignof(scudo::Allocator), +#if SCUDO_CAN_USE_PRIMARY64 + alignof(scudo::Allocator), +#endif + alignof(scudo::Allocator) +}); + +#if SCUDO_RISCV64 +// The allocator is over 4MB large. Rather than creating an instance of this on +// the heap, keep it in a global storage to reduce fragmentation from having to +// mmap this at the start of every test. +struct TestAllocatorStorage { + static constexpr size_t kMaxSize = std::max({ + sizeof(scudo::Allocator), +#if SCUDO_CAN_USE_PRIMARY64 + sizeof(scudo::Allocator), +#endif + sizeof(scudo::Allocator) + }); + + // To alleviate some problem, let's skip the thread safety analysis here. + static void *get(size_t size) NO_THREAD_SAFETY_ANALYSIS { + CHECK(size <= kMaxSize && + "Allocation size doesn't fit in the allocator storage"); + M.lock(); + return AllocatorStorage; } - void operator delete(void *ptr) { free(ptr); } + static void release(void *ptr) NO_THREAD_SAFETY_ANALYSIS { + M.assertHeld(); + M.unlock(); + ASSERT_EQ(ptr, AllocatorStorage); + } + + static scudo::HybridMutex M; + static uint8_t AllocatorStorage[kMaxSize]; }; +scudo::HybridMutex TestAllocatorStorage::M; +alignas(kMaxAlign) uint8_t TestAllocatorStorage::AllocatorStorage[kMaxSize]; +#else +struct TestAllocatorStorage { + static void *get(size_t size) NO_THREAD_SAFETY_ANALYSIS { + void *p = nullptr; + EXPECT_EQ(0, posix_memalign(&p, kMaxAlign, size)); + return p; + } + static void release(void *ptr) NO_THREAD_SAFETY_ANALYSIS { free(ptr); } +}; +#endif + +template +void *TestAllocator::operator new(size_t size) { + return TestAllocatorStorage::get(size); +} + +template +void TestAllocator::operator delete(void *ptr) { + TestAllocatorStorage::release(ptr); +} template struct ScudoCombinedTest : public Test { ScudoCombinedTest() { @@ -107,15 +166,60 @@ template struct ScudoCombinedTest : public Test { template using ScudoCombinedDeathTest = ScudoCombinedTest; +namespace scudo { +struct TestConditionVariableConfig { + static const bool MaySupportMemoryTagging = true; + template + using TSDRegistryT = + scudo::TSDRegistrySharedT; // Shared, max 8 TSDs. + + struct Primary { + using SizeClassMap = scudo::AndroidSizeClassMap; +#if SCUDO_CAN_USE_PRIMARY64 + static const scudo::uptr RegionSizeLog = 28U; + typedef scudo::u32 CompactPtrT; + static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const scudo::uptr GroupSizeLog = 20U; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; +#else + static const scudo::uptr RegionSizeLog = 18U; + static const scudo::uptr GroupSizeLog = 18U; + typedef scudo::uptr CompactPtrT; +#endif + static const scudo::s32 MinReleaseToOsIntervalMs = 1000; + static const scudo::s32 MaxReleaseToOsIntervalMs = 1000; + static const bool UseConditionVariable = true; +#if SCUDO_LINUX + using ConditionVariableT = scudo::ConditionVariableLinux; +#else + using ConditionVariableT = scudo::ConditionVariableDummy; +#endif + }; +#if SCUDO_CAN_USE_PRIMARY64 + template + using PrimaryT = scudo::SizeClassAllocator64; +#else + template + using PrimaryT = scudo::SizeClassAllocator32; +#endif + + struct Secondary { + template + using CacheT = scudo::MapAllocatorNoCache; + }; + template using SecondaryT = scudo::MapAllocator; +}; +} // namespace scudo + #if SCUDO_FUCHSIA #define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \ - SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidSvelteConfig) \ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, FuchsiaConfig) #else #define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \ - SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidSvelteConfig) \ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, DefaultConfig) \ - SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidConfig) + SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidConfig) \ + SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConditionVariableConfig) #endif #define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE) \ @@ -170,6 +274,7 @@ void ScudoCombinedTest::BasicTest(scudo::uptr SizeLog) { } Allocator->printStats(); + Allocator->printFragmentationInfo(); } #define SCUDO_MAKE_BASIC_TEST(SizeLog) \ @@ -209,7 +314,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroContents) { void *P = Allocator->allocate(Size, Origin, 1U << MinAlignLog, true); EXPECT_NE(P, nullptr); for (scudo::uptr I = 0; I < Size; I++) - ASSERT_EQ((reinterpret_cast(P))[I], 0); + ASSERT_EQ((reinterpret_cast(P))[I], '\0'); memset(P, 0xaa, Size); Allocator->deallocate(P, Origin, Size); } @@ -227,7 +332,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroFill) { void *P = Allocator->allocate(Size, Origin, 1U << MinAlignLog, false); EXPECT_NE(P, nullptr); for (scudo::uptr I = 0; I < Size; I++) - ASSERT_EQ((reinterpret_cast(P))[I], 0); + ASSERT_EQ((reinterpret_cast(P))[I], '\0'); memset(P, 0xaa, Size); Allocator->deallocate(P, Origin, Size); } @@ -286,7 +391,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeIncreasing) { // we preserve the data in the process. scudo::uptr Size = 16; void *P = Allocator->allocate(Size, Origin); - const char Marker = 0xab; + const char Marker = 'A'; memset(P, Marker, Size); while (Size < TypeParam::Primary::SizeClassMap::MaxSize * 4) { void *NewP = Allocator->reallocate(P, Size * 2); @@ -308,7 +413,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeDecreasing) { scudo::uptr Size = TypeParam::Primary::SizeClassMap::MaxSize * 2; const scudo::uptr DataSize = 2048U; void *P = Allocator->allocate(Size, Origin); - const char Marker = 0xab; + const char Marker = 'A'; memset(P, Marker, scudo::Min(Size, DataSize)); while (Size > 1U) { Size /= 2U; @@ -331,7 +436,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, ReallocateSame) { constexpr scudo::uptr ReallocSize = TypeParam::Primary::SizeClassMap::MaxSize - 64; void *P = Allocator->allocate(ReallocSize, Origin); - const char Marker = 0xab; + const char Marker = 'A'; memset(P, Marker, ReallocSize); for (scudo::sptr Delta = -32; Delta < 32; Delta += 8) { const scudo::uptr NewSize = @@ -388,7 +493,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, UseAfterFree) { disableDebuggerdMaybe(); void *P = Allocator->allocate(Size, Origin); Allocator->deallocate(P, Origin); - reinterpret_cast(P)[0] = 0xaa; + reinterpret_cast(P)[0] = 'A'; }, ""); EXPECT_DEATH( @@ -396,7 +501,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, UseAfterFree) { disableDebuggerdMaybe(); void *P = Allocator->allocate(Size, Origin); Allocator->deallocate(P, Origin); - reinterpret_cast(P)[Size - 1] = 0xaa; + reinterpret_cast(P)[Size - 1] = 'A'; }, ""); } @@ -408,15 +513,15 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, DisableMemoryTagging) { if (Allocator->useMemoryTaggingTestOnly()) { // Check that disabling memory tagging works correctly. void *P = Allocator->allocate(2048, Origin); - EXPECT_DEATH(reinterpret_cast(P)[2048] = 0xaa, ""); + EXPECT_DEATH(reinterpret_cast(P)[2048] = 'A', ""); scudo::ScopedDisableMemoryTagChecks NoTagChecks; Allocator->disableMemoryTagging(); - reinterpret_cast(P)[2048] = 0xaa; + reinterpret_cast(P)[2048] = 'A'; Allocator->deallocate(P, Origin); P = Allocator->allocate(2048, Origin); EXPECT_EQ(scudo::untagPointer(P), P); - reinterpret_cast(P)[2048] = 0xaa; + reinterpret_cast(P)[2048] = 'A'; Allocator->deallocate(P, Origin); Allocator->releaseToOS(scudo::ReleaseToOS::Force); @@ -456,6 +561,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, CacheDrain) NO_THREAD_SAFETY_ANALYSIS { bool UnlockRequired; auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); EXPECT_TRUE(!TSD->getCache().isEmpty()); TSD->getCache().drain(); EXPECT_TRUE(TSD->getCache().isEmpty()); @@ -480,6 +586,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ForceCacheDrain) NO_THREAD_SAFETY_ANALYSIS { bool UnlockRequired; auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); EXPECT_TRUE(TSD->getCache().isEmpty()); EXPECT_EQ(TSD->getQuarantineCache().getSize(), 0U); EXPECT_TRUE(Allocator->getQuarantine()->isEmpty()); @@ -723,7 +830,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, DisableMemInit) { for (unsigned I = 0; I != Ptrs.size(); ++I) { Ptrs[I] = Allocator->allocate(Size, Origin, 1U << MinAlignLog, true); for (scudo::uptr J = 0; J < Size; ++J) - ASSERT_EQ((reinterpret_cast(Ptrs[I]))[J], 0); + ASSERT_EQ((reinterpret_cast(Ptrs[I]))[J], '\0'); } } @@ -786,6 +893,7 @@ TEST(ScudoCombinedTest, BasicTrustyConfig) { bool UnlockRequired; auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); TSD->getCache().drain(); Allocator->releaseToOS(scudo::ReleaseToOS::Force); @@ -793,41 +901,3 @@ TEST(ScudoCombinedTest, BasicTrustyConfig) { #endif #endif - -#if SCUDO_LINUX - -SCUDO_TYPED_TEST(ScudoCombinedTest, SoftRssLimit) { - auto *Allocator = this->Allocator.get(); - Allocator->setRssLimitsTestOnly(1, 0, true); - - size_t Megabyte = 1024 * 1024; - size_t ChunkSize = 16; - size_t Error = 256; - - std::vector Ptrs; - for (size_t index = 0; index < Megabyte + Error; index += ChunkSize) { - void *Ptr = Allocator->allocate(ChunkSize, Origin); - Ptrs.push_back(Ptr); - } - - EXPECT_EQ(nullptr, Allocator->allocate(ChunkSize, Origin)); - - for (void *Ptr : Ptrs) - Allocator->deallocate(Ptr, Origin); -} - -SCUDO_TYPED_TEST(ScudoCombinedTest, HardRssLimit) { - auto *Allocator = this->Allocator.get(); - Allocator->setRssLimitsTestOnly(0, 1, false); - - size_t Megabyte = 1024 * 1024; - - EXPECT_DEATH( - { - disableDebuggerdMaybe(); - Allocator->allocate(Megabyte, Origin); - }, - ""); -} - -#endif diff --git a/Telegram/ThirdParty/scudo/tests/common_test.cpp b/Telegram/ThirdParty/scudo/tests/common_test.cpp index b1e55e80d..fff7c662a 100644 --- a/Telegram/ThirdParty/scudo/tests/common_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/common_test.cpp @@ -72,27 +72,4 @@ TEST(ScudoCommonTest, Zeros) { MemMap.unmap(MemMap.getBase(), Size); } -#if 0 -// This test is temorarily disabled because it may not work as expected. E.g., -// it doesn't dirty the pages so the pages may not be commited and it may only -// work on the single thread environment. As a result, this test is flaky and is -// impacting many test scenarios. -TEST(ScudoCommonTest, GetRssFromBuffer) { - constexpr int64_t AllocSize = 10000000; - constexpr int64_t Error = 3000000; - constexpr size_t Runs = 10; - - int64_t Rss = scudo::GetRSS(); - EXPECT_GT(Rss, 0); - - std::vector> Allocs(Runs); - for (auto &Alloc : Allocs) { - Alloc.reset(new char[AllocSize]()); - int64_t Prev = Rss; - Rss = scudo::GetRSS(); - EXPECT_LE(std::abs(Rss - AllocSize - Prev), Error); - } -} -#endif - } // namespace scudo diff --git a/Telegram/ThirdParty/scudo/tests/condition_variable_test.cpp b/Telegram/ThirdParty/scudo/tests/condition_variable_test.cpp new file mode 100644 index 000000000..caba1f64a --- /dev/null +++ b/Telegram/ThirdParty/scudo/tests/condition_variable_test.cpp @@ -0,0 +1,59 @@ +//===-- condition_variable_test.cpp -----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "tests/scudo_unit_test.h" + +#include "common.h" +#include "condition_variable.h" +#include "mutex.h" + +#include + +template void simpleWaitAndNotifyAll() { + constexpr scudo::u32 NumThreads = 2; + constexpr scudo::u32 CounterMax = 1024; + std::thread Threads[NumThreads]; + + scudo::HybridMutex M; + ConditionVariableT CV; + CV.bindTestOnly(M); + scudo::u32 Counter = 0; + + for (scudo::u32 I = 0; I < NumThreads; ++I) { + Threads[I] = std::thread( + [&](scudo::u32 Id) { + do { + scudo::ScopedLock L(M); + if (Counter % NumThreads != Id && Counter < CounterMax) + CV.wait(M); + if (Counter >= CounterMax) { + break; + } else { + ++Counter; + CV.notifyAll(M); + } + } while (true); + }, + I); + } + + for (std::thread &T : Threads) + T.join(); + + EXPECT_EQ(Counter, CounterMax); +} + +TEST(ScudoConditionVariableTest, DummyCVWaitAndNotifyAll) { + simpleWaitAndNotifyAll(); +} + +#ifdef SCUDO_LINUX +TEST(ScudoConditionVariableTest, LinuxCVWaitAndNotifyAll) { + simpleWaitAndNotifyAll(); +} +#endif diff --git a/Telegram/ThirdParty/scudo/tests/memtag_test.cpp b/Telegram/ThirdParty/scudo/tests/memtag_test.cpp index 2488e0485..fd277f962 100644 --- a/Telegram/ThirdParty/scudo/tests/memtag_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/memtag_test.cpp @@ -120,7 +120,15 @@ TEST_F(MemtagTest, SelectRandomTag) { uptr Tags = 0; for (uptr I = 0; I < 100000; ++I) Tags = Tags | (1u << extractTag(selectRandomTag(Ptr, 0))); - EXPECT_EQ(0xfffeull, Tags); + // std::popcnt is C++20 + int PopCnt = 0; + while (Tags) { + PopCnt += Tags & 1; + Tags >>= 1; + } + // Random tags are not always very random, and this test is not about PRNG + // quality. Anything above half would be satisfactory. + EXPECT_GE(PopCnt, 8); } } diff --git a/Telegram/ThirdParty/scudo/tests/primary_test.cpp b/Telegram/ThirdParty/scudo/tests/primary_test.cpp index 6e580e4fe..181715117 100644 --- a/Telegram/ThirdParty/scudo/tests/primary_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/primary_test.cpp @@ -8,6 +8,8 @@ #include "tests/scudo_unit_test.h" +#include "allocator_config.h" +#include "condition_variable.h" #include "primary32.h" #include "primary64.h" #include "size_class_map.h" @@ -104,6 +106,34 @@ template struct TestConfig4 { }; }; +// This is the only test config that enables the condition variable. +template struct TestConfig5 { + static const bool MaySupportMemoryTagging = true; + + struct Primary { + using SizeClassMap = SizeClassMapT; +#if defined(__mips__) + // Unable to allocate greater size on QEMU-user. + static const scudo::uptr RegionSizeLog = 23U; +#else + static const scudo::uptr RegionSizeLog = 24U; +#endif + static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX; + static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const scudo::uptr GroupSizeLog = 18U; + typedef scudo::u32 CompactPtrT; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; + static const bool UseConditionVariable = true; +#if SCUDO_LINUX + using ConditionVariableT = scudo::ConditionVariableLinux; +#else + using ConditionVariableT = scudo::ConditionVariableDummy; +#endif + }; +}; + template