Implement round window with Qt-based custom decorations

ui/platform/ui_platform_window.cpp

@@ -13,6 +13,7 @@
 #include "ui/painter.h"
 #include "styles/style_widgets.h"
 #include "styles/style_layers.h"
+#include "styles/palette.h"
 
 #include <QtCore/QCoreApplication>
 #include <QtGui/QWindow>
@@ -26,6 +27,10 @@ namespace {
 	return st::callShadow;
 }
 
+[[nodiscard]] int Radius() {
+	return st::callRadius;
+}
+
 } // namespace
 
 BasicWindowHelper::BasicWindowHelper(not_null<RpWidget*> window)
@@ -181,7 +186,8 @@ void BasicWindowHelper::setupBodyTitleAreaEvents() {
 DefaultWindowHelper::DefaultWindowHelper(not_null<RpWidget*> window)
 : BasicWindowHelper(window)
 , _title(Ui::CreateChild<DefaultTitleWidget>(window.get()))
-, _body(Ui::CreateChild<RpWidget>(window.get())) {
+, _body(Ui::CreateChild<RpWidget>(window.get()))
+, _roundRect(Radius(), st::windowBg) {
 	init();
 }
 
@@ -230,34 +236,27 @@ void DefaultWindowHelper::init() {
 			area.top() + (titleShown ? titleHeight : 0));
 
 		_body->setGeometry(QRect(topLeft, sizeWithoutMargins));
+		updateRoundingOverlay();
 	}, _body->lifetime());
 
 	window()->paintRequest(
-	) | rpl::start_with_next([=] {
-		const auto area = resizeArea();
-
-		if (area.isNull()) {
-			return;
-		}
-
+	) | rpl::filter([=] {
+		return !hasShadow() && !resizeArea().isNull();
+	}) | rpl::start_with_next([=] {
 		Painter p(window());
-
-		if (hasShadow()) {
-			Ui::Shadow::paint(
-				p,
-				QRect(QPoint(), window()->size()).marginsRemoved(area),
-				window()->width(),
-				Shadow());
-		} else {
-			paintBorders(p);
-		}
+		paintBorders(p);
 	}, window()->lifetime());
 
 	rpl::combine(
 		window()->shownValue(),
+		_title->shownValue(),
 		_windowState.value()
-	) | rpl::start_with_next([=](bool shown, Qt::WindowStates windowState) {
+	) | rpl::start_with_next([=](
+			bool shown,
+			bool titleShown,
+			Qt::WindowStates windowState) {
 		if (shown) {
+			window()->windowHandle()->setFlag(Qt::FramelessWindowHint, titleShown);
 			updateWindowExtents();
 		}
 	}, window()->lifetime());
@@ -279,6 +278,52 @@ void DefaultWindowHelper::init() {
 	QCoreApplication::instance()->installEventFilter(this);
 }
 
+void DefaultWindowHelper::updateRoundingOverlay() {
+	if (!hasShadow() || resizeArea().isNull()) {
+		_roundingOverlay.destroy();
+		return;
+	} else if (_roundingOverlay) {
+		return;
+	}
+
+	_roundingOverlay.create(window());
+	_roundingOverlay->setAttribute(Qt::WA_TransparentForMouseEvents);
+	_roundingOverlay->show();
+
+	window()->sizeValue(
+	) | rpl::start_with_next([=](QSize size) {
+		_roundingOverlay->setGeometry(QRect(QPoint(), size));
+	}, _roundingOverlay->lifetime());
+
+	_roundingOverlay->paintRequest(
+	) | rpl::filter([=](QRect clip) {
+		const auto rect = window()->rect().marginsRemoved(resizeArea());
+		const auto radius = Radius();
+		const auto radiusWithFix = radius - 1;
+		const auto radiusSize = QSize(radius, radius);
+		return clip.intersects(QRect(
+				rect.topLeft(),
+				radiusSize
+			)) || clip.intersects(QRect(
+				rect.topRight() - QPoint(radiusWithFix, 0),
+				radiusSize
+			)) || clip.intersects(QRect(
+				rect.bottomRight() - QPoint(0, radiusWithFix),
+				radiusSize
+			)) || clip.intersects(QRect(
+				rect.bottomRight() - QPoint(radiusWithFix, radiusWithFix),
+				radiusSize
+			)) || !rect.contains(clip);
+	}) | rpl::start_with_next([=] {
+		Painter p(_roundingOverlay);
+		const auto rect = window()->rect().marginsRemoved(resizeArea());
+		p.setCompositionMode(QPainter::CompositionMode_DestinationIn);
+		_roundRect.paint(p, rect, RectPart::AllCorners);
+		p.setCompositionMode(QPainter::CompositionMode_DestinationOver);
+		Shadow::paint(p, rect, window()->width(), Shadow());
+	}, _roundingOverlay->lifetime());
+}
+
 not_null<RpWidget*> DefaultWindowHelper::body() {
 	return _body;
 }
@@ -375,9 +420,7 @@ void DefaultWindowHelper::setTitleStyle(const style::WindowTitle &st) {
 }
 
 void DefaultWindowHelper::setNativeFrame(bool enabled) {
-	window()->windowHandle()->setFlag(Qt::FramelessWindowHint, !enabled);
 	_title->setVisible(!enabled);
-	updateWindowExtents();
 }
 
 void DefaultWindowHelper::setMinimumSize(QSize size) {
@@ -441,13 +484,10 @@ void DefaultWindowHelper::paintBorders(QPainter &p) {
 
 void DefaultWindowHelper::updateWindowExtents() {
 	if (hasShadow() && !_title->isHidden()) {
-		Platform::SetWindowExtents(
-			window()->windowHandle(),
-			resizeArea());
-
+		SetWindowExtents(window()->windowHandle(), resizeArea());
 		_extentsSet = true;
 	} else if (_extentsSet) {
-		Platform::UnsetWindowExtents(window()->windowHandle());
+		UnsetWindowExtents(window()->windowHandle());
 		_extentsSet = false;
 	}
 }

ui/platform/ui_platform_window.h

@@ -7,6 +7,8 @@
 #pragma once
 
 #include "base/flags.h"
+#include "base/object_ptr.h"
+#include "ui/round_rect.h"
 
 namespace style {
 struct WindowTitle;
@@ -95,6 +97,7 @@ protected:
 
 private:
 	void init();
+	void updateRoundingOverlay();
 	[[nodiscard]] bool hasShadow() const;
 	[[nodiscard]] QMargins resizeArea() const;
 	[[nodiscard]] Qt::Edges edgesFromPos(const QPoint &pos) const;
@@ -106,6 +109,8 @@ private:
 
 	const not_null<DefaultTitleWidget*> _title;
 	const not_null<RpWidget*> _body;
+	RoundRect _roundRect;
+	object_ptr<RpWidget> _roundingOverlay = { nullptr };
 	bool _extentsSet = false;
 	rpl::variable<Qt::WindowStates> _windowState = Qt::WindowNoState;