Move cache database to a separate module.

Also start compactor code.

Telegram/SourceFiles/storage/cache/storage_cache_cleaner.cpp

@@ -19,7 +19,6 @@ namespace details {
 
 class CleanerObject {
 public:
-	using Wrapper = Cache::Cleaner;
 	CleanerObject(
 		crl::weak_on_queue<CleanerObject> weak,
 		const QString &base,
@@ -98,8 +97,6 @@ void CleanerObject::done() {
 	}
 }
 
-} // namespace details
-
 Cleaner::Cleaner(
 	const QString &base,
 	base::binary_guard &&guard,
@@ -109,5 +106,6 @@ Cleaner::Cleaner(
 
 Cleaner::~Cleaner() = default;
 
+} // namespace details
 } // namespace Cache
 } // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_cleaner.h

@@ -13,8 +13,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 namespace Storage {
 namespace Cache {
 namespace details {
+
 class CleanerObject;
-} // namespace details
 
 class Cleaner {
 public:
@@ -31,5 +31,6 @@ private:
 
 };
 
+} // namespace details
 } // namespace Cache
 } // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_compactor.cpp

@@ -0,0 +1,53 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/cache/storage_cache_compactor.h"
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+class CompactorObject {
+public:
+	CompactorObject(
+		crl::weak_on_queue<CompactorObject> weak,
+		const QString &path,
+		crl::weak_on_queue<DatabaseObject> database);
+
+private:
+	void start();
+
+	crl::weak_on_queue<CompactorObject> _weak;
+	crl::weak_on_queue<DatabaseObject> _database;
+	QString _path;
+
+};
+
+CompactorObject::CompactorObject(
+	crl::weak_on_queue<CompactorObject> weak,
+	const QString &path,
+	crl::weak_on_queue<DatabaseObject> database)
+: _weak(std::move(weak))
+, _database(std::move(database))
+, _path(path) {
+	start();
+}
+
+void CompactorObject::start() {
+}
+
+Compactor::Compactor(
+	const QString &path,
+	crl::weak_on_queue<DatabaseObject> database)
+: _wrapped(path, std::move(database)) {
+}
+
+Compactor::~Compactor() = default;
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_compactor.h

@@ -0,0 +1,36 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/cache/storage_cache_types.h"
+#include <crl/crl_object_on_queue.h>
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+class CompactorObject;
+class DatabaseObject;
+
+class Compactor {
+public:
+	Compactor(
+		const QString &path,
+		crl::weak_on_queue<DatabaseObject> database);
+
+	~Compactor();
+
+private:
+	using Implementation = CompactorObject;
+	crl::object_on_queue<Implementation> _wrapped;
+
+};
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_database.h

@@ -17,25 +17,11 @@ namespace Storage {
 class EncryptionKey;
 namespace Cache {
 namespace details {
-class Database;
+class DatabaseObject;
 } // namespace details
 
-struct Key {
-	uint64 high = 0;
-	uint64 low = 0;
-};
-
-inline bool operator==(const Key &a, const Key &b) {
-	return (a.high == b.high) && (a.low == b.low);
-}
-
-inline bool operator!=(const Key &a, const Key &b) {
-	return !(a == b);
-}
-
-inline bool operator<(const Key &a, const Key &b) {
-	return std::tie(a.high, a.low) < std::tie(b.high, b.low);
-}
+using Key = details::Key;
+using Error = details::Error;
 
 class Database {
 public:
@@ -45,6 +31,9 @@ public:
 		size_type maxDataSize = 10 * 1024 * 1024;
 		crl::time_type writeBundleDelay = 15 * 60 * crl::time_type(1000);
 
+		int64 compactAfterExcess = 8 * 1024 * 1024;
+		int64 compactAfterFullSize = 0;
+
 		bool trackEstimatedTime = true;
 		int64 totalSizeLimit = 1024 * 1024 * 1024;
 		size_type totalTimeLimit = 30 * 86400; // One month in seconds.
@@ -66,7 +55,7 @@ public:
 	~Database();
 
 private:
-	using Implementation = details::Database;
+	using Implementation = details::DatabaseObject;
 	crl::object_on_queue<Implementation> _wrapped;
 
 };

Telegram/SourceFiles/storage/cache/storage_cache_database_object.h

@@ -0,0 +1,181 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/cache/storage_cache_database.h"
+#include "storage/storage_encrypted_file.h"
+#include "base/binary_guard.h"
+#include "base/concurrent_timer.h"
+#include "base/bytes.h"
+#include "base/flat_set.h"
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+class Cleaner;
+class Compactor;
+
+class DatabaseObject {
+public:
+	using Settings = Cache::Database::Settings;
+	DatabaseObject(
+		crl::weak_on_queue<DatabaseObject> weak,
+		const QString &path,
+		const Settings &settings);
+
+	void open(EncryptionKey key, FnMut<void(Error)> done);
+	void close(FnMut<void()> done);
+
+	void put(const Key &key, QByteArray value, FnMut<void(Error)> done);
+	void get(const Key &key, FnMut<void(QByteArray)> done);
+	void remove(const Key &key, FnMut<void()> done = nullptr);
+
+	void clear(FnMut<void(Error)> done);
+
+	~DatabaseObject();
+
+private:
+	struct Entry {
+		Entry() = default;
+		Entry(
+			PlaceId place,
+			uint8 tag,
+			uint32 checksum,
+			size_type size,
+			int64 useTime);
+
+		int64 useTime = 0;
+		size_type size = 0;
+		uint32 checksum = 0;
+		PlaceId place = { { 0 } };
+		uint8 tag = 0;
+	};
+	struct CleanerWrap {
+		std::unique_ptr<Cleaner> object;
+		base::binary_guard guard;
+	};
+	using Map = std::unordered_map<Key, Entry>;
+
+	template <typename Callback, typename ...Args>
+	void invokeCallback(Callback &&callback, Args &&...args);
+
+	Error ioError(const QString &path) const;
+
+	QString computePath(Version version) const;
+	QString binlogPath(Version version) const;
+	QString binlogPath() const;
+	QString binlogFilename() const;
+	File::Result openBinlog(
+		Version version,
+		File::Mode mode,
+		EncryptionKey &key);
+	bool readHeader();
+	bool writeHeader();
+
+	void readBinlog();
+	size_type readBinlogRecords(bytes::const_span data);
+	size_type readBinlogRecordSize(bytes::const_span data) const;
+	bool readBinlogRecord(bytes::const_span data);
+	template <typename RecordStore>
+	bool readRecordStoreGeneric(bytes::const_span data);
+	bool readRecordStore(bytes::const_span data);
+	template <typename StorePart>
+	bool readRecordMultiStoreGeneric(bytes::const_span data);
+	bool readRecordMultiStore(bytes::const_span data);
+	bool readRecordMultiRemove(bytes::const_span data);
+	bool readRecordMultiAccess(bytes::const_span data);
+	template <typename RecordStore, typename Postprocess>
+	bool processRecordStoreGeneric(
+		const RecordStore *record,
+		Postprocess &&postprocess);
+	bool processRecordStore(const Store *record, std::is_class<Store>);
+	bool processRecordStore(
+		const StoreWithTime *record,
+		std::is_class<StoreWithTime>);
+	size_type storeRecordSize() const;
+
+	void optimize();
+	void checkCompactor();
+	void adjustRelativeTime();
+	bool startDelayedPruning();
+	int64 countRelativeTime() const;
+	EstimatedTimePoint countTimePoint() const;
+	void applyTimePoint(EstimatedTimePoint time);
+	int64 pruneBeforeTime() const;
+	void prune();
+	void collectTimePrune(
+		base::flat_set<Key> &stale,
+		int64 &staleTotalSize);
+	void collectSizePrune(
+		base::flat_set<Key> &stale,
+		int64 &staleTotalSize);
+
+	void setMapEntry(const Key &key, Entry &&entry);
+	void eraseMapEntry(const Map::const_iterator &i);
+	void recordEntryAccess(const Key &key);
+	QByteArray readValueData(PlaceId place, size_type size) const;
+
+	Version findAvailableVersion() const;
+	QString versionPath() const;
+	bool writeVersion(Version version);
+	Version readVersion() const;
+
+	QString placePath(PlaceId place) const;
+	bool isFreePlace(PlaceId place) const;
+
+	template <typename StoreRecord>
+	base::optional<QString> writeKeyPlaceGeneric(
+		StoreRecord &&record,
+		const Key &key,
+		const QByteArray &value,
+		uint32 checksum);
+	base::optional<QString> writeKeyPlace(
+		const Key &key,
+		const QByteArray &value,
+		uint32 checksum);
+	void writeMultiRemoveLazy();
+	void writeMultiRemove();
+	void writeMultiAccessLazy();
+	void writeMultiAccess();
+	void writeMultiAccessBlock();
+	void writeBundlesLazy();
+	void writeBundles();
+
+	void createCleaner();
+	void cleanerDone(Error error);
+
+	crl::weak_on_queue<DatabaseObject> _weak;
+	QString _base, _path;
+	const Settings _settings;
+	EncryptionKey _key;
+	File _binlog;
+	Map _map;
+	std::set<Key> _removing;
+	std::set<Key> _accessed;
+
+	int64 _relativeTime = 0;
+	int64 _timeCorrection = 0;
+	uint32 _latestSystemTime = 0;
+
+	int64 _binlogExcessLength = 0;
+	int64 _totalSize = 0;
+	int64 _minimalEntryTime = 0;
+	size_type _entriesWithMinimalTimeCount = 0;
+
+	base::ConcurrentTimer _writeBundlesTimer;
+	base::ConcurrentTimer _pruneTimer;
+
+	CleanerWrap _cleaner;
+	std::unique_ptr<Compactor> _compactor;
+
+};
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_types.cpp

@@ -11,6 +11,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 
 namespace Storage {
 namespace Cache {
+namespace details {
 
 QString ComputeBasePath(const QString &original) {
 	const auto result = QDir(original).absolutePath();
@@ -51,5 +52,32 @@ bool WriteVersionValue(const QString &base, Version value) {
 	return file.flush();
 }
 
+BasicHeader::BasicHeader()
+: format(Format::Format_0)
+, flags(0) {
+}
+
+MultiStoreHeader::MultiStoreHeader(size_type count)
+: type(kType)
+, count(ReadTo<RecordsCount>(count)) {
+	Expects(count >= 0 && count < kBundledRecordsLimit);
+}
+
+MultiRemoveHeader::MultiRemoveHeader(size_type count)
+: type(kType)
+, count(ReadTo<RecordsCount>(count)) {
+	Expects(count >= 0 && count < kBundledRecordsLimit);
+}
+
+MultiAccessHeader::MultiAccessHeader(
+	EstimatedTimePoint time,
+	size_type count)
+: type(kType)
+, count(ReadTo<RecordsCount>(count))
+, time(time) {
+	Expects(count >= 0 && count < kBundledRecordsLimit);
+}
+
+} // namespace details
 } // namespace Cache
 } // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_types.h

@@ -12,6 +12,24 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 
 namespace Storage {
 namespace Cache {
+namespace details {
+
+struct Key {
+	uint64 high = 0;
+	uint64 low = 0;
+};
+
+inline bool operator==(const Key &a, const Key &b) {
+	return (a.high == b.high) && (a.low == b.low);
+}
+
+inline bool operator!=(const Key &a, const Key &b) {
+	return !(a == b);
+}
+
+inline bool operator<(const Key &a, const Key &b) {
+	return std::tie(a.high, a.low) < std::tie(b.high, b.low);
+}
 
 using Version = int32;
 
@@ -37,5 +55,143 @@ inline Error Error::NoError() {
 	return Error();
 }
 
+using RecordType = uint8;
+using PlaceId = std::array<uint8, 7>;
+using EntrySize = std::array<uint8, 3>;
+using RecordsCount = std::array<uint8, 3>;
+
+template <typename Packed>
+Packed ReadTo(size_type count) {
+	Expects(count >= 0 && count < (1 << (Packed().size() * 8)));
+
+	auto result = Packed();
+	for (auto &element : result) {
+		element = uint8(count & 0xFF);
+		count >>= 8;
+	}
+	return result;
+}
+
+template <typename Packed>
+size_type ReadFrom(Packed count) {
+	auto result = size_type();
+	for (auto &element : (count | ranges::view::reverse)) {
+		result <<= 8;
+		result |= size_type(element);
+	}
+	return result;
+}
+
+constexpr auto kRecordSizeUnknown = size_type(-1);
+constexpr auto kRecordSizeInvalid = size_type(-2);
+constexpr auto kBundledRecordsLimit = (1 << (RecordsCount().size() * 8));
+constexpr auto kDataSizeLimit = (1 << (EntrySize().size() * 8));
+
+template <typename Record>
+constexpr auto GoodForEncryption = ((sizeof(Record) & 0x0F) == 0);
+
+enum class Format : uint32 {
+	Format_0,
+};
+
+struct BasicHeader {
+	BasicHeader();
+
+	static constexpr auto kTrackEstimatedTime = 0x01U;
+
+	Format format : 8;
+	uint32 flags : 24;
+	uint32 systemTime = 0;
+	uint32 reserved1 = 0;
+	uint32 reserved2 = 0;
+};
+static_assert(GoodForEncryption<BasicHeader>);
+
+struct EstimatedTimePoint {
+	uint32 system = 0;
+	uint32 relativeAdvancement = 0;
+};
+
+struct Store {
+	static constexpr auto kType = RecordType(0x01);
+
+	RecordType type = kType;
+	uint8 tag = 0;
+	EntrySize size = { { 0 } };
+	PlaceId place = { { 0 } };
+	uint32 checksum = 0;
+	Key key;
+};
+static_assert(GoodForEncryption<Store>);
+
+struct StoreWithTime : Store {
+	EstimatedTimePoint time;
+	uint32 reserved1 = 0;
+	uint32 reserved2 = 0;
+};
+static_assert(GoodForEncryption<StoreWithTime>);
+
+struct MultiStoreHeader {
+	static constexpr auto kType = RecordType(0x02);
+
+	explicit MultiStoreHeader(size_type count = 0);
+
+	RecordType type = kType;
+	RecordsCount count = { { 0 } };
+	uint32 reserved1 = 0;
+	uint32 reserved2 = 0;
+	uint32 reserved3 = 0;
+};
+using MultiStorePart = Store;
+using MultiStoreWithTimePart = StoreWithTime;
+static_assert(GoodForEncryption<MultiStoreHeader>);
+
+struct MultiRemoveHeader {
+	static constexpr auto kType = RecordType(0x03);
+
+	explicit MultiRemoveHeader(size_type count = 0);
+
+	RecordType type = kType;
+	RecordsCount count = { { 0 } };
+	uint32 reserved1 = 0;
+	uint32 reserved2 = 0;
+	uint32 reserved3 = 0;
+};
+struct MultiRemovePart {
+	Key key;
+};
+static_assert(GoodForEncryption<MultiRemoveHeader>);
+static_assert(GoodForEncryption<MultiRemovePart>);
+
+struct MultiAccessHeader {
+	static constexpr auto kType = RecordType(0x04);
+
+	explicit MultiAccessHeader(
+		EstimatedTimePoint time,
+		size_type count = 0);
+
+	RecordType type = kType;
+	RecordsCount count = { { 0 } };
+	EstimatedTimePoint time;
+	uint32 reserved = 0;
+};
+struct MultiAccessPart {
+	Key key;
+};
+static_assert(GoodForEncryption<MultiAccessHeader>);
+static_assert(GoodForEncryption<MultiAccessPart>);
+
+} // namespace details
 } // namespace Cache
 } // namespace Storage
+
+namespace std {
+
+template <>
+struct hash<Storage::Cache::details::Key> {
+	size_t operator()(const Storage::Cache::details::Key &key) const {
+		return (hash<uint64>()(key.high) ^ hash<uint64>()(key.low));
+	}
+};
+
+} // namespace std

Telegram/gyp/lib_storage.gyp

@@ -59,8 +59,12 @@
       '<(src_loc)/storage/storage_file_lock.h',
       '<(src_loc)/storage/cache/storage_cache_cleaner.cpp',
       '<(src_loc)/storage/cache/storage_cache_cleaner.h',
+      '<(src_loc)/storage/cache/storage_cache_compactor.cpp',
+      '<(src_loc)/storage/cache/storage_cache_compactor.h',
       '<(src_loc)/storage/cache/storage_cache_database.cpp',
       '<(src_loc)/storage/cache/storage_cache_database.h',
+      '<(src_loc)/storage/cache/storage_cache_database_object.cpp',
+      '<(src_loc)/storage/cache/storage_cache_database_object.h',
       '<(src_loc)/storage/cache/storage_cache_types.cpp',
       '<(src_loc)/storage/cache/storage_cache_types.h',
     ],