schemeshard_impl.h 71 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357
  1. #pragma once
  2. #include "dedicated_pipe_pool.h"
  3. #include "operation_queue_timer.h"
  4. #include "schemeshard.h"
  5. #include "schemeshard_export.h"
  6. #include "schemeshard_import.h"
  7. #include "schemeshard_build_index.h"
  8. #include "schemeshard_private.h"
  9. #include "schemeshard_types.h"
  10. #include "schemeshard_path_element.h"
  11. #include "schemeshard_path.h"
  12. #include "schemeshard_domain_links.h"
  13. #include "schemeshard_info_types.h"
  14. #include "schemeshard_tables_storage.h"
  15. #include "schemeshard_tx_infly.h"
  16. #include "schemeshard_utils.h"
  17. #include "schemeshard_schema.h"
  18. #include "schemeshard__operation.h"
  19. #include "schemeshard__stats.h"
  20. #include <ydb/core/base/hive.h>
  21. #include <ydb/core/base/storage_pools.h>
  22. #include <ydb/core/base/subdomain.h>
  23. #include <ydb/core/base/channel_profiles.h>
  24. #include <ydb/core/base/tx_processing.h>
  25. #include <ydb/core/cms/console/configs_dispatcher.h>
  26. #include <ydb/core/cms/console/console.h>
  27. #include <ydb/core/external_sources/external_source_factory.h>
  28. #include <ydb/core/kesus/tablet/events.h>
  29. #include <ydb/core/persqueue/events/global.h>
  30. #include <ydb/core/protos/blockstore_config.pb.h>
  31. #include <ydb/core/protos/counters_schemeshard.pb.h>
  32. #include <ydb/core/protos/filestore_config.pb.h>
  33. #include <ydb/core/protos/flat_scheme_op.pb.h>
  34. #include <ydb/core/sys_view/common/events.h>
  35. #include <ydb/core/statistics/events.h>
  36. #include <ydb/core/tablet/pipe_tracker.h>
  37. #include <ydb/core/tablet/tablet_counters.h>
  38. #include <ydb/core/tablet/tablet_pipe_client_cache.h>
  39. #include <ydb/core/tablet_flat/flat_cxx_database.h>
  40. #include <ydb/core/tablet_flat/flat_dbase_scheme.h>
  41. #include <ydb/core/tablet_flat/tablet_flat_executed.h>
  42. #include <ydb/core/tx/message_seqno.h>
  43. #include <ydb/core/tx/scheme_board/events.h>
  44. #include <ydb/core/tx/tx_allocator_client/actor_client.h>
  45. #include <ydb/core/tx/replication/controller/public_events.h>
  46. #include <ydb/core/tx/scheme_cache/scheme_cache.h>
  47. #include <ydb/core/tx/sequenceshard/public/events.h>
  48. #include <ydb/core/tx/tx_processing.h>
  49. #include <ydb/core/util/pb.h>
  50. #include <ydb/core/util/token_bucket.h>
  51. #include <ydb/core/ydb_convert/table_profiles.h>
  52. #include <ydb/core/blockstore/core/blockstore.h>
  53. #include <ydb/core/filestore/core/filestore.h>
  54. #include <ydb/library/login/login.h>
  55. #include <ydb/services/bg_tasks/service.h>
  56. #include <util/generic/ptr.h>
  57. namespace NKikimr {
  58. namespace NSchemeShard {
  59. extern const ui64 NEW_TABLE_ALTER_VERSION;
  60. class TSchemeShard
  61. : public TActor<TSchemeShard>
  62. , public NTabletFlatExecutor::TTabletExecutedFlat
  63. , public IQuotaCounters
  64. {
  65. private:
  66. class TPipeClientFactory : public NTabletPipe::IClientFactory {
  67. public:
  68. TPipeClientFactory(TSchemeShard* self)
  69. : Self(self)
  70. { }
  71. TActorId CreateClient(const TActorContext& ctx, ui64 tabletId, const NTabletPipe::TClientConfig& pipeConfig) override;
  72. private:
  73. TSchemeShard* Self;
  74. };
  75. using TCompactionQueue = NOperationQueue::TOperationQueueWithTimer<
  76. TShardCompactionInfo,
  77. TCompactionQueueImpl,
  78. TEvPrivate::EvRunBackgroundCompaction,
  79. NKikimrServices::FLAT_TX_SCHEMESHARD,
  80. NKikimrServices::TActivity::SCHEMESHARD_BACKGROUND_COMPACTION>;
  81. class TCompactionStarter : public TCompactionQueue::IStarter {
  82. public:
  83. TCompactionStarter(TSchemeShard* self)
  84. : Self(self)
  85. { }
  86. NOperationQueue::EStartStatus StartOperation(const TShardCompactionInfo& info) override {
  87. return Self->StartBackgroundCompaction(info);
  88. }
  89. void OnTimeout(const TShardCompactionInfo& info) override {
  90. Self->OnBackgroundCompactionTimeout(info);
  91. }
  92. private:
  93. TSchemeShard* Self;
  94. };
  95. using TBorrowedCompactionQueue = NOperationQueue::TOperationQueueWithTimer<
  96. TShardIdx,
  97. TFifoQueue<TShardIdx>,
  98. TEvPrivate::EvRunBorrowedCompaction,
  99. NKikimrServices::FLAT_TX_SCHEMESHARD,
  100. NKikimrServices::TActivity::SCHEMESHARD_BORROWED_COMPACTION>;
  101. class TBorrowedCompactionStarter : public TBorrowedCompactionQueue::IStarter {
  102. public:
  103. TBorrowedCompactionStarter(TSchemeShard* self)
  104. : Self(self)
  105. { }
  106. NOperationQueue::EStartStatus StartOperation(const TShardIdx& shardIdx) override {
  107. return Self->StartBorrowedCompaction(shardIdx);
  108. }
  109. void OnTimeout(const TShardIdx& shardIdx) override {
  110. Self->OnBorrowedCompactionTimeout(shardIdx);
  111. }
  112. private:
  113. TSchemeShard* Self;
  114. };
  115. public:
  116. static constexpr ui32 DefaultPQTabletPartitionsCount = 1;
  117. static constexpr ui32 MaxPQTabletPartitionsCount = 1000;
  118. static constexpr ui32 MaxPQGroupTabletsCount = 10*1000;
  119. static constexpr ui32 MaxPQGroupPartitionsCount = 20*1000;
  120. static constexpr ui32 MaxPQWriteSpeedPerPartition = 50*1024*1024;
  121. static constexpr ui32 MaxPQLifetimeSeconds = 31 * 86400;
  122. static constexpr ui32 PublishChunkSize = 1000;
  123. static const TSchemeLimits DefaultLimits;
  124. TIntrusivePtr<TChannelProfiles> ChannelProfiles;
  125. TTableProfiles TableProfiles;
  126. bool TableProfilesLoaded = false;
  127. THashSet<std::pair<ui64, ui32>> TableProfilesWaiters;
  128. TControlWrapper AllowConditionalEraseOperations;
  129. TControlWrapper AllowServerlessStorageBilling;
  130. TControlWrapper DisablePublicationsOfDropping;
  131. TControlWrapper FillAllocatePQ;
  132. TSplitSettings SplitSettings;
  133. struct TTenantInitState {
  134. enum EInitState {
  135. InvalidState = 0,
  136. Uninitialized = 1,
  137. Inprogress = 2,
  138. ReadOnlyPreview = 50,
  139. Done = 100,
  140. };
  141. };
  142. TTenantInitState::EInitState InitState = TTenantInitState::InvalidState;
  143. // In RO mode we don't accept any modifications from users but process all in-flight operations in normal way
  144. bool IsReadOnlyMode = false;
  145. bool IsDomainSchemeShard = false;
  146. TPathId ParentDomainId = InvalidPathId;
  147. TString ParentDomainEffectiveACL;
  148. ui64 ParentDomainEffectiveACLVersion = 0;
  149. TEffectiveACL ParentDomainCachedEffectiveACL;
  150. TString ParentDomainOwner;
  151. THashSet<TString> SystemBackupSIDs;
  152. TInstant ServerlessStorageLastBillTime;
  153. TParentDomainLink ParentDomainLink;
  154. TSubDomainsLinks SubDomainsLinks;
  155. TVector<TString> RootPathElements;
  156. ui64 MaxIncompatibleChange = 0;
  157. THashMap<TPathId, TPathElement::TPtr> PathsById;
  158. TLocalPathId NextLocalPathId = 0;
  159. THashMap<TPathId, TTableInfo::TPtr> Tables;
  160. THashMap<TPathId, TTableInfo::TPtr> TTLEnabledTables;
  161. THashMap<TPathId, TTableIndexInfo::TPtr> Indexes;
  162. THashMap<TPathId, TCdcStreamInfo::TPtr> CdcStreams;
  163. THashMap<TPathId, TSequenceInfo::TPtr> Sequences;
  164. THashMap<TPathId, TReplicationInfo::TPtr> Replications;
  165. THashMap<TPathId, TBlobDepotInfo::TPtr> BlobDepots;
  166. THashMap<TPathId, TTxId> TablesWithSnapshots;
  167. THashMap<TTxId, TSet<TPathId>> SnapshotTables;
  168. THashMap<TTxId, TStepId> SnapshotsStepIds;
  169. THashMap<TPathId, TTxId> LockedPaths;
  170. THashMap<TPathId, TTopicInfo::TPtr> Topics;
  171. THashMap<TPathId, TRtmrVolumeInfo::TPtr> RtmrVolumes;
  172. THashMap<TPathId, TSolomonVolumeInfo::TPtr> SolomonVolumes;
  173. THashMap<TPathId, TSubDomainInfo::TPtr> SubDomains;
  174. THashMap<TPathId, TBlockStoreVolumeInfo::TPtr> BlockStoreVolumes;
  175. THashMap<TPathId, TFileStoreInfo::TPtr> FileStoreInfos;
  176. THashMap<TPathId, TKesusInfo::TPtr> KesusInfos;
  177. THashMap<TPathId, TOlapStoreInfo::TPtr> OlapStores;
  178. THashMap<TPathId, TExternalTableInfo::TPtr> ExternalTables;
  179. THashMap<TPathId, TExternalDataSourceInfo::TPtr> ExternalDataSources;
  180. THashMap<TPathId, TViewInfo::TPtr> Views;
  181. TTablesStorage ColumnTables;
  182. // it is only because we need to manage undo of upgrade subdomain, finally remove it
  183. THashMap<TPathId, TVector<TTabletId>> RevertedMigrations;
  184. THashMap<TTxId, TOperation::TPtr> Operations;
  185. THashMap<TTxId, TPublicationInfo> Publications;
  186. THashMap<TOperationId, TTxState> TxInFlight;
  187. ui64 NextLocalShardIdx = 0;
  188. THashMap<TShardIdx, TShardInfo> ShardInfos;
  189. THashMap<TShardIdx, TAdoptedShard> AdoptedShards;
  190. THashMap<TTabletId, TShardIdx> TabletIdToShardIdx;
  191. THashMap<TShardIdx, TVector<TActorId>> ShardDeletionSubscribers; // for tests
  192. // in case of integral hists we need to remember what values we have set
  193. struct TPartitionMetrics {
  194. ui64 SearchHeight = 0;
  195. ui64 RowDeletes = 0;
  196. ui32 HoursSinceFullCompaction = 0;
  197. };
  198. THashMap<TShardIdx, TPartitionMetrics> PartitionMetricsMap;
  199. TActorId SchemeBoardPopulator;
  200. static constexpr ui32 InitiateCachedTxIdsCount = 100;
  201. TDeque<TTxId> CachedTxIds;
  202. TActorId TxAllocatorClient;
  203. TAutoPtr<NTabletPipe::IClientCache> PipeClientCache;
  204. TPipeTracker PipeTracker;
  205. TCompactionStarter CompactionStarter;
  206. TCompactionQueue* CompactionQueue = nullptr;
  207. TBorrowedCompactionStarter BorrowedCompactionStarter;
  208. TBorrowedCompactionQueue* BorrowedCompactionQueue = nullptr;
  209. // shardIdx -> clientId
  210. THashMap<TShardIdx, TActorId> RunningBorrowedCompactions;
  211. THashSet<TShardIdx> ShardsWithBorrowed; // shards have parts from another shards
  212. THashSet<TShardIdx> ShardsWithLoaned; // shards have parts loaned to another shards
  213. bool EnableBackgroundCompaction = false;
  214. bool EnableBackgroundCompactionServerless = false;
  215. bool EnableBorrowedSplitCompaction = false;
  216. bool EnableMoveIndex = true;
  217. bool EnableAlterDatabaseCreateHiveFirst = false;
  218. bool EnablePQConfigTransactionsAtSchemeShard = false;
  219. bool EnableStatistics = false;
  220. bool EnableTablePgTypes = false;
  221. bool EnableServerlessExclusiveDynamicNodes = false;
  222. TShardDeleter ShardDeleter;
  223. // Counter-strike stuff
  224. TTabletCountersBase* TabletCounters = nullptr;
  225. TAutoPtr<TTabletCountersBase> TabletCountersPtr;
  226. TAutoPtr<TSelfPinger> SelfPinger;
  227. TActorId SysPartitionStatsCollector;
  228. TActorId SVPMigrator;
  229. TActorId CdcStreamScanFinalizer;
  230. TDuration StatsMaxExecuteTime;
  231. TDuration StatsBatchTimeout;
  232. ui32 StatsMaxBatchSize = 0;
  233. THashMap<TTxState::ETxType, ui32> InFlightLimits;
  234. // time when we opened the batch
  235. bool TableStatsBatchScheduled = false;
  236. bool TablePersistStatsPending = false;
  237. TStatsQueue<TEvDataShard::TEvPeriodicTableStats> TableStatsQueue;
  238. bool TopicStatsBatchScheduled = false;
  239. bool TopicPersistStatsPending = false;
  240. TStatsQueue<TEvPersQueue::TEvPeriodicTopicStats> TopicStatsQueue;
  241. TSet<TPathId> CleanDroppedPathsCandidates;
  242. TSet<TPathId> CleanDroppedSubDomainsCandidates;
  243. bool CleanDroppedPathsInFly = false;
  244. bool CleanDroppedPathsDisabled = true;
  245. bool CleanDroppedSubDomainsInFly = false;
  246. TTokenBucket DropBlockStoreVolumeRateLimiter;
  247. TActorId DelayedInitTenantDestination;
  248. TAutoPtr<TEvSchemeShard::TEvInitTenantSchemeShardResult> DelayedInitTenantReply;
  249. NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory{NExternalSource::CreateExternalSourceFactory({})};
  250. THolder<TProposeResponse> IgniteOperation(TProposeRequest& request, TOperationContext& context);
  251. THolder<TEvDataShard::TEvProposeTransaction> MakeDataShardProposal(const TPathId& pathId, const TOperationId& opId,
  252. const TString& body, const TActorContext& ctx) const;
  253. TPathId RootPathId() const {
  254. return MakeLocalId(TPathElement::RootPathId);
  255. }
  256. bool IsRootPathId(const TPathId& pId) const {
  257. return pId == RootPathId();
  258. }
  259. bool IsServerlessDomain(TSubDomainInfo::TPtr domainInfo) const {
  260. const auto& resourcesDomainId = domainInfo->GetResourcesDomainId();
  261. return !IsDomainSchemeShard && resourcesDomainId && resourcesDomainId != ParentDomainId;
  262. }
  263. bool IsServerlessDomain(const TPath& domain) const {
  264. return IsServerlessDomain(domain.DomainInfo());
  265. }
  266. bool IsServerlessDomainGlobal(TPathId domainPathId, TSubDomainInfo::TConstPtr domainInfo) const {
  267. const auto& resourcesDomainId = domainInfo->GetResourcesDomainId();
  268. return IsDomainSchemeShard && resourcesDomainId && resourcesDomainId != domainPathId;
  269. }
  270. TPathId MakeLocalId(const TLocalPathId& localPathId) const {
  271. return TPathId(TabletID(), localPathId);
  272. }
  273. TShardIdx MakeLocalId(const TLocalShardIdx& localShardIdx) const {
  274. return TShardIdx(TabletID(), localShardIdx);
  275. }
  276. bool IsLocalId(const TPathId& pathId) const {
  277. return pathId.OwnerId == TabletID();
  278. }
  279. bool IsLocalId(const TShardIdx& shardIdx) const {
  280. return shardIdx.GetOwnerId() == TabletID();
  281. }
  282. TPathId GetCurrentSubDomainPathId() const {
  283. return RootPathId();
  284. }
  285. TPathId PeekNextPathId() const {
  286. return MakeLocalId(NextLocalPathId);
  287. }
  288. TPathId AllocatePathId () {
  289. TPathId next = PeekNextPathId();
  290. ++NextLocalPathId;
  291. return next;
  292. }
  293. TTxId GetCachedTxId(const TActorContext& ctx);
  294. EAttachChildResult AttachChild(TPathElement::TPtr child);
  295. bool PathIsActive(TPathId pathId) const;
  296. // Transient sequence number that monotonically increases within SS tablet generation. It is included in events
  297. // sent from SS to DS and is used for deduplication.
  298. ui64 SchemeOpRound = 1;
  299. TMessageSeqNo StartRound(TTxState& state);// For SS -> DS propose events
  300. TMessageSeqNo NextRound();
  301. void Clear();
  302. void BreakTabletAndRestart(const TActorContext& ctx);
  303. bool IsSchemeShardConfigured() const;
  304. ui64 Generation() const;
  305. void SubscribeConsoleConfigs(const TActorContext& ctx);
  306. void ApplyConsoleConfigs(const NKikimrConfig::TAppConfig& appConfig, const TActorContext& ctx);
  307. void ApplyConsoleConfigs(const NKikimrConfig::TFeatureFlags& featureFlags, const TActorContext& ctx);
  308. void Handle(TEvPrivate::TEvConsoleConfigsTimeout::TPtr& ev, const TActorContext& ctx);
  309. void ConfigureStatsBatching(
  310. const NKikimrConfig::TSchemeShardConfig& config,
  311. const TActorContext &ctx);
  312. void ConfigureStatsOperations(
  313. const NKikimrConfig::TSchemeShardConfig& config,
  314. const TActorContext &ctx);
  315. void ConfigureCompactionQueues(
  316. const NKikimrConfig::TCompactionConfig& config,
  317. const TActorContext &ctx);
  318. void ConfigureBackgroundCompactionQueue(
  319. const NKikimrConfig::TCompactionConfig::TBackgroundCompactionConfig& config,
  320. const TActorContext &ctx);
  321. void ConfigureBorrowedCompactionQueue(
  322. const NKikimrConfig::TCompactionConfig::TBorrowedCompactionConfig& config,
  323. const TActorContext &ctx);
  324. void StartStopCompactionQueues();
  325. void WaitForTableProfiles(ui64 importId, ui32 itemIdx);
  326. void LoadTableProfiles(const NKikimrConfig::TTableProfilesConfig* config, const TActorContext& ctx);
  327. bool ApplyStorageConfig(const TStoragePools& storagePools,
  328. const NKikimrSchemeOp::TStorageConfig& storageConfig,
  329. TChannelsBindings& channelsBinding,
  330. THashMap<TString, ui32>& reverseBinding,
  331. TStorageRoom& room,
  332. TString& errorMsg);
  333. bool GetBindingsRooms(const TPathId domainId,
  334. const NKikimrSchemeOp::TPartitionConfig& partitionConfig,
  335. TVector<TStorageRoom>& rooms,
  336. THashMap<ui32, ui32>& familyRooms,
  337. TChannelsBindings& binding,
  338. TString& errStr);
  339. /**
  340. * For each existing partition generates possible changes to channels
  341. * cand per-shard partition config based on an updated partitionConfig
  342. * for a table in the given domain.
  343. */
  344. bool GetBindingsRoomsChanges(
  345. const TPathId domainId,
  346. const TVector<TTableShardInfo>& partitions,
  347. const NKikimrSchemeOp::TPartitionConfig& partitionConfig,
  348. TBindingsRoomsChanges& changes,
  349. TString& errStr);
  350. /**
  351. * Generates channels bindings for column shards based on the given storage config
  352. */
  353. bool GetOlapChannelsBindings(const TPathId domainId,
  354. const NKikimrSchemeOp::TColumnStorageConfig& channelsConfig,
  355. TChannelsBindings& channelsBindings,
  356. TString& errStr);
  357. bool IsStorageConfigLogic(const TTableInfo::TCPtr tableInfo) const;
  358. bool IsCompatibleChannelProfileLogic(const TPathId domainId, const TTableInfo::TCPtr tableInfo) const;
  359. bool GetChannelsBindings(const TPathId domainId, const TTableInfo::TCPtr tableInfo, TChannelsBindings& binding, TString& errStr) const;
  360. bool ResolveTabletChannels(ui32 profileId, const TPathId domainId, TChannelsBindings& channelsBinding) const;
  361. bool ResolveRtmrChannels(const TPathId domainId, TChannelsBindings& channelsBinding) const;
  362. bool ResolveSolomonChannels(ui32 profileId, const TPathId domainId, TChannelsBindings& channelsBinding) const;
  363. bool ResolveSolomonChannels(const NKikimrSchemeOp::TKeyValueStorageConfig &config, const TPathId domainId, TChannelsBindings& channelsBinding) const;
  364. bool ResolvePqChannels(ui32 profileId, const TPathId domainId, TChannelsBindings& channelsBinding) const;
  365. bool ResolveChannelsByPoolKinds(
  366. const TVector<TStringBuf>& channelPoolKinds,
  367. const TPathId domainId,
  368. TChannelsBindings& channelsBinding) const;
  369. static void SetNbsChannelsParams(
  370. const google::protobuf::RepeatedPtrField<NKikimrBlockStore::TChannelProfile>& ecps,
  371. TChannelsBindings& channelsBinding);
  372. static void SetNfsChannelsParams(
  373. const google::protobuf::RepeatedPtrField<NKikimrFileStore::TChannelProfile>& ecps,
  374. TChannelsBindings& channelsBinding);
  375. static void SetPqChannelsParams(
  376. const google::protobuf::RepeatedPtrField<NKikimrPQ::TChannelProfile>& ecps,
  377. TChannelsBindings& channelsBinding);
  378. bool ResolveSubdomainsChannels(const TStoragePools& storagePools, TChannelsBindings& channelsBinding);
  379. using TChannelResolveDetails = std::function<bool (ui32 profileId,
  380. const TChannelProfiles::TProfile& profile,
  381. const TStoragePools& storagePools,
  382. TChannelsBindings& channelsBinding)>;
  383. bool ResolveChannelCommon(ui32 profileId, const TPathId domainId, TChannelsBindings& channelsBinding, TChannelResolveDetails resolveDetails) const;
  384. static bool ResolveChannelsDetailsAsIs(ui32 /*profileId*/, const TChannelProfiles::TProfile& profile, const TStoragePools& storagePools, TChannelsBindings& channelsBinding);
  385. static bool TabletResolveChannelsDetails(ui32 profileId, const TChannelProfiles::TProfile& profile, const TStoragePools& storagePools, TChannelsBindings& channelsBinding);
  386. void ClearDescribePathCaches(const TPathElement::TPtr node, bool force = false);
  387. TString PathToString(TPathElement::TPtr item);
  388. NKikimrSchemeOp::TPathVersion GetPathVersion(const TPath& pathEl) const;
  389. ui64 GetAliveChildren(TPathElement::TPtr pathEl, const std::optional<TPathElement::EPathType>& type = std::nullopt) const;
  390. const TTableInfo* GetMainTableForIndex(TPathId indexTableId) const;
  391. bool IsBackupTable(TPathId pathId) const;
  392. TPathId ResolvePathIdForDomain(TPathId pathId) const;
  393. TPathId ResolvePathIdForDomain(TPathElement::TPtr pathEl) const;
  394. TSubDomainInfo::TPtr ResolveDomainInfo(TPathId pathId) const;
  395. TSubDomainInfo::TPtr ResolveDomainInfo(TPathElement::TPtr pathEl) const;
  396. TPathId GetDomainKey(TPathElement::TPtr pathEl) const;
  397. TPathId GetDomainKey(TPathId pathId) const;
  398. const NKikimrSubDomains::TProcessingParams& SelectProcessingParams(TPathId id) const;
  399. const NKikimrSubDomains::TProcessingParams& SelectProcessingParams(TPathElement::TPtr pathEl) const;
  400. TTabletId SelectCoordinator(TTxId txId, TPathId pathId) const;
  401. TTabletId SelectCoordinator(TTxId txId, TPathElement::TPtr pathEl) const;
  402. bool CheckApplyIf(const NKikimrSchemeOp::TModifyScheme& scheme, TString& errStr);
  403. bool CheckLocks(const TPathId pathId, const TTxId lockTxId, TString& errStr) const;
  404. bool CheckLocks(const TPathId pathId, const NKikimrSchemeOp::TModifyScheme& scheme, TString& errStr) const;
  405. bool CheckInFlightLimit(TTxState::ETxType txType, TString& errStr) const;
  406. bool CheckInFlightLimit(NKikimrSchemeOp::EOperationType opType, TString& errStr) const;
  407. bool CanCreateSnapshot(const TPathId& tablePathId, TTxId txId, NKikimrScheme::EStatus& status, TString& errStr) const;
  408. TShardIdx ReserveShardIdxs(ui64 count);
  409. TShardIdx NextShardIdx(const TShardIdx& shardIdx, ui64 inc) const;
  410. template <typename T>
  411. TShardIdx RegisterShardInfo(T&& shardInfo) {
  412. return RegisterShardInfo(ReserveShardIdxs(1), std::forward<T>(shardInfo));
  413. }
  414. template <typename T>
  415. TShardIdx RegisterShardInfo(const TShardIdx& shardIdx, T&& shardInfo) {
  416. Y_ABORT_UNLESS(shardIdx.GetOwnerId() == TabletID());
  417. const auto localId = ui64(shardIdx.GetLocalId());
  418. Y_VERIFY_S(localId < NextLocalShardIdx, "shardIdx: " << shardIdx << " NextLocalShardIdx: " << NextLocalShardIdx);
  419. Y_VERIFY_S(!ShardInfos.contains(shardIdx), "shardIdx: " << shardIdx << " already registered");
  420. IncrementPathDbRefCount(shardInfo.PathId, "new shard created");
  421. ShardInfos.emplace(shardIdx, std::forward<T>(shardInfo));
  422. return shardIdx;
  423. }
  424. TTxState& CreateTx(TOperationId opId, TTxState::ETxType txType, TPathId targetPath, TPathId sourcePath = InvalidPathId);
  425. TTxState* FindTx(TOperationId opId);
  426. TTxState* FindTxSafe(TOperationId opId, const TTxState::ETxType& txType);
  427. void RemoveTx(const TActorContext &ctx, NIceDb::TNiceDb& db, TOperationId opId, TTxState* txState);
  428. static TPathElement::EPathState CalcPathState(TTxState::ETxType txType, TPathElement::EPathState oldState);
  429. TMaybe<NKikimrSchemeOp::TPartitionConfig> GetTablePartitionConfigWithAlterData(TPathId pathId) const;
  430. void DeleteSplitOp(TOperationId txId, TTxState& txState);
  431. bool ShardIsUnderSplitMergeOp(const TShardIdx& idx) const;
  432. THashSet<TShardIdx> CollectAllShards(const THashSet<TPathId>& paths) const;
  433. void ExamineTreeVFS(TPathId nodeId, std::function<void(TPathElement::TPtr)> func, const TActorContext& ctx);
  434. THashSet<TPathId> ListSubTree(TPathId subdomain_root, const TActorContext& ctx);
  435. THashSet<TTxId> GetRelatedTransactions(const THashSet<TPathId>& paths, const TActorContext &ctx);
  436. void MarkAsDropping(TPathElement::TPtr node, TTxId txId, const TActorContext& ctx);
  437. void MarkAsDropping(const THashSet<TPathId>& paths, TTxId txId, const TActorContext& ctx);
  438. void UncountNode(TPathElement::TPtr node);
  439. void MarkAsMigrated(TPathElement::TPtr node, const TActorContext& ctx);
  440. void DropNode(TPathElement::TPtr node, TStepId step, TTxId txId, NIceDb::TNiceDb& db, const TActorContext& ctx);
  441. void DropPaths(const THashSet<TPathId>& paths, TStepId step, TTxId txId, NIceDb::TNiceDb& db, const TActorContext& ctx);
  442. void DoShardsDeletion(const THashSet<TShardIdx>& shardIdx, const TActorContext& ctx);
  443. void SetPartitioning(TPathId pathId, const TVector<TShardIdx>& partitioning);
  444. void SetPartitioning(TPathId pathId, TOlapStoreInfo::TPtr storeInfo);
  445. void SetPartitioning(TPathId pathId, TColumnTableInfo::TPtr tableInfo);
  446. void SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, TVector<TTableShardInfo>&& newPartitioning);
  447. auto BuildStatsForCollector(TPathId tableId, TShardIdx shardIdx, TTabletId datashardId,
  448. TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TPartitionStats& stats);
  449. bool ReadSysValue(NIceDb::TNiceDb& db, ui64 sysTag, TString& value, TString defValue = TString());
  450. bool ReadSysValue(NIceDb::TNiceDb& db, ui64 sysTag, ui64& value, ui64 defVal = 0);
  451. void IncrementPathDbRefCount(const TPathId& pathId, const TStringBuf& debug = TStringBuf());
  452. void DecrementPathDbRefCount(const TPathId& pathId, const TStringBuf& debug = TStringBuf());
  453. // incompatible changes
  454. void BumpIncompatibleChanges(NIceDb::TNiceDb& db, ui64 incompatibleChange);
  455. // path
  456. void PersistPath(NIceDb::TNiceDb& db, const TPathId& pathId);
  457. void PersistRemovePath(NIceDb::TNiceDb& db, const TPathElement::TPtr path);
  458. void PersistLastTxId(NIceDb::TNiceDb& db, const TPathElement::TPtr path);
  459. void PersistPathDirAlterVersion(NIceDb::TNiceDb& db, const TPathElement::TPtr path);
  460. void PersistACL(NIceDb::TNiceDb& db, const TPathElement::TPtr path);
  461. void PersistOwner(NIceDb::TNiceDb& db, const TPathElement::TPtr path);
  462. void PersistCreateTxId(NIceDb::TNiceDb& db, const TPathId pathId, TTxId txId);
  463. void PersistCreateStep(NIceDb::TNiceDb& db, const TPathId pathId, TStepId step);
  464. void PersistDropStep(NIceDb::TNiceDb& db, const TPathId pathId, TStepId step, TOperationId opId);
  465. // user attrs
  466. void ApplyAndPersistUserAttrs(NIceDb::TNiceDb& db, const TPathId& pathId);
  467. void PersistUserAttributes(NIceDb::TNiceDb& db, TPathId pathId, TUserAttributes::TPtr oldAttrs, TUserAttributes::TPtr alterAttrs);
  468. void PersistAlterUserAttributes(NIceDb::TNiceDb& db, TPathId pathId);
  469. // table index
  470. void PersistTableIndex(NIceDb::TNiceDb& db, const TPathId& pathId);
  471. void PersistTableIndexAlterData(NIceDb::TNiceDb& db, const TPathId& pathId);
  472. // cdc stream
  473. void PersistCdcStream(NIceDb::TNiceDb& db, const TPathId& pathId);
  474. void PersistCdcStreamAlterData(NIceDb::TNiceDb& db, const TPathId& pathId);
  475. void PersistRemoveCdcStream(NIceDb::TNiceDb& db, const TPathId& tableId);
  476. static void PersistTxMinStep(NIceDb::TNiceDb& db, const TOperationId opId, TStepId minStep);
  477. void PersistRemoveTx(NIceDb::TNiceDb& db, const TOperationId opId, const TTxState& txState);
  478. void PersistTable(NIceDb::TNiceDb &db, const TPathId pathId);
  479. void PersistChannelsBinding(NIceDb::TNiceDb& db, const TShardIdx shardId, const TChannelsBindings& bindedChannels);
  480. void PersistTablePartitioning(NIceDb::TNiceDb &db, const TPathId pathId, const TTableInfo::TPtr tableInfo);
  481. void PersistTablePartitioningDeletion(NIceDb::TNiceDb& db, const TPathId tableId, const TTableInfo::TPtr tableInfo);
  482. void PersistTablePartitionCondErase(NIceDb::TNiceDb& db, const TPathId& pathId, ui64 id, const TTableInfo::TPtr tableInfo);
  483. void PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, ui64 partitionId, const TPartitionStats& stats);
  484. void PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, const TShardIdx& shardIdx, const TTableInfo::TPtr tableInfo);
  485. void PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, const TTableInfo::TPtr tableInfo);
  486. void PersistTableCreated(NIceDb::TNiceDb& db, const TPathId tableId);
  487. void PersistTableAlterVersion(NIceDb::TNiceDb &db, const TPathId pathId, const TTableInfo::TPtr tableInfo);
  488. void PersistTableAltered(NIceDb::TNiceDb &db, const TPathId pathId, const TTableInfo::TPtr tableInfo);
  489. void PersistAddAlterTable(NIceDb::TNiceDb& db, TPathId pathId, const TTableInfo::TAlterDataPtr alter);
  490. void PersistPersQueueGroup(NIceDb::TNiceDb &db, TPathId pathId, const TTopicInfo::TPtr);
  491. void PersistPersQueueGroupStats(NIceDb::TNiceDb &db, const TPathId pathId, const TTopicStats& stats);
  492. void PersistRemovePersQueueGroup(NIceDb::TNiceDb &db, TPathId pathId);
  493. void PersistAddPersQueueGroupAlter(NIceDb::TNiceDb &db, TPathId pathId, const TTopicInfo::TPtr);
  494. void PersistRemovePersQueueGroupAlter(NIceDb::TNiceDb &db, TPathId pathId);
  495. void PersistPersQueue(NIceDb::TNiceDb &db, TPathId pathId, TShardIdx shardIdx, const TTopicTabletInfo::TTopicPartitionInfo& pqInfo);
  496. void PersistRemovePersQueue(NIceDb::TNiceDb &db, TPathId pathId, ui32 pqId);
  497. void PersistRtmrVolume(NIceDb::TNiceDb &db, TPathId pathId, const TRtmrVolumeInfo::TPtr rtmrVol);
  498. void PersistRemoveRtmrVolume(NIceDb::TNiceDb &db, TPathId pathId);
  499. void PersistSolomonVolume(NIceDb::TNiceDb &db, TPathId pathId, const TSolomonVolumeInfo::TPtr rtmrVol);
  500. void PersistRemoveSolomonVolume(NIceDb::TNiceDb &db, TPathId pathId);
  501. void PersistAlterSolomonVolume(NIceDb::TNiceDb &db, TPathId pathId, const TSolomonVolumeInfo::TPtr rtmrVol);
  502. static void PersistAddTxDependency(NIceDb::TNiceDb& db, const TTxId parentOpId, TTxId txId);
  503. static void PersistRemoveTxDependency(NIceDb::TNiceDb& db, TTxId opId, TTxId dependentOpId);
  504. void PersistUpdateTxShard(NIceDb::TNiceDb& db, TOperationId txId, TShardIdx shardIdx, ui32 operation);
  505. void PersistRemoveTxShard(NIceDb::TNiceDb& db, TOperationId txId, TShardIdx shardIdx);
  506. void PersistShardMapping(NIceDb::TNiceDb& db, TShardIdx shardIdx, TTabletId tabletId, TPathId pathId, TTxId txId, TTabletTypes::EType type);
  507. void PersistAdoptedShardMapping(NIceDb::TNiceDb& db, TShardIdx shardIdx, TTabletId tabletId, ui64 prevOwner, TLocalShardIdx prevShardIdx);
  508. void PersistShardPathId(NIceDb::TNiceDb& db, TShardIdx shardIdx, TPathId pathId);
  509. void PersistDeleteAdopted(NIceDb::TNiceDb& db, TShardIdx shardIdx);
  510. void PersistSnapshotTable(NIceDb::TNiceDb& db, const TTxId snapshotId, const TPathId tableId);
  511. void PersistSnapshotStepId(NIceDb::TNiceDb& db, const TTxId snapshotId, const TStepId stepId);
  512. void PersistDropSnapshot(NIceDb::TNiceDb& db, const TTxId snapshotId, const TPathId tableId);
  513. void PersistLongLock(NIceDb::TNiceDb& db, const TTxId lockId, const TPathId pathId);
  514. void PersistUnLock(NIceDb::TNiceDb& db, const TPathId pathId);
  515. void PersistTxState(NIceDb::TNiceDb& db, const TOperationId opId);
  516. void ChangeTxState(NIceDb::TNiceDb& db, const TOperationId opId, TTxState::ETxState newState);
  517. void PersistCancelTx(NIceDb::TNiceDb& db, const TOperationId opId, const TTxState& txState);
  518. void PersistTxPlanStep(NIceDb::TNiceDb& db, TOperationId opId, TStepId step);
  519. void PersistShardTx(NIceDb::TNiceDb& db, TShardIdx shardIdx, TTxId txId);
  520. void PersistUpdateNextPathId(NIceDb::TNiceDb& db) const;
  521. void PersistUpdateNextShardIdx(NIceDb::TNiceDb& db) const;
  522. void PersistParentDomain(NIceDb::TNiceDb& db, TPathId parentDomain) const;
  523. void PersistParentDomainEffectiveACL(NIceDb::TNiceDb& db, const TString& owner, const TString& effectiveACL, ui64 effectiveACLVersion) const;
  524. void PersistShardsToDelete(NIceDb::TNiceDb& db, const THashSet<TShardIdx>& shardsIdxs);
  525. void PersistShardDeleted(NIceDb::TNiceDb& db, TShardIdx shardIdx, const TChannelsBindings& bindedChannels);
  526. void PersistUnknownShardDeleted(NIceDb::TNiceDb& db, TShardIdx shardIdx);
  527. void PersistTxShardStatus(NIceDb::TNiceDb& db, TOperationId opId, TShardIdx shardIdx, const TTxState::TShardStatus& status);
  528. void PersistBackupSettings(NIceDb::TNiceDb& db, TPathId pathId, const NKikimrSchemeOp::TBackupTask& settings);
  529. void PersistBackupDone(NIceDb::TNiceDb& db, TPathId pathId);
  530. void PersistCompletedBackupRestore(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& info, TTableInfo::TBackupRestoreResult::EKind kind);
  531. void PersistCompletedBackup(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& backupInfo);
  532. void PersistCompletedRestore(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& restoreInfo);
  533. void PersistSchemeLimit(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  534. void PersistStoragePools(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  535. void PersistSubDomain(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  536. void PersistRemoveSubDomain(NIceDb::TNiceDb& db, const TPathId& pathId);
  537. void PersistSubDomainVersion(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  538. void PersistSubDomainAlter(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  539. void PersistSubDomainDeclaredSchemeQuotas(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  540. void PersistSubDomainDatabaseQuotas(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  541. void PersistSubDomainState(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  542. void PersistSubDomainSchemeQuotas(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  543. void PersistSubDomainSecurityStateVersion(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  544. void PersistSubDomainPrivateShards(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  545. void PersistDeleteSubDomainAlter(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  546. void PersistSubDomainAuditSettings(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  547. void PersistSubDomainAuditSettingsAlter(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  548. void PersistSubDomainServerlessComputeResourcesMode(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  549. void PersistSubDomainServerlessComputeResourcesModeAlter(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain);
  550. void PersistKesusInfo(NIceDb::TNiceDb& db, TPathId pathId, const TKesusInfo::TPtr);
  551. void PersistKesusVersion(NIceDb::TNiceDb& db, TPathId pathId, const TKesusInfo::TPtr);
  552. void PersistAddKesusAlter(NIceDb::TNiceDb& db, TPathId pathId, const TKesusInfo::TPtr);
  553. void PersistRemoveKesusAlter(NIceDb::TNiceDb& db, TPathId pathId);
  554. void PersistRemoveKesusInfo(NIceDb::TNiceDb& db, TPathId pathId);
  555. void PersistRemoveTableIndex(NIceDb::TNiceDb& db, TPathId tableId);
  556. void PersistRemoveTable(NIceDb::TNiceDb& db, TPathId tableId, const TActorContext& ctx);
  557. void PersistRevertedMigration(NIceDb::TNiceDb& db, TPathId pathId, TTabletId abandonedSchemeShardId);
  558. // BlockStore
  559. void PersistBlockStorePartition(NIceDb::TNiceDb& db, TPathId pathId, ui32 partitionId, TShardIdx shardIdx, ui64 version);
  560. void PersistBlockStoreVolume(NIceDb::TNiceDb& db, TPathId pathId, const TBlockStoreVolumeInfo::TPtr);
  561. void PersistBlockStoreVolumeMountToken(NIceDb::TNiceDb& db, TPathId pathId, const TBlockStoreVolumeInfo::TPtr volume);
  562. void PersistAddBlockStoreVolumeAlter(NIceDb::TNiceDb& db, TPathId pathId, const TBlockStoreVolumeInfo::TPtr);
  563. void PersistRemoveBlockStoreVolumeAlter(NIceDb::TNiceDb& db, TPathId pathId);
  564. void PersistRemoveBlockStorePartition(NIceDb::TNiceDb& db, TPathId pathId, ui32 partitionId);
  565. void PersistRemoveBlockStoreVolume(NIceDb::TNiceDb& db, TPathId pathId);
  566. // FileStore
  567. void PersistFileStoreInfo(NIceDb::TNiceDb& db, TPathId pathId, const TFileStoreInfo::TPtr);
  568. void PersistAddFileStoreAlter(NIceDb::TNiceDb& db, TPathId pathId, const TFileStoreInfo::TPtr);
  569. void PersistRemoveFileStoreAlter(NIceDb::TNiceDb& db, TPathId pathId);
  570. void PersistRemoveFileStoreInfo(NIceDb::TNiceDb& db, TPathId pathId);
  571. // OlapStore
  572. void PersistOlapStore(NIceDb::TNiceDb& db, TPathId pathId, const TOlapStoreInfo& storeInfo, bool isAlter = false);
  573. void PersistOlapStoreRemove(NIceDb::TNiceDb& db, TPathId pathId, bool isAlter = false);
  574. void PersistOlapStoreAlter(NIceDb::TNiceDb& db, TPathId pathId, const TOlapStoreInfo& storeInfo);
  575. void PersistOlapStoreAlterRemove(NIceDb::TNiceDb& db, TPathId pathId);
  576. // ColumnTable
  577. void PersistColumnTable(NIceDb::TNiceDb& db, TPathId pathId, const TColumnTableInfo& tableInfo, bool isAlter = false);
  578. void PersistColumnTableRemove(NIceDb::TNiceDb& db, TPathId pathId);
  579. void PersistColumnTableAlter(NIceDb::TNiceDb& db, TPathId pathId, const TColumnTableInfo& tableInfo);
  580. void PersistColumnTableAlterRemove(NIceDb::TNiceDb& db, TPathId pathId);
  581. // Sequence
  582. void PersistSequence(NIceDb::TNiceDb& db, TPathId pathId, const TSequenceInfo& sequenceInfo);
  583. void PersistSequenceRemove(NIceDb::TNiceDb& db, TPathId pathId);
  584. void PersistSequenceAlter(NIceDb::TNiceDb& db, TPathId pathId, const TSequenceInfo& sequenceInfo);
  585. void PersistSequenceAlterRemove(NIceDb::TNiceDb& db, TPathId pathId);
  586. // Replication
  587. void PersistReplication(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo);
  588. void PersistReplicationRemove(NIceDb::TNiceDb& db, TPathId pathId);
  589. void PersistReplicationAlter(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo);
  590. void PersistReplicationAlterRemove(NIceDb::TNiceDb& db, TPathId pathId);
  591. // BlobDepot
  592. void PersistBlobDepot(NIceDb::TNiceDb& db, TPathId pathId, const TBlobDepotInfo& blobDepotInfo);
  593. void PersistAddTableShardPartitionConfig(NIceDb::TNiceDb& db, TShardIdx shardIdx, const NKikimrSchemeOp::TPartitionConfig& config);
  594. void PersistPublishingPath(NIceDb::TNiceDb& db, TTxId txId, TPathId pathId, ui64 version);
  595. void PersistRemovePublishingPath(NIceDb::TNiceDb& db, TTxId txId, TPathId pathId, ui64 version);
  596. void PersistInitState(NIceDb::TNiceDb& db);
  597. void PersistStorageBillingTime(NIceDb::TNiceDb& db);
  598. // ExternalTable
  599. void PersistExternalTable(NIceDb::TNiceDb &db, TPathId pathId, const TExternalTableInfo::TPtr externalTable);
  600. void PersistRemoveExternalTable(NIceDb::TNiceDb& db, TPathId pathId);
  601. // ExternalDataSource
  602. void PersistExternalDataSource(NIceDb::TNiceDb &db, TPathId pathId, const TExternalDataSourceInfo::TPtr externalDataSource);
  603. void PersistRemoveExternalDataSource(NIceDb::TNiceDb& db, TPathId pathId);
  604. void PersistView(NIceDb::TNiceDb &db, TPathId pathId);
  605. void PersistRemoveView(NIceDb::TNiceDb& db, TPathId pathId);
  606. TTabletId GetGlobalHive(const TActorContext& ctx) const;
  607. enum class EHiveSelection : uint8_t {
  608. ANY,
  609. IGNORE_TENANT,
  610. };
  611. TTabletId ResolveHive(TPathId pathId, const TActorContext& ctx, EHiveSelection selection) const;
  612. TTabletId ResolveHive(TPathId pathId, const TActorContext& ctx) const;
  613. TTabletId ResolveHive(TShardIdx shardIdx, const TActorContext& ctx) const;
  614. TShardIdx GetShardIdx(TTabletId tabletId) const;
  615. TShardIdx MustGetShardIdx(TTabletId tabletId) const;
  616. TTabletTypes::EType GetTabletType(TTabletId tabletId) const;
  617. struct TTxMonitoring;
  618. //OnRenderAppHtmlPage
  619. struct TTxInit;
  620. NTabletFlatExecutor::ITransaction* CreateTxInit();
  621. struct TTxInitRoot;
  622. NTabletFlatExecutor::ITransaction* CreateTxInitRoot();
  623. struct TTxInitRootCompatibility;
  624. NTabletFlatExecutor::ITransaction* CreateTxInitRootCompatibility(TEvSchemeShard::TEvInitRootShard::TPtr &ev);
  625. struct TTxInitTenantSchemeShard;
  626. NTabletFlatExecutor::ITransaction* CreateTxInitTenantSchemeShard(TEvSchemeShard::TEvInitTenantSchemeShard::TPtr &ev);
  627. struct TActivationOpts {
  628. TSideEffects::TPublications DelayPublications;
  629. TVector<ui64> ExportIds;
  630. TVector<ui64> ImportsIds;
  631. TVector<TPathId> CdcStreamScans;
  632. TVector<TPathId> TablesToClean;
  633. TDeque<TPathId> BlockStoreVolumesToClean;
  634. };
  635. void ActivateAfterInitialization(const TActorContext& ctx, TActivationOpts&& opts);
  636. struct TTxInitPopulator;
  637. NTabletFlatExecutor::ITransaction* CreateTxInitPopulator(TSideEffects::TPublications&& publications);
  638. struct TTxInitSchema;
  639. NTabletFlatExecutor::ITransaction* CreateTxInitSchema();
  640. struct TTxUpgradeSchema;
  641. NTabletFlatExecutor::ITransaction* CreateTxUpgradeSchema();
  642. struct TTxCleanTables;
  643. NTabletFlatExecutor::ITransaction* CreateTxCleanTables(TVector<TPathId> tablesToClean);
  644. struct TTxCleanBlockStoreVolumes;
  645. NTabletFlatExecutor::ITransaction* CreateTxCleanBlockStoreVolumes(TDeque<TPathId>&& blockStoreVolumes);
  646. struct TTxCleanDroppedPaths;
  647. NTabletFlatExecutor::ITransaction* CreateTxCleanDroppedPaths();
  648. void ScheduleCleanDroppedPaths();
  649. void Handle(TEvPrivate::TEvCleanDroppedPaths::TPtr& ev, const TActorContext& ctx);
  650. void EnqueueBackgroundCompaction(const TShardIdx& shardIdx, const TPartitionStats& stats);
  651. void UpdateBackgroundCompaction(const TShardIdx& shardIdx, const TPartitionStats& stats);
  652. void RemoveBackgroundCompaction(const TShardIdx& shardIdx);
  653. void EnqueueBorrowedCompaction(const TShardIdx& shardIdx);
  654. void RemoveBorrowedCompaction(const TShardIdx& shardIdx);
  655. void UpdateShardMetrics(const TShardIdx& shardIdx, const TPartitionStats& newStats);
  656. void RemoveShardMetrics(const TShardIdx& shardIdx);
  657. void ShardRemoved(const TShardIdx& shardIdx);
  658. NOperationQueue::EStartStatus StartBackgroundCompaction(const TShardCompactionInfo& info);
  659. void OnBackgroundCompactionTimeout(const TShardCompactionInfo& info);
  660. void UpdateBackgroundCompactionQueueMetrics();
  661. NOperationQueue::EStartStatus StartBorrowedCompaction(const TShardIdx& shardIdx);
  662. void OnBorrowedCompactionTimeout(const TShardIdx& shardIdx);
  663. void BorrowedCompactionHandleDisconnect(TTabletId tabletId, const TActorId& clientId);
  664. void UpdateBorrowedCompactionQueueMetrics();
  665. struct TTxCleanDroppedSubDomains;
  666. NTabletFlatExecutor::ITransaction* CreateTxCleanDroppedSubDomains();
  667. void ScheduleCleanDroppedSubDomains();
  668. void Handle(TEvPrivate::TEvCleanDroppedSubDomains::TPtr& ev, const TActorContext& ctx);
  669. struct TTxFixBadPaths;
  670. NTabletFlatExecutor::ITransaction* CreateTxFixBadPaths();
  671. struct TTxPublishTenantAsReadOnly;
  672. NTabletFlatExecutor::ITransaction* CreateTxPublishTenantAsReadOnly(TEvSchemeShard::TEvPublishTenantAsReadOnly::TPtr &ev);
  673. struct TTxPublishTenant;
  674. NTabletFlatExecutor::ITransaction* CreateTxPublishTenant(TEvSchemeShard::TEvPublishTenant::TPtr &ev);
  675. struct TTxMigrate;
  676. NTabletFlatExecutor::ITransaction* CreateTxMigrate(TEvSchemeShard::TEvMigrateSchemeShard::TPtr &ev);
  677. struct TTxDescribeScheme;
  678. NTabletFlatExecutor::ITransaction* CreateTxDescribeScheme(TEvSchemeShard::TEvDescribeScheme::TPtr &ev);
  679. struct TTxNotifyCompletion;
  680. NTabletFlatExecutor::ITransaction* CreateTxNotifyTxCompletion(TEvSchemeShard::TEvNotifyTxCompletion::TPtr &ev);
  681. struct TTxDeleteTabletReply;
  682. NTabletFlatExecutor::ITransaction* CreateTxDeleteTabletReply(TEvHive::TEvDeleteTabletReply::TPtr& ev);
  683. struct TTxShardStateChanged;
  684. NTabletFlatExecutor::ITransaction* CreateTxShardStateChanged(TEvDataShard::TEvStateChanged::TPtr& ev);
  685. struct TTxRunConditionalErase;
  686. NTabletFlatExecutor::ITransaction* CreateTxRunConditionalErase(TEvPrivate::TEvRunConditionalErase::TPtr& ev);
  687. struct TTxScheduleConditionalErase;
  688. NTabletFlatExecutor::ITransaction* CreateTxScheduleConditionalErase(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev);
  689. struct TTxSyncTenant;
  690. NTabletFlatExecutor::ITransaction* CreateTxSyncTenant(TPathId tabletId);
  691. struct TTxUpdateTenant;
  692. NTabletFlatExecutor::ITransaction* CreateTxUpdateTenant(TEvSchemeShard::TEvUpdateTenantSchemeShard::TPtr& ev);
  693. struct TTxPublishToSchemeBoard;
  694. NTabletFlatExecutor::ITransaction* CreateTxPublishToSchemeBoard(THashMap<TTxId, TDeque<TPathId>>&& paths);
  695. struct TTxAckPublishToSchemeBoard;
  696. NTabletFlatExecutor::ITransaction* CreateTxAckPublishToSchemeBoard(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev);
  697. struct TTxOperationPropose;
  698. NTabletFlatExecutor::ITransaction* CreateTxOperationPropose(TEvSchemeShard::TEvModifySchemeTransaction::TPtr& ev);
  699. struct TTxOperationProposeCancelTx;
  700. NTabletFlatExecutor::ITransaction* CreateTxOperationPropose(TEvSchemeShard::TEvCancelTx::TPtr& ev);
  701. struct TTxOperationProgress;
  702. NTabletFlatExecutor::ITransaction* CreateTxOperationProgress(TOperationId opId);
  703. struct TTxOperationPlanStep;
  704. NTabletFlatExecutor::ITransaction* CreateTxOperationPlanStep(TEvTxProcessing::TEvPlanStep::TPtr& ev);
  705. struct TTxUpgradeAccessDatabaseRights;
  706. NTabletFlatExecutor::ITransaction* CreateTxUpgradeAccessDatabaseRights(const TActorId& answerTo, bool isDryRun, std::function< NActors::IEventBase* (const TMap<TPathId, TSet<TString>>&) >);
  707. struct TTxMakeAccessDatabaseNoInheritable;
  708. NTabletFlatExecutor::ITransaction* CreateTxMakeAccessDatabaseNoInheritable(const TActorId& answerTo, bool isDryRun, std::function< NActors::IEventBase* (const TMap<TPathId, TSet<TString>>&) >);
  709. struct TTxServerlessStorageBilling;
  710. NTabletFlatExecutor::ITransaction* CreateTxServerlessStorageBilling();
  711. struct TTxLogin;
  712. NTabletFlatExecutor::ITransaction* CreateTxLogin(TEvSchemeShard::TEvLogin::TPtr &ev);
  713. template <EventBasePtr TEvPtr>
  714. NTabletFlatExecutor::ITransaction* CreateTxOperationReply(TOperationId id, TEvPtr& ev);
  715. void PublishToSchemeBoard(THashMap<TTxId, TDeque<TPathId>>&& paths, const TActorContext& ctx);
  716. void PublishToSchemeBoard(TTxId txId, TDeque<TPathId>&& paths, const TActorContext& ctx);
  717. void ApplyPartitionConfigStoragePatch(
  718. NKikimrSchemeOp::TPartitionConfig& config,
  719. const NKikimrSchemeOp::TPartitionConfig& patch) const;
  720. void FillTableDescriptionForShardIdx(
  721. TPathId tableId, TShardIdx shardIdx, NKikimrSchemeOp::TTableDescription* tableDescr,
  722. TString rangeBegin, TString rangeEnd,
  723. bool rangeBeginInclusive, bool rangeEndInclusive,
  724. bool newTable = false);
  725. void FillTableDescription(TPathId tableId, ui32 partitionIdx, ui64 schemaVersion, NKikimrSchemeOp::TTableDescription* tableDescr);
  726. static bool FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 keySize, NScheme::TTypeInfo firstKeyColType,
  727. ui32 partitionCount, const NScheme::TTypeRegistry* typeRegistry, TString& errStr);
  728. static bool FillSplitPartitioning(TVector<TString>& rangeEnds, const TConstArrayRef<NScheme::TTypeInfo>& keyColTypes,
  729. const ::google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TSplitBoundary>& boundaries,
  730. TString& errStr);
  731. TString FillAlterTableTxBody(TPathId tableId, TShardIdx shardIdx, TMessageSeqNo seqNo) const;
  732. TString FillBackupTxBody(TPathId pathId, const NKikimrSchemeOp::TBackupTask& task, ui32 shardNum, TMessageSeqNo seqNo) const;
  733. static void FillSeqNo(NKikimrTxDataShard::TFlatSchemeTransaction &tx, TMessageSeqNo seqNo);
  734. static void FillSeqNo(NKikimrTxColumnShard::TSchemaTxBody &tx, TMessageSeqNo seqNo);
  735. void FillAsyncIndexInfo(const TPathId& tableId, NKikimrTxDataShard::TFlatSchemeTransaction& tx);
  736. void DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme::TTypeRegistry* typeRegistry,
  737. bool fillConfig, bool fillBoundaries, NKikimrSchemeOp::TTableDescription* entry) const;
  738. void DescribeTableIndex(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TIndexDescription& entry);
  739. void DescribeTableIndex(const TPathId& pathId, const TString& name, TTableIndexInfo::TPtr indexInfo, NKikimrSchemeOp::TIndexDescription& entry);
  740. void DescribeCdcStream(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TCdcStreamDescription& desc);
  741. void DescribeCdcStream(const TPathId& pathId, const TString& name, TCdcStreamInfo::TPtr info, NKikimrSchemeOp::TCdcStreamDescription& desc);
  742. void DescribeSequence(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TSequenceDescription& desc);
  743. void DescribeSequence(const TPathId& pathId, const TString& name, TSequenceInfo::TPtr info, NKikimrSchemeOp::TSequenceDescription& desc);
  744. void DescribeReplication(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TReplicationDescription& desc);
  745. void DescribeReplication(const TPathId& pathId, const TString& name, TReplicationInfo::TPtr info, NKikimrSchemeOp::TReplicationDescription& desc);
  746. void DescribeBlobDepot(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TBlobDepotDescription& desc);
  747. static void FillTableBoundaries(const TTableInfo::TPtr tableInfo, google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TSplitBoundary>& boundaries);
  748. void Handle(TEvSchemeShard::TEvInitRootShard::TPtr &ev, const TActorContext &ctx);
  749. void Handle(TEvSchemeShard::TEvInitTenantSchemeShard::TPtr &ev, const TActorContext &ctx);
  750. void Handle(TEvSchemeShard::TEvModifySchemeTransaction::TPtr &ev, const TActorContext &ctx);
  751. void Handle(TEvSchemeShard::TEvDescribeScheme::TPtr &ev, const TActorContext &ctx);
  752. void Handle(TEvSchemeShard::TEvNotifyTxCompletion::TPtr &ev, const TActorContext &ctx);
  753. void Handle(TEvSchemeShard::TEvCancelTx::TPtr& ev, const TActorContext& ctx);
  754. void Handle(TEvPrivate::TEvProgressOperation::TPtr &ev, const TActorContext &ctx);
  755. void Handle(TEvPersQueue::TEvProposeTransactionAttachResult::TPtr& ev, const TActorContext& ctx);
  756. void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx);
  757. void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr &ev, const TActorContext &ctx);
  758. void Handle(TEvTabletPipe::TEvServerConnected::TPtr &ev, const TActorContext &ctx);
  759. void Handle(TEvTabletPipe::TEvServerDisconnected::TPtr &ev, const TActorContext &ctx);
  760. void Handle(TEvHive::TEvCreateTabletReply::TPtr &ev, const TActorContext &ctx);
  761. void Handle(TEvHive::TEvAdoptTabletReply::TPtr &ev, const TActorContext &ctx);
  762. void Handle(TEvHive::TEvDeleteTabletReply::TPtr &ev, const TActorContext &ctx);
  763. void Handle(TEvPrivate::TEvSubscribeToShardDeletion::TPtr &ev, const TActorContext &ctx);
  764. void Handle(TEvHive::TEvDeleteOwnerTabletsReply::TPtr &ev, const TActorContext &ctx);
  765. void Handle(TEvHive::TEvUpdateTabletsObjectReply::TPtr &ev, const TActorContext &ctx);
  766. void Handle(TEvHive::TEvUpdateDomainReply::TPtr &ev, const TActorContext &ctx);
  767. void Handle(TEvPersQueue::TEvDropTabletReply::TPtr &ev, const TActorContext &ctx);
  768. void Handle(TEvColumnShard::TEvProposeTransactionResult::TPtr& ev, const TActorContext& ctx);
  769. void Handle(NBackgroundTasks::TEvAddTaskResult::TPtr& ev, const TActorContext& ctx);
  770. void Handle(TEvColumnShard::TEvNotifyTxCompletionResult::TPtr &ev, const TActorContext &ctx);
  771. void Handle(NSequenceShard::TEvSequenceShard::TEvCreateSequenceResult::TPtr &ev, const TActorContext &ctx);
  772. void Handle(NSequenceShard::TEvSequenceShard::TEvDropSequenceResult::TPtr &ev, const TActorContext &ctx);
  773. void Handle(NSequenceShard::TEvSequenceShard::TEvUpdateSequenceResult::TPtr &ev, const TActorContext &ctx);
  774. void Handle(NSequenceShard::TEvSequenceShard::TEvFreezeSequenceResult::TPtr &ev, const TActorContext &ctx);
  775. void Handle(NSequenceShard::TEvSequenceShard::TEvRestoreSequenceResult::TPtr &ev, const TActorContext &ctx);
  776. void Handle(NSequenceShard::TEvSequenceShard::TEvRedirectSequenceResult::TPtr &ev, const TActorContext &ctx);
  777. void Handle(NReplication::TEvController::TEvCreateReplicationResult::TPtr &ev, const TActorContext &ctx);
  778. void Handle(NReplication::TEvController::TEvDropReplicationResult::TPtr &ev, const TActorContext &ctx);
  779. void Handle(TEvDataShard::TEvProposeTransactionResult::TPtr &ev, const TActorContext &ctx);
  780. void Handle(TEvDataShard::TEvSchemaChanged::TPtr &ev, const TActorContext &ctx);
  781. void Handle(TEvDataShard::TEvStateChanged::TPtr &ev, const TActorContext &ctx);
  782. void Handle(TEvPersQueue::TEvUpdateConfigResponse::TPtr &ev, const TActorContext &ctx);
  783. void Handle(TEvPersQueue::TEvProposeTransactionResult::TPtr& ev, const TActorContext& ctx);
  784. void Handle(TEvBlobDepot::TEvApplyConfigResult::TPtr &ev, const TActorContext &ctx);
  785. void Handle(TEvSubDomain::TEvConfigureStatus::TPtr &ev, const TActorContext &ctx);
  786. void Handle(TEvBlockStore::TEvUpdateVolumeConfigResponse::TPtr& ev, const TActorContext& ctx);
  787. void Handle(TEvFileStore::TEvUpdateConfigResponse::TPtr& ev, const TActorContext& ctx);
  788. void Handle(NKesus::TEvKesus::TEvSetConfigResult::TPtr& ev, const TActorContext& ctx);
  789. void Handle(TEvSchemeShard::TEvInitTenantSchemeShardResult::TPtr& ev, const TActorContext& ctx);
  790. void Handle(TEvSchemeShard::TEvPublishTenantAsReadOnly::TPtr& ev, const TActorContext& ctx);
  791. void Handle(TEvSchemeShard::TEvPublishTenantAsReadOnlyResult::TPtr& ev, const TActorContext& ctx);
  792. void Handle(TEvSchemeShard::TEvPublishTenant::TPtr& ev, const TActorContext& ctx);
  793. void Handle(TEvSchemeShard::TEvPublishTenantResult::TPtr& ev, const TActorContext& ctx);
  794. void Handle(TEvSchemeShard::TEvMigrateSchemeShard::TPtr& ev, const TActorContext& ctx);
  795. void Handle(TEvSchemeShard::TEvMigrateSchemeShardResult::TPtr& ev, const TActorContext& ctx);
  796. void Handle(TEvDataShard::TEvMigrateSchemeShardResponse::TPtr& ev, const TActorContext& ctx);
  797. void Handle(TEvDataShard::TEvCompactTableResult::TPtr &ev, const TActorContext &ctx);
  798. void Handle(TEvDataShard::TEvCompactBorrowedResult::TPtr &ev, const TActorContext &ctx);
  799. void Handle(TEvSchemeShard::TEvProcessingRequest::TPtr& ev, const TActorContext& ctx);
  800. void Handle(TEvSchemeShard::TEvSyncTenantSchemeShard::TPtr& ev, const TActorContext& ctx);
  801. void Handle(TEvSchemeShard::TEvUpdateTenantSchemeShard::TPtr& ev, const TActorContext& ctx);
  802. void Handle(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev, const TActorContext& ctx);
  803. void Handle(TEvTxProcessing::TEvPlanStep::TPtr &ev, const TActorContext &ctx);
  804. void Handle(TEvents::TEvUndelivered::TPtr& ev, const TActorContext& ctx);
  805. void Handle(NMon::TEvRemoteHttpInfo::TPtr& ev, const TActorContext& ctx);
  806. void Handle(TEvDataShard::TEvInitSplitMergeDestinationAck::TPtr& ev, const TActorContext& ctx);
  807. void Handle(TEvDataShard::TEvSplitAck::TPtr& ev, const TActorContext& ctx);
  808. void Handle(TEvDataShard::TEvSplitPartitioningChangedAck::TPtr& ev, const TActorContext& ctx);
  809. void ExecuteTableStatsBatch(const TActorContext& ctx);
  810. void ScheduleTableStatsBatch(const TActorContext& ctx);
  811. void Handle(TEvPrivate::TEvPersistTableStats::TPtr& ev, const TActorContext& ctx);
  812. void Handle(TEvDataShard::TEvPeriodicTableStats::TPtr& ev, const TActorContext& ctx);
  813. void Handle(TEvDataShard::TEvGetTableStatsResult::TPtr& ev, const TActorContext& ctx);
  814. void ExecuteTopicStatsBatch(const TActorContext& ctx);
  815. void ScheduleTopicStatsBatch(const TActorContext& ctx);
  816. void Handle(TEvPrivate::TEvPersistTopicStats::TPtr& ev, const TActorContext& ctx);
  817. void Handle(TEvPersQueue::TEvPeriodicTopicStats::TPtr& ev, const TActorContext& ctx);
  818. void Handle(TEvSchemeShard::TEvFindTabletSubDomainPathId::TPtr& ev, const TActorContext& ctx);
  819. void ScheduleConditionalEraseRun(const TActorContext& ctx);
  820. void Handle(TEvPrivate::TEvRunConditionalErase::TPtr& ev, const TActorContext& ctx);
  821. void Handle(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev, const TActorContext& ctx);
  822. void ConditionalEraseHandleDisconnect(TTabletId tabletId, const TActorId& clientId, const TActorContext& ctx);
  823. void Handle(NSysView::TEvSysView::TEvGetPartitionStats::TPtr& ev, const TActorContext& ctx);
  824. void ScheduleServerlessStorageBilling(const TActorContext& ctx);
  825. void Handle(TEvPrivate::TEvServerlessStorageBilling::TPtr& ev, const TActorContext& ctx);
  826. void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr &ev, const TActorContext &ctx);
  827. void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr &ev, const TActorContext &ctx);
  828. void Handle(TEvSchemeShard::TEvLogin::TPtr& ev, const TActorContext& ctx);
  829. void RestartPipeTx(TTabletId tabletId, const TActorContext& ctx);
  830. TOperationId RouteIncoming(TTabletId tabletId, const TActorContext& ctx);
  831. // namespace NLongRunningCommon {
  832. struct TXxport {
  833. class TTxBase;
  834. template <typename TInfo, typename TEvRequest, typename TEvResponse> struct TTxGet;
  835. template <typename TInfo, typename TEvRequest, typename TEvResponse, typename TDerived> struct TTxList;
  836. };
  837. void Handle(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev, const TActorContext& ctx);
  838. void Handle(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev, const TActorContext& ctx);
  839. void Handle(TEvIndexBuilder::TEvCreateResponse::TPtr& ev, const TActorContext& ctx);
  840. void Handle(TEvSchemeShard::TEvNotifyTxCompletionRegistered::TPtr& ev, const TActorContext& ctx);
  841. void Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx);
  842. void Handle(TEvSchemeShard::TEvCancelTxResult::TPtr& ev, const TActorContext& ctx);
  843. void Handle(TEvIndexBuilder::TEvCancelResponse::TPtr& ev, const TActorContext& ctx);
  844. // } // NLongRunningCommon
  845. // namespace NExport {
  846. THashMap<ui64, TExportInfo::TPtr> Exports;
  847. THashMap<TString, TExportInfo::TPtr> ExportsByUid;
  848. THashMap<TTxId, std::pair<ui64, ui32>> TxIdToExport;
  849. THashMap<TTxId, THashSet<ui64>> TxIdToDependentExport;
  850. void FromXxportInfo(NKikimrExport::TExport& exprt, const TExportInfo::TPtr exportInfo);
  851. static void PersistCreateExport(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo);
  852. static void PersistRemoveExport(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo);
  853. static void PersistExportPathId(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo);
  854. static void PersistExportState(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo);
  855. static void PersistExportItemState(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo, ui32 targetIdx);
  856. struct TExport {
  857. struct TTxCreate;
  858. struct TTxGet;
  859. struct TTxCancel;
  860. struct TTxCancelAck;
  861. struct TTxForget;
  862. struct TTxList;
  863. struct TTxProgress;
  864. };
  865. NTabletFlatExecutor::ITransaction* CreateTxCreateExport(TEvExport::TEvCreateExportRequest::TPtr& ev);
  866. NTabletFlatExecutor::ITransaction* CreateTxGetExport(TEvExport::TEvGetExportRequest::TPtr& ev);
  867. NTabletFlatExecutor::ITransaction* CreateTxCancelExport(TEvExport::TEvCancelExportRequest::TPtr& ev);
  868. NTabletFlatExecutor::ITransaction* CreateTxCancelExportAck(TEvSchemeShard::TEvCancelTxResult::TPtr& ev);
  869. NTabletFlatExecutor::ITransaction* CreateTxForgetExport(TEvExport::TEvForgetExportRequest::TPtr& ev);
  870. NTabletFlatExecutor::ITransaction* CreateTxListExports(TEvExport::TEvListExportsRequest::TPtr& ev);
  871. NTabletFlatExecutor::ITransaction* CreateTxProgressExport(ui64 id);
  872. NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev);
  873. NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev);
  874. NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TTxId completedTxId);
  875. void Handle(TEvExport::TEvCreateExportRequest::TPtr& ev, const TActorContext& ctx);
  876. void Handle(TEvExport::TEvGetExportRequest::TPtr& ev, const TActorContext& ctx);
  877. void Handle(TEvExport::TEvCancelExportRequest::TPtr& ev, const TActorContext& ctx);
  878. void Handle(TEvExport::TEvForgetExportRequest::TPtr& ev, const TActorContext& ctx);
  879. void Handle(TEvExport::TEvListExportsRequest::TPtr& ev, const TActorContext& ctx);
  880. void ResumeExports(const TVector<ui64>& exportIds, const TActorContext& ctx);
  881. // } // NExport
  882. // namespace NImport {
  883. THashMap<ui64, TImportInfo::TPtr> Imports;
  884. THashMap<TString, TImportInfo::TPtr> ImportsByUid;
  885. THashMap<TTxId, std::pair<ui64, ui32>> TxIdToImport;
  886. void FromXxportInfo(NKikimrImport::TImport& exprt, const TImportInfo::TPtr importInfo);
  887. static void PersistCreateImport(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo);
  888. static void PersistRemoveImport(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo);
  889. static void PersistImportState(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo);
  890. static void PersistImportItemState(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx);
  891. static void PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx);
  892. static void PersistImportItemDstPathId(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx);
  893. struct TImport {
  894. struct TTxCreate;
  895. struct TTxGet;
  896. struct TTxCancel;
  897. struct TTxCancelAck;
  898. struct TTxForget;
  899. struct TTxList;
  900. struct TTxProgress;
  901. };
  902. NTabletFlatExecutor::ITransaction* CreateTxCreateImport(TEvImport::TEvCreateImportRequest::TPtr& ev);
  903. NTabletFlatExecutor::ITransaction* CreateTxGetImport(TEvImport::TEvGetImportRequest::TPtr& ev);
  904. NTabletFlatExecutor::ITransaction* CreateTxCancelImport(TEvImport::TEvCancelImportRequest::TPtr& ev);
  905. NTabletFlatExecutor::ITransaction* CreateTxCancelImportAck(TEvSchemeShard::TEvCancelTxResult::TPtr& ev);
  906. NTabletFlatExecutor::ITransaction* CreateTxCancelImportAck(TEvIndexBuilder::TEvCancelResponse::TPtr& ev);
  907. NTabletFlatExecutor::ITransaction* CreateTxForgetImport(TEvImport::TEvForgetImportRequest::TPtr& ev);
  908. NTabletFlatExecutor::ITransaction* CreateTxListImports(TEvImport::TEvListImportsRequest::TPtr& ev);
  909. NTabletFlatExecutor::ITransaction* CreateTxProgressImport(ui64 id, const TMaybe<ui32>& itemIdx = Nothing());
  910. NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvPrivate::TEvImportSchemeReady::TPtr& ev);
  911. NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev);
  912. NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev);
  913. NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvIndexBuilder::TEvCreateResponse::TPtr& ev);
  914. NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TTxId completedTxId);
  915. void Handle(TEvImport::TEvCreateImportRequest::TPtr& ev, const TActorContext& ctx);
  916. void Handle(TEvImport::TEvGetImportRequest::TPtr& ev, const TActorContext& ctx);
  917. void Handle(TEvImport::TEvCancelImportRequest::TPtr& ev, const TActorContext& ctx);
  918. void Handle(TEvImport::TEvForgetImportRequest::TPtr& ev, const TActorContext& ctx);
  919. void Handle(TEvImport::TEvListImportsRequest::TPtr& ev, const TActorContext& ctx);
  920. void Handle(TEvPrivate::TEvImportSchemeReady::TPtr& ev, const TActorContext& ctx);
  921. void ResumeImports(const TVector<ui64>& ids, const TActorContext& ctx);
  922. // } // NImport
  923. void FillTableSchemaVersion(ui64 schemaVersion, NKikimrSchemeOp::TTableDescription *tableDescr) const;
  924. // namespace NIndexBuilder {
  925. TControlWrapper AllowDataColumnForIndexTable;
  926. THashMap<TIndexBuildId, TIndexBuildInfo::TPtr> IndexBuilds;
  927. THashMap<TString, TIndexBuildInfo::TPtr> IndexBuildsByUid;
  928. THashMap<TTxId, TIndexBuildId> TxIdToIndexBuilds;
  929. // do not share pipes with operations
  930. // also do not share pipes between IndexBuilds
  931. TDedicatedPipePool<TIndexBuildId> IndexBuildPipes;
  932. void PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  933. void PersistBuildIndexState(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  934. void PersistBuildIndexIssue(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  935. void PersistBuildIndexCancelRequest(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  936. void PersistBuildIndexAlterMainTableTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  937. void PersistBuildIndexAlterMainTableTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  938. void PersistBuildIndexAlterMainTableTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  939. void PersistBuildIndexInitiateTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  940. void PersistBuildIndexInitiateTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  941. void PersistBuildIndexInitiateTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  942. void PersistBuildIndexLockTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  943. void PersistBuildIndexLockTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  944. void PersistBuildIndexLockTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  945. void PersistBuildIndexApplyTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  946. void PersistBuildIndexApplyTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  947. void PersistBuildIndexApplyTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  948. void PersistBuildIndexUnlockTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  949. void PersistBuildIndexUnlockTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  950. void PersistBuildIndexUnlockTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  951. void PersistBuildIndexUploadProgress(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo, const TShardIdx& shardIdx);
  952. void PersistBuildIndexUploadInitiate(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo, const TShardIdx& shardIdx);
  953. void PersistBuildIndexBilling(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  954. void PersistBuildIndexForget(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo);
  955. struct TIndexBuilder {
  956. class TTxBase;
  957. template<typename TRequest, typename TResponse>
  958. class TTxSimple;
  959. class TTxCreate;
  960. struct TTxGet;
  961. struct TTxCancel;
  962. struct TTxForget;
  963. struct TTxList;
  964. struct TTxProgress;
  965. struct TTxReply;
  966. struct TTxPipeReset;
  967. struct TTxBilling;
  968. };
  969. NTabletFlatExecutor::ITransaction* CreateTxCreate(TEvIndexBuilder::TEvCreateRequest::TPtr& ev);
  970. NTabletFlatExecutor::ITransaction* CreateTxGet(TEvIndexBuilder::TEvGetRequest::TPtr& ev);
  971. NTabletFlatExecutor::ITransaction* CreateTxCancel(TEvIndexBuilder::TEvCancelRequest::TPtr& ev);
  972. NTabletFlatExecutor::ITransaction* CreateTxForget(TEvIndexBuilder::TEvForgetRequest::TPtr& ev);
  973. NTabletFlatExecutor::ITransaction* CreateTxList(TEvIndexBuilder::TEvListRequest::TPtr& ev);
  974. NTabletFlatExecutor::ITransaction* CreateTxProgress(TIndexBuildId id);
  975. NTabletFlatExecutor::ITransaction* CreateTxReply(TEvTxAllocatorClient::TEvAllocateResult::TPtr& allocateResult);
  976. NTabletFlatExecutor::ITransaction* CreateTxReply(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& modifyResult);
  977. NTabletFlatExecutor::ITransaction* CreateTxReply(TTxId completedTxId);
  978. NTabletFlatExecutor::ITransaction* CreateTxReply(TEvDataShard::TEvBuildIndexProgressResponse::TPtr& progress);
  979. NTabletFlatExecutor::ITransaction* CreatePipeRetry(TIndexBuildId indexBuildId, TTabletId tabletId);
  980. NTabletFlatExecutor::ITransaction* CreateTxBilling(TEvPrivate::TEvIndexBuildingMakeABill::TPtr& ev);
  981. void Handle(TEvIndexBuilder::TEvCreateRequest::TPtr& ev, const TActorContext& ctx);
  982. void Handle(TEvIndexBuilder::TEvGetRequest::TPtr& ev, const TActorContext& ctx);
  983. void Handle(TEvIndexBuilder::TEvCancelRequest::TPtr& ev, const TActorContext& ctx);
  984. void Handle(TEvIndexBuilder::TEvForgetRequest::TPtr& ev, const TActorContext& ctx);
  985. void Handle(TEvIndexBuilder::TEvListRequest::TPtr& ev, const TActorContext& ctx);
  986. void Handle(TEvDataShard::TEvBuildIndexProgressResponse::TPtr& ev, const TActorContext& ctx);
  987. void Handle(TEvPrivate::TEvIndexBuildingMakeABill::TPtr& ev, const TActorContext& ctx);
  988. void Resume(const TDeque<TIndexBuildId>& indexIds, const TActorContext& ctx);
  989. void SetupRouting(const TDeque<TIndexBuildId>& indexIds, const TActorContext& ctx);
  990. // } //NIndexBuilder
  991. // namespace NCdcStreamScan {
  992. struct TCdcStreamScan {
  993. struct TTxProgress;
  994. };
  995. TDedicatedPipePool<TPathId> CdcStreamScanPipes;
  996. NTabletFlatExecutor::ITransaction* CreateTxProgressCdcStreamScan(TEvPrivate::TEvRunCdcStreamScan::TPtr& ev);
  997. NTabletFlatExecutor::ITransaction* CreateTxProgressCdcStreamScan(TEvDataShard::TEvCdcStreamScanResponse::TPtr& ev);
  998. NTabletFlatExecutor::ITransaction* CreatePipeRetry(const TPathId& streamPathId, TTabletId tabletId);
  999. void Handle(TEvPrivate::TEvRunCdcStreamScan::TPtr& ev, const TActorContext& ctx);
  1000. void Handle(TEvDataShard::TEvCdcStreamScanResponse::TPtr& ev, const TActorContext& ctx);
  1001. void ResumeCdcStreamScans(const TVector<TPathId>& ids, const TActorContext& ctx);
  1002. void PersistCdcStreamScanShardStatus(NIceDb::TNiceDb& db, const TPathId& streamPathId, const TShardIdx& shardIdx,
  1003. const TCdcStreamInfo::TShardStatus& status);
  1004. void RemoveCdcStreamScanShardStatus(NIceDb::TNiceDb& db, const TPathId& streamPathId, const TShardIdx& shardIdx);
  1005. // } // NCdcStreamScan
  1006. // statistics
  1007. TTabletId StatisticsAggregatorId;
  1008. TActorId SAPipeClientId;
  1009. static constexpr ui64 SendStatsIntervalMinSeconds = 180;
  1010. static constexpr ui64 SendStatsIntervalMaxSeconds = 240;
  1011. void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr&, const TActorContext& ctx);
  1012. void Handle(TEvPrivate::TEvSendBaseStatsToSA::TPtr& ev, const TActorContext& ctx);
  1013. void InitializeStatistics(const TActorContext& ctx);
  1014. void ResolveSA();
  1015. void ConnectToSA();
  1016. void SendBaseStatsToSA();
  1017. public:
  1018. void ChangeStreamShardsCount(i64 delta) override;
  1019. void ChangeStreamShardsQuota(i64 delta) override;
  1020. void ChangeStreamReservedStorageCount(i64 delta) override;
  1021. void ChangeStreamReservedStorageQuota(i64 delta) override;
  1022. void ChangeDiskSpaceTablesDataBytes(i64 delta) override;
  1023. void ChangeDiskSpaceTablesIndexBytes(i64 delta) override;
  1024. void ChangeDiskSpaceTablesTotalBytes(i64 delta) override;
  1025. void ChangeDiskSpaceTopicsTotalBytes(ui64 value) override;
  1026. void ChangeDiskSpaceQuotaExceeded(i64 delta) override;
  1027. void ChangeDiskSpaceHardQuotaBytes(i64 delta) override;
  1028. void ChangeDiskSpaceSoftQuotaBytes(i64 delta) override;
  1029. NLogin::TLoginProvider LoginProvider;
  1030. private:
  1031. void OnDetach(const TActorContext &ctx) override;
  1032. void OnTabletDead(TEvTablet::TEvTabletDead::TPtr &ev, const TActorContext &ctx) override;
  1033. void OnActivateExecutor(const TActorContext &ctx) override;
  1034. bool OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext &ctx) override;
  1035. void DefaultSignalTabletActive(const TActorContext &ctx) override;
  1036. void Cleanup(const TActorContext &ctx);
  1037. void Enqueue(STFUNC_SIG) override;
  1038. void Die(const TActorContext &ctx) override;
  1039. bool ReassignChannelsEnabled() const override {
  1040. return true;
  1041. }
  1042. const TDomainsInfo::TDomain& GetDomainDescription(const TActorContext &ctx) const;
  1043. NKikimrSubDomains::TProcessingParams CreateRootProcessingParams(const TActorContext &ctx);
  1044. static NTabletPipe::TClientConfig GetPipeClientConfig();
  1045. public:
  1046. static const NKikimrConfig::TDomainsConfig& GetDomainsConfig();
  1047. static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
  1048. return NKikimrServices::TActivity::FLAT_SCHEMESHARD_ACTOR;
  1049. }
  1050. TSchemeShard(const TActorId &tablet, TTabletStorageInfo *info);
  1051. //TTabletId TabletID() const { return TTabletId(ITablet::TabletID()); }
  1052. TTabletId SelfTabletId() const { return TTabletId(ITablet::TabletID()); }
  1053. STFUNC(StateInit);
  1054. STFUNC(StateConfigure);
  1055. STFUNC(StateWork);
  1056. STFUNC(BrokenState);
  1057. // A helper that enforces write-only access to the internal DB (reads must be done from the
  1058. // internal structures)
  1059. class TRwTxBase : public NTabletFlatExecutor::TTransactionBase<TSchemeShard> {
  1060. protected:
  1061. TDuration ExecuteDuration;
  1062. protected:
  1063. TRwTxBase(TSchemeShard* self) : TBase(self) {}
  1064. public:
  1065. virtual ~TRwTxBase() {}
  1066. bool Execute(NTabletFlatExecutor::TTransactionContext &txc, const TActorContext &ctx) override;
  1067. void Complete(const TActorContext &ctx) override;
  1068. virtual void DoExecute(NTabletFlatExecutor::TTransactionContext &txc, const TActorContext &ctx) = 0;
  1069. virtual void DoComplete(const TActorContext &ctx) = 0;
  1070. };
  1071. };
  1072. }
  1073. }