Просмотр исходного кода

Various queue static export improvements: add lower bound naming and export_ttl

I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en

---

Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/197
Andrey Chulkov 1 год назад
Родитель
Сommit
40d1dd813a
2 измененных файлов с 15 добавлено и 0 удалено
  1. 4 0
      yt/yt/client/queue_client/config.cpp
  2. 11 0
      yt/yt/client/queue_client/config.h

+ 4 - 0
yt/yt/client/queue_client/config.cpp

@@ -58,8 +58,12 @@ void TQueueStaticExportConfig::Register(TRegistrar registrar)
     registrar.Parameter("export_period", &TThis::ExportPeriod)
         .GreaterThan(TDuration::Zero());
     registrar.Parameter("export_directory", &TThis::ExportDirectory);
+    registrar.Parameter("export_ttl", &TThis::ExportTtl)
+        .Default(TDuration::Zero());
     registrar.Parameter("output_table_name_pattern", &TThis::OutputTableNamePattern)
         .Default("%UNIX_TS-%PERIOD");
+    registrar.Parameter("use_upper_bound_for_table_names", &TThis::UseUpperBoundForTableNames)
+        .Default(false);
 
     registrar.Postprocessor([] (TThis* config) {
         if (config->ExportPeriod.GetValue() % TDuration::Seconds(1).GetValue() != 0) {

+ 11 - 0
yt/yt/client/queue_client/config.h

@@ -73,6 +73,9 @@ public:
     //! Path to directory that will contain resulting static tables with exported data.
     NYPath::TYPath ExportDirectory;
 
+    //! If set to a value larger than zero, the created output tables will be created with an expiration time computed as now + ttl.
+    TDuration ExportTtl;
+
     //! A format-string supporting the following specifiers:
     //!  - %UNIX_TS: the unix timestamp corresponding to the exported table
     //!  - %PERIOD: the length of the export period in seconds
@@ -83,6 +86,14 @@ public:
     //! An attempt to produce a table which already exists will lead to an error, in which case the data will be exported
     //! on the next iteration.
     TString OutputTableNamePattern;
+    //! If true, the unix timestamp used in formatting the output table name will be the upper bound actually used in gathering chunks for this table.
+    //! Otherwise, the unix timestamp used in the name formatting will be equal to the upper bound minus one export period.
+    //! E.g. if hourly exports are set up, setting this value to true will mean that a table named 17:00 has data from 16:00 to 17:00,
+    //! whereas setting it to false would mean that it has data from 17:00 to 18:00.
+    //! NB: In any case, it should be understood that the table exported with an upper bound timestamp of T is not guaranteed to contain all
+    //! data with timestamp <= T. In case of delays, some of the data can end up in latter tables. You should be especially careful with
+    //! commit_ordering=%false queues, since commit timestamp monotonicty within a tablet is not guaranteed for them.
+    bool UseUpperBoundForTableNames;
 
     REGISTER_YSON_STRUCT_LITE(TQueueStaticExportConfig);