PR from branch users/jansenin/static-config-validation2

tests for static validator

add static validator builder part and make make enums case-insensitive

.mapping.json

@@ -6624,6 +6624,11 @@
   "ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-x86_64.txt":"",
   "ydb/library/yaml_config/static_validator/ut/CMakeLists.txt":"",
   "ydb/library/yaml_config/static_validator/ut/CMakeLists.windows-x86_64.txt":"",
+  "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.darwin-x86_64.txt":"",
+  "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-aarch64.txt":"",
+  "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-x86_64.txt":"",
+  "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.txt":"",
+  "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.windows-x86_64.txt":"",
   "ydb/library/yaml_config/ut/CMakeLists.darwin-x86_64.txt":"",
   "ydb/library/yaml_config/ut/CMakeLists.linux-aarch64.txt":"",
   "ydb/library/yaml_config/ut/CMakeLists.linux-x86_64.txt":"",

ydb/library/yaml_config/static_validator/builders.cpp

@@ -7,13 +7,17 @@ using namespace NYamlConfig::NValidator;
 using namespace NYamlConfig::NValidator::Configurators;
 
 TArrayBuilder HostConfigBuilder() {
-  return TArrayBuilder([](auto& b) {b
+  return TArrayBuilder([](auto& hostsConfig) {
+    hostsConfig
     .Configure(uniqueByPath("host_config_id"))
-    .MapItem([](auto& b) {b
+    .MapItem([](auto& host) {
+      host
       .Int64("host_config_id", nonNegative()) // >= 0?
-      .Array("drive", [](auto& b) {b
+      .Array("drive", [](auto& drive) {
+        drive
         .Configure(uniqueByPath("path"))
-        .MapItem([](auto& b) {b
+        .MapItem([](auto& driveItem) {
+          driveItem
           .String("path")
           .Enum("type", {"ssd", "nvme", "rot"});
         });
@@ -23,33 +27,38 @@ TArrayBuilder HostConfigBuilder() {
 }
 
 TArrayBuilder HostsBuilder() {
-  return TArrayBuilder([](auto& b){b
-    .MapItem([](auto& b) {b
+  return TArrayBuilder([](auto& hosts){
+    hosts
+    .MapItem([](auto& host) {
+      host
       .String("host")
       .Int64("host_config_id", nonNegative())
-      .Int64("node_id", nonNegative())
-      .Int64("port", 0, 65535)
-      .Map("location", [](auto& b){b
-        .String("unit")
-        .String("data_center")
-        .String("rack");
+      .String("node_id", [](auto& nodeId){
+        nodeId.Optional();
+      })
+      .Int64("port", [](auto& port){
+        port
+        .Range(0, 65535)
+        .Optional();
       });
     })
     .Configure(uniqueByPath("node_id"))
-    .AddCheck("Must not have to hosts with same host name and port", [](auto& c) {
-      auto array = c.Node();
+    .AddCheck("Must not have two hosts with same host name and port", [](auto& hostsContext) {
+      auto array = hostsContext.Node();
       THashMap<std::pair<TString, i64>, i64> itemsToIndex;
 
       for (int i = 0; i < array.Length(); ++i) {
         TString host = array[i].Map()["host"].String();
-        i64 port = array[i].Map()["port"].Int64();
+        auto portNode = array[i].Map()["port"];
+        i64 defaultPort = 19001;
+        i64 port = portNode.Exists() ? portNode.Int64() : defaultPort;
         auto item = std::pair{host, port};
 
         if (itemsToIndex.contains(item)) {
           TString i1 = ToString(itemsToIndex[item]);
           TString i2 = ToString(i);
 
-          c.Fail("items with indexes " + i1 + " and " + i2 + " are conflicting");
+          hostsContext.Fail("items with indexes " + i1 + " and " + i2 + " are conflicting");
         }
         itemsToIndex[item] = i;
       }
@@ -58,9 +67,12 @@ TArrayBuilder HostsBuilder() {
 }
 
 TMapBuilder DomainsConfigBuilder() {
-  return TMapBuilder([](auto& b){b
-    .Array("domain", [](auto& b){b
-      .MapItem([](auto& b){b
+  return TMapBuilder([](auto& domainsConfig){
+    domainsConfig
+    .Array("domain", [](auto& domain){
+      domain
+      .MapItem([](auto& domainItem){
+        domainItem
         .String("name")
         .Field("storage_pool_types", StoragePoolTypesConfigBuilder());
       });
@@ -71,44 +83,58 @@ TMapBuilder DomainsConfigBuilder() {
 }
 
 TArrayBuilder StoragePoolTypesConfigBuilder() {
-  return TArrayBuilder([](auto& b){b
-    .MapItem([](auto& b){b
+  return TArrayBuilder([](auto& storagePoolTypes){
+    storagePoolTypes
+    .MapItem([](auto& storagePoolType){
+      storagePoolType
       .String("kind")
-      .Map("pool_config", [](auto& b){b
+      .Map("pool_config", [](auto& poolConfig){
+        poolConfig
         .Int64("box_id", nonNegative())
-        .Int64("encryption", [](auto&b){b
+        .Int64("encryption", [](auto&encryption){
+          encryption
           .Range(0, 1)
           .Optional();
         })
         .Enum("erasure_species", {"none", "block-4-2", "mirror-3-dc", "mirror-3dc-3-nodes"})
         .String("kind")
-        .Array("pdisk_filter", [](auto& b){b
-          .MapItem([](auto& b){b
-            .Array("property", [](auto& b){b
-              .MapItem([](auto& b){b
+        .Array("pdisk_filter", [](auto& pdiskFilter){
+          pdiskFilter
+          .MapItem([](auto& pdiskFilterItem){
+            pdiskFilterItem
+            .Array("property", [](auto& property){
+              property
+              .MapItem([](auto& propertyItem){
+                propertyItem
                 .Enum("type", {"ssd", "nvme", "rot"});
               });
             });
           });
         });
-      })
-      .Configure(mustBeEqual("kind", "pool_config/kind"));
+      });
+      storagePoolType.Configure(mustBeEqual("kind", "pool_config/kind"));
     });
   });
 }
 
 TArrayBuilder StateStorageBuilder() {
-  return TArrayBuilder([](auto& b){b
-    .MapItem([](auto& b){b
-      .Map("ring", [](auto& b){b
-        .Array("node", [](auto& b){b
-          .Int64Item(nonNegative());
+  return TArrayBuilder([](auto& stateStorage){
+    stateStorage
+    .MapItem([](auto& stateStorageItem){
+      stateStorageItem
+      .Map("ring", [](auto& ring){
+        ring
+        .Array("node", [](auto& node){
+          node.Int64Item(nonNegative());
         })
         .Int64("nto_select", nonNegative())
-        .AddCheck("nto_select must not be greater, than node array size", [](auto& c){
-          i64 nto_select = c.Node()["nto_select"].Int64();
-          i64 arr_size = c.Node()["node"].Array().Length();
-          c.Expect(nto_select <= arr_size);
+        .AddCheck("nto_select must not be greater, than node array size", [](auto& ringContext){
+          i64 nto_select = ringContext.Node()["nto_select"].Int64();
+          i64 arr_size = ringContext.Node()["node"].Array().Length();
+          ringContext.Expect(nto_select <= arr_size);
+        })
+        .AddCheck("nto_select should be odd, because even number usage doesn't improve fault tolerance", [](auto& ringContext){
+          ringContext.Expect(ringContext.Node()["nto_select"].Int64() % 2 == 1);
         });
       })
       .Int64("ssid", nonNegative());
@@ -117,10 +143,168 @@ TArrayBuilder StateStorageBuilder() {
 }
 
 TMapBuilder SecurityConfigBuilder() {
-  return TMapBuilder([](auto& b){b
-    .Bool("enforce_user_token_requirement", [](auto& b){
-      b.Optional();
+  return TMapBuilder([](auto& securityConfig){
+    securityConfig
+    .Bool("enforce_user_token_requirement", [](auto& enforceUserTokenRequirements){
+      enforceUserTokenRequirements.Optional();
     })
     .Optional();
   });
 }
+
+TMapBuilder ActorSystemConfigBuilder() {
+  return TMapBuilder([](auto& actorSystem){
+    actorSystem
+    .Bool("use_auto_config", [](auto& useAutoConfig){
+      useAutoConfig.Optional();
+    })
+    .Enum("node_type", [](auto& nodeType){
+      nodeType
+      .SetItems({"STORAGE", "COMPUTE", "HYBRID"})
+      .Optional();
+    })
+    .Int64("cpu_count", [](auto& cpuCount){
+      cpuCount
+      .Min(0)
+      .Optional();
+    })
+    .Array("executor", [](auto& executor){
+      executor
+      .Optional()
+      .MapItem([](auto& executorItem){
+        executorItem
+        .Enum("name", {"System", "User", "Batch", "IO", "IC"})
+        .Int64("spin_threshold", [](auto& spinThreshold){
+          spinThreshold
+          .Min(0)
+          .Optional();
+        })
+        .Int64("threads", nonNegative())
+        .Int64("max_threads", [](auto& maxThreads){
+          maxThreads
+          .Optional()
+          .Min(0);
+        })
+        .Int64("max_avg_ping_deviation", [](auto& maxAvgPingDeviation){
+          maxAvgPingDeviation
+          .Min(0)
+          .Optional();
+        })
+        .Int64("time_per_mailbox_micro_secs", [](auto& timePerMailboxMicroSecs){
+          timePerMailboxMicroSecs
+          .Optional()
+          .Min(0);
+        })
+        .Enum("type", {"IO", "BASIC"});
+      });
+    })
+    .Map("scheduler", [](auto& scheduler){
+      scheduler
+      .Optional()
+      .Int64("progress_threshold", nonNegative())
+      .Int64("resolution", nonNegative())
+      .Int64("spin_threshold", nonNegative());
+    })
+    .AddCheck("Must either be auto config or manual config", [](auto& actorSystemContext){
+      bool autoConfig = false;
+      auto node = actorSystemContext.Node();
+      if (node["use_auto_config"].Exists() && node["use_auto_config"].Bool()) {
+        autoConfig = true;
+      }
+      if (autoConfig) {
+        actorSystemContext.Expect(node["node_type"].Exists(), "node_type must exist when using auto congfig");
+        actorSystemContext.Expect(node["cpu_count"].Exists(), "cpu_count must exist when using auto congfig");
+
+        actorSystemContext.Expect(!node["executor"].Exists(), "executor must not exist when using auto congfig");
+        actorSystemContext.Expect(!node["scheduler"].Exists(), "scheduler must not exist when using auto congfig");
+      } else {
+        actorSystemContext.Expect(node["executor"].Exists(), "executor must exist when not using auto congfig");
+        actorSystemContext.Expect(node["scheduler"].Exists(), "scheduler must exist when not using auto congfig");
+
+        actorSystemContext.Expect(!node["node_type"].Exists(), "node_type must not exist when not using auto congfig");
+        actorSystemContext.Expect(!node["cpu_count"].Exists(), "cpu_count must not exist when not using auto congfig");
+      }
+    });
+  });
+}
+
+NYamlConfig::NValidator::TMapBuilder BlobStorageConfigBuilder() {
+  return TMapBuilder([](auto& blobStorageConfig){
+    blobStorageConfig
+    .Map("service_set", [](auto& serviceSet){
+      serviceSet
+      .Array("groups", [](auto& groups){
+        groups
+        .MapItem([](auto& group){
+          group
+          .Enum("erasure_species", {"none", "block-4-2", "mirror-3-dc", "mirror-3dc-3-nodes"})
+          .Array("rings", [](auto& rings){
+            rings
+            .MapItem([](auto& ring){
+              ring
+              .Array("fail_domains", [](auto& failDomains){
+                failDomains
+                .MapItem([](auto& failDomain){
+                  failDomain
+                  .Array("vdisk_locations", [](auto& vdiskLocations){
+                    vdiskLocations
+                    .MapItem([](auto& vdiskLocation){
+                      vdiskLocation
+                      .String("node_id")
+                      .String("path")
+                      .Enum("pdisk_category", {"ssd", "nvme", "rot"});
+                    });
+                  });
+                });
+              });
+            });
+          });
+        });
+      });
+    });
+  });
+}
+
+NYamlConfig::NValidator::TMapBuilder ChannelProfileConfigBuilder() {
+  return TMapBuilder([](auto& channelProfileConfig){
+    channelProfileConfig
+    .Array("profile", [](auto& profile){
+      profile
+      .MapItem([](auto& profileItem){
+        profileItem
+        .Array("channel", [](auto& channel){
+          channel
+          .MapItem([](auto& channelItem){
+            channelItem
+            .Enum("erasure_species", {"none", "block-4-2", "mirror-3-dc", "mirror-3dc-3-nodes"})
+            .Int64("pdisk_category")
+            .Enum("storage_pool_kind", {"nvme", "ssd", "rot"});
+          });
+        })
+        .Int64("profile_id", nonNegative());
+      });
+    });
+  });
+}
+
+NYamlConfig::NValidator::TMapBuilder StaticConfigBuilder() {
+  return TMapBuilder([](auto& staticConfig){
+    staticConfig
+    .Enum("static_erasure", [](auto& staticErasure){
+      staticErasure
+      .SetItems({"none", "block-4-2", "mirror-3-dc", "mirror-3dc-3-nodes"})
+      .Optional();
+    })
+    .Field("host_configs", HostConfigBuilder())
+    .Field("hosts", HostsBuilder())
+    .Field("domains_config", DomainsConfigBuilder())
+    .Map("table_service_config", [](auto& tableServiceConfig){
+      tableServiceConfig
+      .Optional()
+      .String("sql_version");
+    })
+    .Field("actor_system_config", ActorSystemConfigBuilder())
+    .Field("blob_storage_config", BlobStorageConfigBuilder())
+    .Field("channel_profile_config", ChannelProfileConfigBuilder());
+  });
+}

ydb/library/yaml_config/static_validator/builders.h

@@ -7,3 +7,10 @@ NYamlConfig::NValidator::TMapBuilder DomainsConfigBuilder();
 NYamlConfig::NValidator::TArrayBuilder StoragePoolTypesConfigBuilder();
 NYamlConfig::NValidator::TArrayBuilder StateStorageBuilder();
 NYamlConfig::NValidator::TMapBuilder SecurityConfigBuilder();
+
+NYamlConfig::NValidator::TMapBuilder ActorSystemConfigBuilder();
+NYamlConfig::NValidator::TMapBuilder BlobStorageConfigBuilder();
+
+NYamlConfig::NValidator::TMapBuilder ChannelProfileConfigBuilder();
+
+NYamlConfig::NValidator::TMapBuilder StaticConfigBuilder();

ydb/library/yaml_config/static_validator/ut/CMakeLists.darwin-x86_64.txt

@@ -6,6 +6,7 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(example_configs)
 
 add_executable(ydb-library-yaml_config-static_validator-ut)
 target_link_libraries(ydb-library-yaml_config-static_validator-ut PUBLIC

ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-aarch64.txt

@@ -6,6 +6,7 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(example_configs)
 
 add_executable(ydb-library-yaml_config-static_validator-ut)
 target_link_libraries(ydb-library-yaml_config-static_validator-ut PUBLIC

ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-x86_64.txt

@@ -6,6 +6,7 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(example_configs)
 
 add_executable(ydb-library-yaml_config-static_validator-ut)
 target_link_libraries(ydb-library-yaml_config-static_validator-ut PUBLIC

ydb/library/yaml_config/static_validator/ut/CMakeLists.windows-x86_64.txt

@@ -6,6 +6,7 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(example_configs)
 
 add_executable(ydb-library-yaml_config-static_validator-ut)
 target_link_libraries(ydb-library-yaml_config-static_validator-ut PUBLIC

ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.darwin-x86_64.txt

@@ -0,0 +1,63 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(static_validator-ut-example_configs)
+target_link_libraries(static_validator-ut-example_configs PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cpuid_check
+  cpp-testing-unittest_main
+  library-yaml_config-static_validator
+  library-yaml_config-validator
+)
+target_link_options(static_validator-ut-example_configs PRIVATE
+  -Wl,-platform_version,macos,11.0,11.0
+  -fPIC
+  -fPIC
+)
+target_sources(static_validator-ut-example_configs PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp
+)
+set_property(
+  TARGET
+  static_validator-ut-example_configs
+  PROPERTY
+  SPLIT_FACTOR
+  1
+)
+add_yunittest(
+  NAME
+  static_validator-ut-example_configs
+  TEST_TARGET
+  static_validator-ut-example_configs
+  TEST_ARG
+  --print-before-suite
+  --print-before-test
+  --fork-tests
+  --print-times
+  --show-fails
+)
+set_yunittest_property(
+  TEST
+  static_validator-ut-example_configs
+  PROPERTY
+  LABELS
+  SMALL
+)
+set_yunittest_property(
+  TEST
+  static_validator-ut-example_configs
+  PROPERTY
+  PROCESSORS
+  1
+)
+target_allocator(static_validator-ut-example_configs
+  system_allocator
+)
+vcs_info(static_validator-ut-example_configs)

ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-aarch64.txt

@@ -0,0 +1,68 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(static_validator-ut-example_configs)
+target_link_libraries(static_validator-ut-example_configs PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  cpp-testing-unittest_main
+  library-yaml_config-static_validator
+  library-yaml_config-validator
+)
+target_link_options(static_validator-ut-example_configs PRIVATE
+  -ldl
+  -lrt
+  -Wl,--no-as-needed
+  -fPIC
+  -fPIC
+  -lpthread
+  -lrt
+  -ldl
+)
+target_sources(static_validator-ut-example_configs PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp
+)
+set_property(
+  TARGET
+  static_validator-ut-example_configs
+  PROPERTY
+  SPLIT_FACTOR
+  1
+)
+add_yunittest(
+  NAME
+  static_validator-ut-example_configs
+  TEST_TARGET
+  static_validator-ut-example_configs
+  TEST_ARG
+  --print-before-suite
+  --print-before-test
+  --fork-tests
+  --print-times
+  --show-fails
+)
+set_yunittest_property(
+  TEST
+  static_validator-ut-example_configs
+  PROPERTY
+  LABELS
+  SMALL
+)
+set_yunittest_property(
+  TEST
+  static_validator-ut-example_configs
+  PROPERTY
+  PROCESSORS
+  1
+)
+target_allocator(static_validator-ut-example_configs
+  cpp-malloc-jemalloc
+)
+vcs_info(static_validator-ut-example_configs)

ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-x86_64.txt

@@ -0,0 +1,70 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(static_validator-ut-example_configs)
+target_link_libraries(static_validator-ut-example_configs PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cpuid_check
+  cpp-testing-unittest_main
+  library-yaml_config-static_validator
+  library-yaml_config-validator
+)
+target_link_options(static_validator-ut-example_configs PRIVATE
+  -ldl
+  -lrt
+  -Wl,--no-as-needed
+  -fPIC
+  -fPIC
+  -lpthread
+  -lrt
+  -ldl
+)
+target_sources(static_validator-ut-example_configs PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp
+)
+set_property(
+  TARGET
+  static_validator-ut-example_configs
+  PROPERTY
+  SPLIT_FACTOR
+  1
+)
+add_yunittest(
+  NAME
+  static_validator-ut-example_configs
+  TEST_TARGET
+  static_validator-ut-example_configs
+  TEST_ARG
+  --print-before-suite
+  --print-before-test
+  --fork-tests
+  --print-times
+  --show-fails
+)
+set_yunittest_property(
+  TEST
+  static_validator-ut-example_configs
+  PROPERTY
+  LABELS
+  SMALL
+)
+set_yunittest_property(
+  TEST
+  static_validator-ut-example_configs
+  PROPERTY
+  PROCESSORS
+  1
+)
+target_allocator(static_validator-ut-example_configs
+  cpp-malloc-tcmalloc
+  libs-tcmalloc-no_percpu_cache
+)
+vcs_info(static_validator-ut-example_configs)