#pragma once /// /// @file yt/cpp/mapreduce/interface/client.h /// /// Main header of the C++ YT Wrapper. /// /// @mainpage C++ library for working with YT /// /// This library provides possibilities to work with YT as a [MapReduce](https://en.wikipedia.org/wiki/MapReduce) system. It allows: /// - to read/write tables and files /// - to run operations /// - to work with transactions. /// /// This library provides only basic functions for working with dynamic tables. /// To access full powers of YT dynamic tables one should use /// [yt/client](https://github.com/ytsaurus/ytsaurus/tree/main/yt/yt/client) library. /// /// Entry points to this library: /// - @ref NYT::Initialize() initialization function for this library; /// - @ref NYT::IClient main interface to work with YT cluster; /// - @ref NYT::CreateClient() function that creates client for particular cluster; /// - @ref NYT::IOperationClient ancestor of @ref NYT::IClient containing the set of methods to run operations. /// /// Tutorial on how to use this library can be found [here](https://ytsaurus.tech/docs/en/api/c/description). #include "fwd.h" #include "client_method_options.h" #include "constants.h" #include "batch_request.h" #include "cypress.h" #include "init.h" #include "io.h" #include "node.h" #include "operation.h" #include #include #include #include /// Main namespace of YT client namespace NYT { //////////////////////////////////////////////////////////////////////////////// /// OAuth info (returned by @ref NYT::IClient::WhoAmI). struct TAuthorizationInfo { /// User's login. TString Login; /// Realm. TString Realm; }; //////////////////////////////////////////////////////////////////////////////// /// @brief Part of @ref NYT::TCheckPermissionResponse. /// /// In case when 'Action == ESecurityAction::Deny' because of a 'deny' rule, /// the "denying" object name and id and "denied" subject name an id may be returned. struct TCheckPermissionResult { /// Was the access granted or not. ESecurityAction Action; /// Id of the object whose ACL's "deny" rule forbids the access. TMaybe ObjectId; /// /// @brief Name of the object whose ACL's "deny" rule forbids the access. /// /// Example is "node //tmp/x/y". TMaybe ObjectName; /// Id of the subject for whom the access was denied by a "deny" rule. TMaybe SubjectId; /// Name of the subject for whom the access was denied by a "deny" rule. TMaybe SubjectName; }; /// @brief Result of @ref NYT::IClient::CheckPermission command. /// /// The base part of the response corresponds to the check result for the node itself. /// `Columns` vector contains check results for the columns (in the same order as in the request). struct TCheckPermissionResponse : public TCheckPermissionResult { /// @brief Results for the table columns access permissions. /// /// @see [Columnar ACL doc](https://ytsaurus.tech/docs/en/user-guide/storage/columnar-acl) TVector Columns; }; //////////////////////////////////////////////////////////////////////////////// /// @brief Interface representing a lock obtained from @ref NYT::ITransaction::Lock. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#start_tx) class ILock : public TThrRefBase { public: virtual ~ILock() = default; /// Get cypress node id of lock itself. virtual const TLockId& GetId() const = 0; /// Get cypress node id of locked object. virtual TNodeId GetLockedNodeId() const = 0; /// /// @brief Get future that will be set once lock is in "acquired" state. /// /// Note that future might contain exception if some error occurred /// e.g. lock transaction was aborted. virtual const ::NThreading::TFuture& GetAcquiredFuture() const = 0; /// /// @brief Wait until lock is in "acquired" state. /// /// Throws exception if timeout exceeded or some error occurred /// e.g. lock transaction was aborted. void Wait(TDuration timeout = TDuration::Max()); }; //////////////////////////////////////////////////////////////////////////////// /// @brief Base class for @ref NYT::IClient and @ref NYT::ITransaction. /// /// This class contains transactional commands. class IClientBase : public TThrRefBase , public ICypressClient , public IIOClient , public IOperationClient { public: /// /// @brief Start a [transaction] (https://ytsaurus.tech/docs/en/user-guide/storage/transactions.html#master_transactions). /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#start_tx) [[nodiscard]] virtual ITransactionPtr StartTransaction( const TStartTransactionOptions& options = TStartTransactionOptions()) = 0; /// /// @brief Change properties of table. /// /// Allows to: /// - switch table between dynamic/static mode /// - or change table schema /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#alter_table) virtual void AlterTable( const TYPath& path, const TAlterTableOptions& options = TAlterTableOptions()) = 0; /// /// @brief Create batch request object that allows to execute several light requests in parallel. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#execute_batch) virtual TBatchRequestPtr CreateBatchRequest() = 0; /// @brief Get root client outside of all transactions. virtual IClientPtr GetParentClient() = 0; }; //////////////////////////////////////////////////////////////////////////////// /// @brief Interface representing a master transaction. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/user-guide/storage/transactions.html#master_transactions) class ITransaction : virtual public IClientBase { public: /// Get id of transaction. virtual const TTransactionId& GetId() const = 0; /// /// @brief Try to lock given path. /// /// Lock will be held until transaction is committed/aborted or @ref NYT::ITransaction::Unlock method is called. /// Lock modes: /// - `LM_EXCLUSIVE`: if exclusive lock is taken no other transaction can take exclusive or shared lock. /// - `LM_SHARED`: if shared lock is taken other transactions can take shared lock but not exclusive. /// - `LM_SNAPSHOT`: snapshot lock always succeeds, when snapshot lock is taken current transaction snapshots object. /// It will not see changes that occurred to it in other transactions. /// /// Exclusive/shared lock can be waitable or not. /// If nonwaitable lock cannot be taken exception is thrown. /// If waitable lock cannot be taken it is created in pending state and client can wait until it actually taken. /// Check @ref NYT::TLockOptions::Waitable and @ref NYT::ILock::GetAcquiredFuture for more details. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#lock) virtual ILockPtr Lock( const TYPath& path, ELockMode mode, const TLockOptions& options = TLockOptions()) = 0; /// /// @brief Remove all the locks (including pending ones) for this transaction from a Cypress node at `path`. /// /// If the locked version of the node differs from the original one, /// an error will be thrown. /// /// Command is successful even if the node has no locks. /// Only explicit (created by @ref NYT::ITransaction::Lock) locks are removed. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#unlock) virtual void Unlock( const TYPath& path, const TUnlockOptions& options = TUnlockOptions()) = 0; /// /// @brief Commit transaction. /// /// All changes that are made by transactions become visible globally or to parent transaction. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#commit_tx) virtual void Commit() = 0; /// /// @brief Abort transaction. /// /// All changes made by current transaction are lost. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#abort_tx) virtual void Abort() = 0; /// @brief Explicitly ping transaction. /// /// User usually does not need this method (as transactions are pinged automatically, /// see @ref NYT::TStartTransactionOptions::AutoPingable). /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#ping_tx) virtual void Ping() = 0; /// /// @brief Detach transaction. /// /// Stop any activities connected with it: pinging, aborting on crashes etc. /// Forget about the transaction totally. virtual void Detach(); }; //////////////////////////////////////////////////////////////////////////////// /// Interface containing non-transactional commands. class IClient : virtual public IClientBase { public: /// /// @brief Attach to existing master transaction. /// /// Returned object WILL NOT: /// - ping transaction automatically (unless @ref NYT::TAttachTransactionOptions::AutoPing is set) /// - abort it on program termination (unless @ref NYT::TAttachTransactionOptions::AbortOnTermination is set). /// Otherwise returned object is similar to the object returned by @ref NYT::IClientBase::StartTransaction. /// and it can see all the changes made inside the transaction. [[nodiscard]] virtual ITransactionPtr AttachTransaction( const TTransactionId& transactionId, const TAttachTransactionOptions& options = TAttachTransactionOptions()) = 0; /// /// @brief Mount dynamic table. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#mount_table) virtual void MountTable( const TYPath& path, const TMountTableOptions& options = TMountTableOptions()) = 0; /// /// @brief Unmount dynamic table. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#unmount_table) virtual void UnmountTable( const TYPath& path, const TUnmountTableOptions& options = TUnmountTableOptions()) = 0; /// /// @brief Remount dynamic table. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#remount_table) virtual void RemountTable( const TYPath& path, const TRemountTableOptions& options = TRemountTableOptions()) = 0; /// /// @brief Switch dynamic table from `mounted' into `frozen' state. /// /// When table is in frozen state all its data is flushed to disk and writes are disabled. /// /// @note this function launches the process of switching, but doesn't wait until switching is accomplished. /// Waiting has to be performed by user. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#freeze_table) virtual void FreezeTable( const TYPath& path, const TFreezeTableOptions& options = TFreezeTableOptions()) = 0; /// /// @brief Switch dynamic table from `frozen` into `mounted` state. /// /// @note this function launches the process of switching, but doesn't wait until switching is accomplished. /// Waiting has to be performed by user. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#unfreeze_table) virtual void UnfreezeTable( const TYPath& path, const TUnfreezeTableOptions& options = TUnfreezeTableOptions()) = 0; /// /// @brief Reshard dynamic table (break it into tablets) by given pivot keys. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#reshard_table) virtual void ReshardTable( const TYPath& path, const TVector& pivotKeys, const TReshardTableOptions& options = TReshardTableOptions()) = 0; /// /// @brief Reshard dynamic table, breaking it into given number of tablets. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#reshard_table) virtual void ReshardTable( const TYPath& path, i64 tabletCount, const TReshardTableOptions& options = TReshardTableOptions()) = 0; /// /// @brief Insert rows into dynamic table. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#insert_rows) virtual void InsertRows( const TYPath& path, const TNode::TListType& rows, const TInsertRowsOptions& options = TInsertRowsOptions()) = 0; /// /// @brief Delete rows from dynamic table. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#delete_rows) virtual void DeleteRows( const TYPath& path, const TNode::TListType& keys, const TDeleteRowsOptions& options = TDeleteRowsOptions()) = 0; /// /// @brief Trim rows from the beginning of ordered dynamic table. /// /// Asynchronously removes `rowCount` rows from the beginning of ordered dynamic table. /// Numeration of remaining rows *does not change*, e.g. after `trim(10)` and `trim(20)` /// you get in total `20` deleted rows. /// /// @param path Path to ordered dynamic table. /// @param tabletIndex Which tablet to trim. /// @param rowCount How many trimmed rows will be in the table after command. /// @param options Optional parameters. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#trim_rows) virtual void TrimRows( const TYPath& path, i64 tabletIndex, i64 rowCount, const TTrimRowsOptions& options = TTrimRowsOptions()) = 0; /// /// @brief Lookup rows with given keys from dynamic table. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#lookup-rows) virtual TNode::TListType LookupRows( const TYPath& path, const TNode::TListType& keys, const TLookupRowsOptions& options = TLookupRowsOptions()) = 0; /// /// @brief Select rows from dynamic table, using [SQL dialect](https://ytsaurus.tech/docs/en//description/dynamic_tables/dyn_query_language.html). /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#select-rows) virtual TNode::TListType SelectRows( const TString& query, const TSelectRowsOptions& options = TSelectRowsOptions()) = 0; /// /// @brief Change properties of table replica. /// /// Allows to enable/disable replica and/or change its mode. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#alter_table_replica) virtual void AlterTableReplica( const TReplicaId& replicaId, const TAlterTableReplicaOptions& alterTableReplicaOptions) = 0; /// /// @brief Generate a monotonously increasing master timestamp. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#generate_timestamp) virtual ui64 GenerateTimestamp() = 0; /// Return YT username of current client. virtual TAuthorizationInfo WhoAmI() = 0; /// /// @brief Get operation attributes. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#get_operation) virtual TOperationAttributes GetOperation( const TOperationId& operationId, const TGetOperationOptions& options = TGetOperationOptions()) = 0; /// /// @brief Get operation attributes. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#get_operation) virtual TOperationAttributes GetOperation( const TString& alias, const TGetOperationOptions& options = TGetOperationOptions()) = 0; /// /// @brief List operations satisfying given filters. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#list_operations) virtual TListOperationsResult ListOperations( const TListOperationsOptions& options = TListOperationsOptions()) = 0; /// /// @brief Update operation runtime parameters. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#update_operation_parameters) virtual void UpdateOperationParameters( const TOperationId& operationId, const TUpdateOperationParametersOptions& options) = 0; /// /// @brief Get job attributes. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#get_job) virtual TJobAttributes GetJob( const TOperationId& operationId, const TJobId& jobId, const TGetJobOptions& options = TGetJobOptions()) = 0; /// /// List attributes of jobs satisfying given filters. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#list_jobs) virtual TListJobsResult ListJobs( const TOperationId& operationId, const TListJobsOptions& options = TListJobsOptions()) = 0; /// /// @brief Get the input of a running or failed job. /// /// @ref NYT::TErrorResponse exception is thrown if job is missing. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#get_job_input) virtual IFileReaderPtr GetJobInput( const TJobId& jobId, const TGetJobInputOptions& options = TGetJobInputOptions()) = 0; /// /// @brief Get fail context of a failed job. /// /// @ref NYT::TErrorResponse exception is thrown if it is missing. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#get_job_fail_context) virtual IFileReaderPtr GetJobFailContext( const TOperationId& operationId, const TJobId& jobId, const TGetJobFailContextOptions& options = TGetJobFailContextOptions()) = 0; /// /// @brief Get stderr of a running or failed job. /// /// @ref NYT::TErrorResponse exception is thrown if it is missing. /// /// @note YT doesn't store all job stderrs /// /// @note If job stderr exceeds few megabytes YT will store only head and tail of stderr. /// /// @see Description of `max_stderr_size` spec option [here](https://ytsaurus.tech/docs/en/user-guide/data-processing/operations/operations-options). /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#get_job_stderr) virtual IFileReaderPtr GetJobStderr( const TOperationId& operationId, const TJobId& jobId, const TGetJobStderrOptions& options = TGetJobStderrOptions()) = 0; /// /// @brief Create one or several rbtorrents for files in a blob table. /// /// If specified, one torrent is created for each value of `KeyColumns` option. /// Otherwise, a single torrent with all files of a table is created. /// /// @return list of nodes, each node has two fields /// * `key`: list of key columns values. Empty if `KeyColumns` is not specified. /// * `rbtorrent`: rbtorrent string (with `rbtorrent:` prefix) /// /// @see [More info.](https://docs.yandex-team.ru/docs/yt/description/storage/blobtables#sky_share) virtual TNode::TListType SkyShareTable( const std::vector& tablePaths, const TSkyShareTableOptions& options) = 0; /// /// @brief Check if `user` has `permission` to access a Cypress node at `path`. /// /// For tables access to columns specified in `options.Columns_` can be checked /// (@see [the doc](https://ytsaurus.tech/docs/en/user-guide/storage/columnar-acl)). /// /// If access is denied (the returned result has `.Action == ESecurityAction::Deny`) /// because of a `deny` rule, the "denying" object name and id /// and "denied" subject name an id may be returned. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#check_permission) virtual TCheckPermissionResponse CheckPermission( const TString& user, EPermission permission, const TYPath& path, const TCheckPermissionOptions& options = TCheckPermissionOptions()) = 0; /// @brief Get information about tablet /// @see NYT::TTabletInfo virtual TVector GetTabletInfos( const TYPath& path, const TVector& tabletIndexes, const TGetTabletInfosOptions& options = TGetTabletInfosOptions()) = 0; /// /// @brief Suspend operation. /// /// Jobs will be aborted. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#suspend_operation) virtual void SuspendOperation( const TOperationId& operationId, const TSuspendOperationOptions& options = TSuspendOperationOptions()) = 0; /// @brief Resume previously suspended operation. /// /// @see [YT doc](https://ytsaurus.tech/docs/en/api/commands.html#resume_operation) virtual void ResumeOperation( const TOperationId& operationId, const TResumeOperationOptions& options = TResumeOperationOptions()) = 0; /// /// @brief Synchronously terminates all client's background activities /// /// e.g. no callbacks will be executed after the function is completed /// /// @note It is safe to call Shutdown multiple times /// /// @note @ref NYT::TApiUsageError will be thrown if any client's method is called after shutdown /// virtual void Shutdown() = 0; }; /// Create a client for particular MapReduce cluster. IClientPtr CreateClient( const TString& serverName, const TCreateClientOptions& options = TCreateClientOptions()); /// Create a client for mapreduce cluster specified in `YT_PROXY` environment variable. IClientPtr CreateClientFromEnv( const TCreateClientOptions& options = TCreateClientOptions()); //////////////////////////////////////////////////////////////////////////////// } // namespace NYT