filer.proto 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483
  1. syntax = "proto3";
  2. package filer_pb;
  3. option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb";
  4. option java_package = "seaweedfs.client";
  5. option java_outer_classname = "FilerProto";
  6. //////////////////////////////////////////////////
  7. service SeaweedFiler {
  8. rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) {
  9. }
  10. rpc ListEntries (ListEntriesRequest) returns (stream ListEntriesResponse) {
  11. }
  12. rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
  13. }
  14. rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
  15. }
  16. rpc AppendToEntry (AppendToEntryRequest) returns (AppendToEntryResponse) {
  17. }
  18. rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
  19. }
  20. rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
  21. }
  22. rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
  23. }
  24. rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
  25. }
  26. rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
  27. }
  28. rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
  29. }
  30. rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
  31. }
  32. rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
  33. }
  34. rpc Ping (PingRequest) returns (PingResponse) {
  35. }
  36. rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
  37. }
  38. rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
  39. }
  40. rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
  41. }
  42. rpc KvGet (KvGetRequest) returns (KvGetResponse) {
  43. }
  44. rpc KvPut (KvPutRequest) returns (KvPutResponse) {
  45. }
  46. rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
  47. }
  48. rpc DistributedLock(LockRequest) returns (LockResponse) {
  49. }
  50. rpc DistributedUnlock(UnlockRequest) returns (UnlockResponse) {
  51. }
  52. rpc FindLockOwner(FindLockOwnerRequest) returns (FindLockOwnerResponse) {
  53. }
  54. // distributed lock management internal use only
  55. rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
  56. }
  57. }
  58. //////////////////////////////////////////////////
  59. message LookupDirectoryEntryRequest {
  60. string directory = 1;
  61. string name = 2;
  62. }
  63. message LookupDirectoryEntryResponse {
  64. Entry entry = 1;
  65. }
  66. message ListEntriesRequest {
  67. string directory = 1;
  68. string prefix = 2;
  69. string startFromFileName = 3;
  70. bool inclusiveStartFrom = 4;
  71. uint32 limit = 5;
  72. }
  73. message ListEntriesResponse {
  74. Entry entry = 1;
  75. }
  76. message RemoteEntry {
  77. string storage_name = 1;
  78. int64 last_local_sync_ts_ns = 2;
  79. string remote_e_tag = 3;
  80. int64 remote_mtime = 4;
  81. int64 remote_size = 5;
  82. }
  83. message Entry {
  84. string name = 1;
  85. bool is_directory = 2;
  86. repeated FileChunk chunks = 3;
  87. FuseAttributes attributes = 4;
  88. map<string, bytes> extended = 5;
  89. bytes hard_link_id = 7;
  90. int32 hard_link_counter = 8; // only exists in hard link meta data
  91. bytes content = 9; // if not empty, the file content
  92. RemoteEntry remote_entry = 10;
  93. int64 quota = 11; // for bucket only. Positive/Negative means enabled/disabled.
  94. }
  95. message FullEntry {
  96. string dir = 1;
  97. Entry entry = 2;
  98. }
  99. message EventNotification {
  100. Entry old_entry = 1;
  101. Entry new_entry = 2;
  102. bool delete_chunks = 3;
  103. string new_parent_path = 4;
  104. bool is_from_other_cluster = 5;
  105. repeated int32 signatures = 6;
  106. }
  107. message FileChunk {
  108. string file_id = 1; // to be deprecated
  109. int64 offset = 2;
  110. uint64 size = 3;
  111. int64 modified_ts_ns = 4;
  112. string e_tag = 5;
  113. string source_file_id = 6; // to be deprecated
  114. FileId fid = 7;
  115. FileId source_fid = 8;
  116. bytes cipher_key = 9;
  117. bool is_compressed = 10;
  118. bool is_chunk_manifest = 11; // content is a list of FileChunks
  119. }
  120. message FileChunkManifest {
  121. repeated FileChunk chunks = 1;
  122. }
  123. message FileId {
  124. uint32 volume_id = 1;
  125. uint64 file_key = 2;
  126. fixed32 cookie = 3;
  127. }
  128. message FuseAttributes {
  129. uint64 file_size = 1;
  130. int64 mtime = 2; // unix time in seconds
  131. uint32 file_mode = 3;
  132. uint32 uid = 4;
  133. uint32 gid = 5;
  134. int64 crtime = 6; // unix time in seconds
  135. string mime = 7;
  136. int32 ttl_sec = 10;
  137. string user_name = 11; // for hdfs
  138. repeated string group_name = 12; // for hdfs
  139. string symlink_target = 13;
  140. bytes md5 = 14;
  141. uint32 rdev = 16;
  142. uint64 inode = 17;
  143. }
  144. message CreateEntryRequest {
  145. string directory = 1;
  146. Entry entry = 2;
  147. bool o_excl = 3;
  148. bool is_from_other_cluster = 4;
  149. repeated int32 signatures = 5;
  150. bool skip_check_parent_directory = 6;
  151. }
  152. message CreateEntryResponse {
  153. string error = 1;
  154. }
  155. message UpdateEntryRequest {
  156. string directory = 1;
  157. Entry entry = 2;
  158. bool is_from_other_cluster = 3;
  159. repeated int32 signatures = 4;
  160. }
  161. message UpdateEntryResponse {
  162. }
  163. message AppendToEntryRequest {
  164. string directory = 1;
  165. string entry_name = 2;
  166. repeated FileChunk chunks = 3;
  167. }
  168. message AppendToEntryResponse {
  169. }
  170. message DeleteEntryRequest {
  171. string directory = 1;
  172. string name = 2;
  173. // bool is_directory = 3;
  174. bool is_delete_data = 4;
  175. bool is_recursive = 5;
  176. bool ignore_recursive_error = 6;
  177. bool is_from_other_cluster = 7;
  178. repeated int32 signatures = 8;
  179. }
  180. message DeleteEntryResponse {
  181. string error = 1;
  182. }
  183. message AtomicRenameEntryRequest {
  184. string old_directory = 1;
  185. string old_name = 2;
  186. string new_directory = 3;
  187. string new_name = 4;
  188. repeated int32 signatures = 5;
  189. }
  190. message AtomicRenameEntryResponse {
  191. }
  192. message StreamRenameEntryRequest {
  193. string old_directory = 1;
  194. string old_name = 2;
  195. string new_directory = 3;
  196. string new_name = 4;
  197. repeated int32 signatures = 5;
  198. }
  199. message StreamRenameEntryResponse {
  200. string directory = 1;
  201. EventNotification event_notification = 2;
  202. int64 ts_ns = 3;
  203. }
  204. message AssignVolumeRequest {
  205. int32 count = 1;
  206. string collection = 2;
  207. string replication = 3;
  208. int32 ttl_sec = 4;
  209. string data_center = 5;
  210. string path = 6;
  211. string rack = 7;
  212. string data_node = 9;
  213. string disk_type = 8;
  214. }
  215. message AssignVolumeResponse {
  216. string file_id = 1;
  217. int32 count = 4;
  218. string auth = 5;
  219. string collection = 6;
  220. string replication = 7;
  221. string error = 8;
  222. Location location = 9;
  223. }
  224. message LookupVolumeRequest {
  225. repeated string volume_ids = 1;
  226. }
  227. message Locations {
  228. repeated Location locations = 1;
  229. }
  230. message Location {
  231. string url = 1;
  232. string public_url = 2;
  233. uint32 grpc_port = 3;
  234. string data_center = 4;
  235. }
  236. message LookupVolumeResponse {
  237. map<string, Locations> locations_map = 1;
  238. }
  239. message Collection {
  240. string name = 1;
  241. }
  242. message CollectionListRequest {
  243. bool include_normal_volumes = 1;
  244. bool include_ec_volumes = 2;
  245. }
  246. message CollectionListResponse {
  247. repeated Collection collections = 1;
  248. }
  249. message DeleteCollectionRequest {
  250. string collection = 1;
  251. }
  252. message DeleteCollectionResponse {
  253. }
  254. message StatisticsRequest {
  255. string replication = 1;
  256. string collection = 2;
  257. string ttl = 3;
  258. string disk_type = 4;
  259. }
  260. message StatisticsResponse {
  261. uint64 total_size = 4;
  262. uint64 used_size = 5;
  263. uint64 file_count = 6;
  264. }
  265. message PingRequest {
  266. string target = 1; // default to ping itself
  267. string target_type = 2;
  268. }
  269. message PingResponse {
  270. int64 start_time_ns = 1;
  271. int64 remote_time_ns = 2;
  272. int64 stop_time_ns = 3;
  273. }
  274. message GetFilerConfigurationRequest {
  275. }
  276. message GetFilerConfigurationResponse {
  277. repeated string masters = 1;
  278. string replication = 2;
  279. string collection = 3;
  280. uint32 max_mb = 4;
  281. string dir_buckets = 5;
  282. bool cipher = 7;
  283. int32 signature = 8;
  284. string metrics_address = 9;
  285. int32 metrics_interval_sec = 10;
  286. string version = 11;
  287. string cluster_id = 12;
  288. string filer_group = 13;
  289. }
  290. message SubscribeMetadataRequest {
  291. string client_name = 1;
  292. string path_prefix = 2;
  293. int64 since_ns = 3;
  294. int32 signature = 4;
  295. repeated string path_prefixes = 6;
  296. int32 client_id = 7;
  297. int64 until_ns = 8;
  298. int32 client_epoch = 9;
  299. repeated string directories = 10; // exact directory to watch
  300. }
  301. message SubscribeMetadataResponse {
  302. string directory = 1;
  303. EventNotification event_notification = 2;
  304. int64 ts_ns = 3;
  305. }
  306. message LogEntry {
  307. int64 ts_ns = 1;
  308. int32 partition_key_hash = 2;
  309. bytes data = 3;
  310. }
  311. message KeepConnectedRequest {
  312. string name = 1;
  313. uint32 grpc_port = 2;
  314. repeated string resources = 3;
  315. }
  316. message KeepConnectedResponse {
  317. }
  318. message LocateBrokerRequest {
  319. string resource = 1;
  320. }
  321. message LocateBrokerResponse {
  322. bool found = 1;
  323. // if found, send the exact address
  324. // if not found, send the full list of existing brokers
  325. message Resource {
  326. string grpc_addresses = 1;
  327. int32 resource_count = 2;
  328. }
  329. repeated Resource resources = 2;
  330. }
  331. /////////////////////////
  332. // Key-Value operations
  333. /////////////////////////
  334. message KvGetRequest {
  335. bytes key = 1;
  336. }
  337. message KvGetResponse {
  338. bytes value = 1;
  339. string error = 2;
  340. }
  341. message KvPutRequest {
  342. bytes key = 1;
  343. bytes value = 2;
  344. }
  345. message KvPutResponse {
  346. string error = 1;
  347. }
  348. /////////////////////////
  349. // path-based configurations
  350. /////////////////////////
  351. message FilerConf {
  352. int32 version = 1;
  353. message PathConf {
  354. string location_prefix = 1;
  355. string collection = 2;
  356. string replication = 3;
  357. string ttl = 4;
  358. string disk_type = 5;
  359. bool fsync = 6;
  360. uint32 volume_growth_count = 7;
  361. bool read_only = 8;
  362. string data_center = 9;
  363. string rack = 10;
  364. string data_node = 11;
  365. }
  366. repeated PathConf locations = 2;
  367. }
  368. /////////////////////////
  369. // Remote Storage related
  370. /////////////////////////
  371. message CacheRemoteObjectToLocalClusterRequest {
  372. string directory = 1;
  373. string name = 2;
  374. }
  375. message CacheRemoteObjectToLocalClusterResponse {
  376. Entry entry = 1;
  377. }
  378. /////////////////////////
  379. // distributed lock management
  380. /////////////////////////
  381. message LockRequest {
  382. string name = 1;
  383. int64 seconds_to_lock = 2;
  384. string renew_token = 3;
  385. bool is_moved = 4;
  386. string owner = 5;
  387. }
  388. message LockResponse {
  389. string renew_token = 1;
  390. string moved_to = 2;
  391. string error = 3;
  392. }
  393. message UnlockRequest {
  394. string name = 1;
  395. string renew_token = 2;
  396. bool is_moved = 3;
  397. }
  398. message UnlockResponse {
  399. string error = 1;
  400. string moved_to = 2;
  401. }
  402. message FindLockOwnerRequest {
  403. string name = 1;
  404. bool is_moved = 2;
  405. }
  406. message FindLockOwnerResponse {
  407. string owner = 1;
  408. }
  409. message Lock {
  410. string name = 1;
  411. string renew_token = 2;
  412. int64 expired_at_ns = 3;
  413. string owner = 4;
  414. }
  415. message TransferLocksRequest {
  416. repeated Lock locks = 1;
  417. }
  418. message TransferLocksResponse {
  419. }