collections.proto 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762
  1. syntax = "proto3";
  2. package qdrant;
  3. option csharp_namespace = "Qdrant.Client.Grpc";
  4. enum Datatype {
  5. Default = 0;
  6. Float32 = 1;
  7. Uint8 = 2;
  8. Float16 = 3;
  9. }
  10. message VectorParams {
  11. uint64 size = 1; // Size of the vectors
  12. Distance distance = 2; // Distance function used for comparing vectors
  13. optional HnswConfigDiff hnsw_config = 3; // Configuration of vector HNSW graph. If omitted - the collection configuration will be used
  14. optional QuantizationConfig quantization_config = 4; // Configuration of vector quantization config. If omitted - the collection configuration will be used
  15. optional bool on_disk = 5; // If true - serve vectors from disk. If set to false, the vectors will be loaded in RAM.
  16. optional Datatype datatype = 6; // Data type of the vectors
  17. optional MultiVectorConfig multivector_config = 7; // Configuration for multi-vector search
  18. }
  19. message VectorParamsDiff {
  20. optional HnswConfigDiff hnsw_config = 1; // Update params for HNSW index. If empty object - it will be unset
  21. optional QuantizationConfigDiff quantization_config = 2; // Update quantization params. If none - it is left unchanged.
  22. optional bool on_disk = 3; // If true - serve vectors from disk. If set to false, the vectors will be loaded in RAM.
  23. }
  24. message VectorParamsMap {
  25. map<string, VectorParams> map = 1;
  26. }
  27. message VectorParamsDiffMap {
  28. map<string, VectorParamsDiff> map = 1;
  29. }
  30. message VectorsConfig {
  31. oneof config {
  32. VectorParams params = 1;
  33. VectorParamsMap params_map = 2;
  34. }
  35. }
  36. message VectorsConfigDiff {
  37. oneof config {
  38. VectorParamsDiff params = 1;
  39. VectorParamsDiffMap params_map = 2;
  40. }
  41. }
  42. enum Modifier {
  43. None = 0;
  44. Idf = 1; // Apply Inverse Document Frequency
  45. }
  46. message SparseVectorParams {
  47. optional SparseIndexConfig index = 1; // Configuration of sparse index
  48. optional Modifier modifier = 2; // If set - apply modifier to the vector values
  49. }
  50. message SparseVectorConfig {
  51. map<string, SparseVectorParams> map = 1;
  52. }
  53. enum MultiVectorComparator {
  54. MaxSim = 0;
  55. }
  56. message MultiVectorConfig {
  57. MultiVectorComparator comparator = 1; // Comparator for multi-vector search
  58. }
  59. message GetCollectionInfoRequest {
  60. string collection_name = 1; // Name of the collection
  61. }
  62. message CollectionExistsRequest {
  63. string collection_name = 1;
  64. }
  65. message CollectionExists {
  66. bool exists = 1;
  67. }
  68. message CollectionExistsResponse {
  69. CollectionExists result = 1;
  70. double time = 2; // Time spent to process
  71. }
  72. message ListCollectionsRequest {}
  73. message CollectionDescription {
  74. string name = 1; // Name of the collection
  75. }
  76. message GetCollectionInfoResponse {
  77. CollectionInfo result = 1;
  78. double time = 2; // Time spent to process
  79. }
  80. message ListCollectionsResponse {
  81. repeated CollectionDescription collections = 1;
  82. double time = 2; // Time spent to process
  83. }
  84. enum Distance {
  85. UnknownDistance = 0;
  86. Cosine = 1;
  87. Euclid = 2;
  88. Dot = 3;
  89. Manhattan = 4;
  90. }
  91. enum CollectionStatus {
  92. UnknownCollectionStatus = 0;
  93. Green = 1; // All segments are ready
  94. Yellow = 2; // Optimization in process
  95. Red = 3; // Something went wrong
  96. Grey = 4; // Optimization is pending
  97. }
  98. enum PayloadSchemaType {
  99. UnknownType = 0;
  100. Keyword = 1;
  101. Integer = 2;
  102. Float = 3;
  103. Geo = 4;
  104. Text = 5;
  105. Bool = 6;
  106. Datetime = 7;
  107. Uuid = 8;
  108. }
  109. enum QuantizationType {
  110. UnknownQuantization = 0;
  111. Int8 = 1;
  112. }
  113. enum CompressionRatio {
  114. x4 = 0;
  115. x8 = 1;
  116. x16 = 2;
  117. x32 = 3;
  118. x64 = 4;
  119. }
  120. message MaxOptimizationThreads {
  121. enum Setting {
  122. Auto = 0;
  123. }
  124. oneof variant {
  125. uint64 value = 1;
  126. Setting setting = 2;
  127. }
  128. }
  129. message OptimizerStatus {
  130. bool ok = 1;
  131. string error = 2;
  132. }
  133. message HnswConfigDiff {
  134. /*
  135. Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.
  136. */
  137. optional uint64 m = 1;
  138. /*
  139. Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build the index.
  140. */
  141. optional uint64 ef_construct = 2;
  142. /*
  143. Minimal size (in KiloBytes) of vectors for additional payload-based indexing.
  144. If the payload chunk is smaller than `full_scan_threshold` additional indexing won't be used -
  145. in this case full-scan search should be preferred by query planner and additional indexing is not required.
  146. Note: 1 Kb = 1 vector of size 256
  147. */
  148. optional uint64 full_scan_threshold = 3;
  149. /*
  150. Number of parallel threads used for background index building.
  151. If 0 - automatically select from 8 to 16.
  152. Best to keep between 8 and 16 to prevent likelihood of building broken/inefficient HNSW graphs.
  153. On small CPUs, less threads are used.
  154. */
  155. optional uint64 max_indexing_threads = 4;
  156. /*
  157. Store HNSW index on disk. If set to false, the index will be stored in RAM.
  158. */
  159. optional bool on_disk = 5;
  160. /*
  161. Number of additional payload-aware links per node in the index graph. If not set - regular M parameter will be used.
  162. */
  163. optional uint64 payload_m = 6;
  164. }
  165. message SparseIndexConfig {
  166. /*
  167. Prefer a full scan search upto (excluding) this number of vectors.
  168. Note: this is number of vectors, not KiloBytes.
  169. */
  170. optional uint64 full_scan_threshold = 1;
  171. /*
  172. Store inverted index on disk. If set to false, the index will be stored in RAM.
  173. */
  174. optional bool on_disk = 2;
  175. /*
  176. Datatype used to store weights in the index.
  177. */
  178. optional Datatype datatype = 3;
  179. }
  180. message WalConfigDiff {
  181. optional uint64 wal_capacity_mb = 1; // Size of a single WAL block file
  182. optional uint64 wal_segments_ahead = 2; // Number of segments to create in advance
  183. }
  184. message OptimizersConfigDiff {
  185. /*
  186. The minimal fraction of deleted vectors in a segment, required to perform segment optimization
  187. */
  188. optional double deleted_threshold = 1;
  189. /*
  190. The minimal number of vectors in a segment, required to perform segment optimization
  191. */
  192. optional uint64 vacuum_min_vector_number = 2;
  193. /*
  194. Target amount of segments the optimizer will try to keep.
  195. Real amount of segments may vary depending on multiple parameters:
  196. - Amount of stored points.
  197. - Current write RPS.
  198. It is recommended to select the default number of segments as a factor of the number of search threads,
  199. so that each segment would be handled evenly by one of the threads.
  200. */
  201. optional uint64 default_segment_number = 3;
  202. /*
  203. Do not create segments larger this size (in kilobytes).
  204. Large segments might require disproportionately long indexation times,
  205. therefore it makes sense to limit the size of segments.
  206. If indexing speed is more important - make this parameter lower.
  207. If search speed is more important - make this parameter higher.
  208. Note: 1Kb = 1 vector of size 256
  209. If not set, will be automatically selected considering the number of available CPUs.
  210. */
  211. optional uint64 max_segment_size = 4;
  212. /*
  213. Maximum size (in kilobytes) of vectors to store in-memory per segment.
  214. Segments larger than this threshold will be stored as read-only memmapped file.
  215. Memmap storage is disabled by default, to enable it, set this threshold to a reasonable value.
  216. To disable memmap storage, set this to `0`.
  217. Note: 1Kb = 1 vector of size 256
  218. */
  219. optional uint64 memmap_threshold = 5;
  220. /*
  221. Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing
  222. Default value is 20,000, based on <https://github.com/google-research/google-research/blob/master/scann/docs/algorithms.md>.
  223. To disable vector indexing, set to `0`.
  224. Note: 1kB = 1 vector of size 256.
  225. */
  226. optional uint64 indexing_threshold = 6;
  227. /*
  228. Interval between forced flushes.
  229. */
  230. optional uint64 flush_interval_sec = 7;
  231. // Deprecated in favor of `max_optimization_threads`
  232. optional uint64 deprecated_max_optimization_threads = 8;
  233. /*
  234. Max number of threads (jobs) for running optimizations per shard.
  235. Note: each optimization job will also use `max_indexing_threads` threads by itself for index building.
  236. If "auto" - have no limit and choose dynamically to saturate CPU.
  237. If 0 - no optimization threads, optimizations will be disabled.
  238. */
  239. optional MaxOptimizationThreads max_optimization_threads = 9;
  240. }
  241. message ScalarQuantization {
  242. QuantizationType type = 1; // Type of quantization
  243. optional float quantile = 2; // Number of bits to use for quantization
  244. optional bool always_ram = 3; // If true - quantized vectors always will be stored in RAM, ignoring the config of main storage
  245. }
  246. message ProductQuantization {
  247. CompressionRatio compression = 1; // Compression ratio
  248. optional bool always_ram = 2; // If true - quantized vectors always will be stored in RAM, ignoring the config of main storage
  249. }
  250. enum BinaryQuantizationEncoding {
  251. OneBit = 0;
  252. TwoBits = 1;
  253. OneAndHalfBits = 2;
  254. }
  255. message BinaryQuantizationQueryEncoding {
  256. enum Setting {
  257. Default = 0;
  258. Binary = 1;
  259. Scalar4Bits = 2;
  260. Scalar8Bits = 3;
  261. }
  262. oneof variant {
  263. Setting setting = 4;
  264. }
  265. }
  266. message BinaryQuantization {
  267. optional bool always_ram = 1; // If true - quantized vectors always will be stored in RAM, ignoring the config of main storage
  268. optional BinaryQuantizationEncoding encoding = 2; // Binary quantization encoding method
  269. /*
  270. Asymmetric quantization configuration allows a query to have different quantization than stored vectors.
  271. It can increase the accuracy of search at the cost of performance.
  272. */
  273. optional BinaryQuantizationQueryEncoding query_encoding = 3;
  274. }
  275. message QuantizationConfig {
  276. oneof quantization {
  277. ScalarQuantization scalar = 1;
  278. ProductQuantization product = 2;
  279. BinaryQuantization binary = 3;
  280. }
  281. }
  282. message Disabled {}
  283. message QuantizationConfigDiff {
  284. oneof quantization {
  285. ScalarQuantization scalar = 1;
  286. ProductQuantization product = 2;
  287. Disabled disabled = 3;
  288. BinaryQuantization binary = 4;
  289. }
  290. }
  291. enum ShardingMethod {
  292. Auto = 0; // Auto-sharding based on record ids
  293. Custom = 1; // Shard by user-defined key
  294. }
  295. message StrictModeConfig {
  296. optional bool enabled = 1;
  297. optional uint32 max_query_limit = 2;
  298. optional uint32 max_timeout = 3;
  299. optional bool unindexed_filtering_retrieve = 4;
  300. optional bool unindexed_filtering_update = 5;
  301. optional uint32 search_max_hnsw_ef = 6;
  302. optional bool search_allow_exact = 7;
  303. optional float search_max_oversampling = 8;
  304. optional uint64 upsert_max_batchsize = 9;
  305. optional uint64 max_collection_vector_size_bytes = 10;
  306. optional uint32 read_rate_limit = 11; // Max number of read operations per minute per replica
  307. optional uint32 write_rate_limit = 12; // Max number of write operations per minute per replica
  308. optional uint64 max_collection_payload_size_bytes = 13;
  309. optional uint64 filter_max_conditions = 14;
  310. optional uint64 condition_max_size = 15;
  311. optional StrictModeMultivectorConfig multivector_config = 16;
  312. optional StrictModeSparseConfig sparse_config = 17;
  313. optional uint64 max_points_count = 18;
  314. }
  315. message StrictModeSparseConfig {
  316. map<string, StrictModeSparse> sparse_config = 1;
  317. }
  318. message StrictModeSparse {
  319. optional uint64 max_length = 10;
  320. }
  321. message StrictModeMultivectorConfig {
  322. map<string, StrictModeMultivector> multivector_config = 1;
  323. }
  324. message StrictModeMultivector {
  325. optional uint64 max_vectors = 1;
  326. }
  327. message CreateCollection {
  328. string collection_name = 1; // Name of the collection
  329. reserved 2; // Deprecated
  330. reserved 3; // Deprecated
  331. optional HnswConfigDiff hnsw_config = 4; // Configuration of vector index
  332. optional WalConfigDiff wal_config = 5; // Configuration of the Write-Ahead-Log
  333. optional OptimizersConfigDiff optimizers_config = 6; // Configuration of the optimizers
  334. optional uint32 shard_number = 7; // Number of shards in the collection, default is 1 for standalone, otherwise equal to the number of nodes. Minimum is 1
  335. optional bool on_disk_payload = 8; // If true - point's payload will not be stored in memory
  336. optional uint64 timeout = 9; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
  337. optional VectorsConfig vectors_config = 10; // Configuration for vectors
  338. optional uint32 replication_factor = 11; // Number of replicas of each shard that network tries to maintain, default = 1
  339. optional uint32 write_consistency_factor = 12; // How many replicas should apply the operation for us to consider it successful, default = 1
  340. optional string init_from_collection = 13; // Specify name of the other collection to copy data from
  341. optional QuantizationConfig quantization_config = 14; // Quantization configuration of vector
  342. optional ShardingMethod sharding_method = 15; // Sharding method
  343. optional SparseVectorConfig sparse_vectors_config = 16; // Configuration for sparse vectors
  344. optional StrictModeConfig strict_mode_config = 17; // Configuration for strict mode
  345. }
  346. message UpdateCollection {
  347. string collection_name = 1; // Name of the collection
  348. optional OptimizersConfigDiff optimizers_config = 2; // New configuration parameters for the collection. This operation is blocking, it will only proceed once all current optimizations are complete
  349. optional uint64 timeout = 3; // Wait timeout for operation commit in seconds if blocking, if not specified - default value will be supplied
  350. optional CollectionParamsDiff params = 4; // New configuration parameters for the collection
  351. optional HnswConfigDiff hnsw_config = 5; // New HNSW parameters for the collection index
  352. optional VectorsConfigDiff vectors_config = 6; // New vector parameters
  353. optional QuantizationConfigDiff quantization_config = 7; // Quantization configuration of vector
  354. optional SparseVectorConfig sparse_vectors_config = 8; // New sparse vector parameters
  355. optional StrictModeConfig strict_mode_config = 9; // New strict mode configuration
  356. }
  357. message DeleteCollection {
  358. string collection_name = 1; // Name of the collection
  359. optional uint64 timeout = 2; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
  360. }
  361. message CollectionOperationResponse {
  362. bool result = 1; // if operation made changes
  363. double time = 2; // Time spent to process
  364. }
  365. message CollectionParams {
  366. reserved 1; // Deprecated
  367. reserved 2; // Deprecated
  368. uint32 shard_number = 3; // Number of shards in collection
  369. bool on_disk_payload = 4; // If true - point's payload will not be stored in memory
  370. optional VectorsConfig vectors_config = 5; // Configuration for vectors
  371. optional uint32 replication_factor = 6; // Number of replicas of each shard that network tries to maintain
  372. optional uint32 write_consistency_factor = 7; // How many replicas should apply the operation for us to consider it successful
  373. optional uint32 read_fan_out_factor = 8; // Fan-out every read request to these many additional remote nodes (and return first available response)
  374. optional ShardingMethod sharding_method = 9; // Sharding method
  375. optional SparseVectorConfig sparse_vectors_config = 10; // Configuration for sparse vectors
  376. }
  377. message CollectionParamsDiff {
  378. optional uint32 replication_factor = 1; // Number of replicas of each shard that network tries to maintain
  379. optional uint32 write_consistency_factor = 2; // How many replicas should apply the operation for us to consider it successful
  380. optional bool on_disk_payload = 3; // If true - point's payload will not be stored in memory
  381. optional uint32 read_fan_out_factor = 4; // Fan-out every read request to these many additional remote nodes (and return first available response)
  382. }
  383. message CollectionConfig {
  384. CollectionParams params = 1; // Collection parameters
  385. HnswConfigDiff hnsw_config = 2; // Configuration of vector index
  386. OptimizersConfigDiff optimizer_config = 3; // Configuration of the optimizers
  387. WalConfigDiff wal_config = 4; // Configuration of the Write-Ahead-Log
  388. optional QuantizationConfig quantization_config = 5; // Configuration of the vector quantization
  389. optional StrictModeConfig strict_mode_config = 6; // Configuration of strict mode.
  390. }
  391. enum TokenizerType {
  392. Unknown = 0;
  393. Prefix = 1;
  394. Whitespace = 2;
  395. Word = 3;
  396. Multilingual = 4;
  397. }
  398. message KeywordIndexParams {
  399. optional bool is_tenant = 1; // If true - used for tenant optimization.
  400. optional bool on_disk = 2; // If true - store index on disk.
  401. }
  402. message IntegerIndexParams {
  403. optional bool lookup = 1; // If true - support direct lookups. Default is true.
  404. optional bool range = 2; // If true - support ranges filters. Default is true.
  405. optional bool is_principal = 3; // If true - use this key to organize storage of the collection data. This option assumes that this key will be used in majority of filtered requests. Default is false.
  406. optional bool on_disk = 4; // If true - store index on disk. Default is false.
  407. }
  408. message FloatIndexParams {
  409. optional bool on_disk = 1; // If true - store index on disk.
  410. optional bool is_principal = 2; // If true - use this key to organize storage of the collection data. This option assumes that this key will be used in majority of filtered requests.
  411. }
  412. message GeoIndexParams {
  413. optional bool on_disk = 1; // If true - store index on disk.
  414. }
  415. message StopwordsSet {
  416. repeated string languages = 1; // List of languages to use stopwords from
  417. repeated string custom = 2; // List of custom stopwords
  418. }
  419. message TextIndexParams {
  420. TokenizerType tokenizer = 1; // Tokenizer type
  421. optional bool lowercase = 2; // If true - all tokens will be lowercase
  422. optional uint64 min_token_len = 3; // Minimal token length
  423. optional uint64 max_token_len = 4; // Maximal token length
  424. optional bool on_disk = 5; // If true - store index on disk.
  425. optional StopwordsSet stopwords = 6; // Stopwords for the text index
  426. optional bool phrase_matching = 7; // If true - support phrase matching.
  427. optional StemmingAlgorithm stemmer = 8; // Set an algorithm for stemming.
  428. }
  429. message StemmingAlgorithm {
  430. oneof stemming_params {
  431. SnowballParams snowball = 1; // Parameters for snowball stemming
  432. }
  433. }
  434. message SnowballParams {
  435. string language = 1; // Which language the algorithm should stem.
  436. }
  437. message BoolIndexParams {
  438. optional bool on_disk = 1; // If true - store index on disk.
  439. }
  440. message DatetimeIndexParams {
  441. optional bool on_disk = 1; // If true - store index on disk.
  442. optional bool is_principal = 2; // If true - use this key to organize storage of the collection data. This option assumes that this key will be used in majority of filtered requests.
  443. }
  444. message UuidIndexParams {
  445. optional bool is_tenant = 1; // If true - used for tenant optimization.
  446. optional bool on_disk = 2; // If true - store index on disk.
  447. }
  448. message PayloadIndexParams {
  449. oneof index_params {
  450. KeywordIndexParams keyword_index_params = 3; // Parameters for keyword index
  451. IntegerIndexParams integer_index_params = 2; // Parameters for integer index
  452. FloatIndexParams float_index_params = 4; // Parameters for float index
  453. GeoIndexParams geo_index_params = 5; // Parameters for geo index
  454. TextIndexParams text_index_params = 1; // Parameters for text index
  455. BoolIndexParams bool_index_params = 6; // Parameters for bool index
  456. DatetimeIndexParams datetime_index_params = 7; // Parameters for datetime index
  457. UuidIndexParams uuid_index_params = 8; // Parameters for uuid index
  458. }
  459. }
  460. message PayloadSchemaInfo {
  461. PayloadSchemaType data_type = 1; // Field data type
  462. optional PayloadIndexParams params = 2; // Field index parameters
  463. optional uint64 points = 3; // Number of points indexed within this field indexed
  464. }
  465. message CollectionInfo {
  466. CollectionStatus status = 1; // operating condition of the collection
  467. OptimizerStatus optimizer_status = 2; // status of collection optimizers
  468. optional uint64 vectors_count = 3; // Approximate number of vectors in the collection
  469. uint64 segments_count = 4; // Number of independent segments
  470. reserved 5; // Deprecated
  471. reserved 6; // Deprecated
  472. CollectionConfig config = 7; // Configuration
  473. map<string, PayloadSchemaInfo> payload_schema = 8; // Collection data types
  474. optional uint64 points_count = 9; // Approximate number of points in the collection
  475. optional uint64 indexed_vectors_count = 10; // Approximate number of indexed vectors in the collection.
  476. }
  477. message ChangeAliases {
  478. repeated AliasOperations actions = 1; // List of actions
  479. optional uint64 timeout = 2; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
  480. }
  481. message AliasOperations {
  482. oneof action {
  483. CreateAlias create_alias = 1;
  484. RenameAlias rename_alias = 2;
  485. DeleteAlias delete_alias = 3;
  486. }
  487. }
  488. message CreateAlias {
  489. string collection_name = 1; // Name of the collection
  490. string alias_name = 2; // New name of the alias
  491. }
  492. message RenameAlias {
  493. string old_alias_name = 1; // Name of the alias to rename
  494. string new_alias_name = 2; // Name of the alias
  495. }
  496. message DeleteAlias {
  497. string alias_name = 1; // Name of the alias
  498. }
  499. message ListAliasesRequest {}
  500. message ListCollectionAliasesRequest {
  501. string collection_name = 1; // Name of the collection
  502. }
  503. message AliasDescription {
  504. string alias_name = 1; // Name of the alias
  505. string collection_name = 2; // Name of the collection
  506. }
  507. message ListAliasesResponse {
  508. repeated AliasDescription aliases = 1;
  509. double time = 2; // Time spent to process
  510. }
  511. message CollectionClusterInfoRequest {
  512. string collection_name = 1; // Name of the collection
  513. }
  514. enum ReplicaState {
  515. Active = 0; // Active and sound
  516. Dead = 1; // Failed for some reason
  517. Partial = 2; // The shard is partially loaded and is currently receiving data from other shards
  518. Initializing = 3; // Collection is being created
  519. Listener = 4; // A shard which receives data, but is not used for search; Useful for backup shards
  520. PartialSnapshot = 5; // Deprecated: snapshot shard transfer is in progress; Updates should not be sent to (and are ignored by) the shard
  521. Recovery = 6; // Shard is undergoing recovered by an external node; Normally rejects updates, accepts updates if force is true
  522. Resharding = 7; // Points are being migrated to this shard as part of scale-up resharding
  523. ReshardingScaleDown = 8; // Points are being migrated to this shard as part of scale-down resharding
  524. }
  525. message ShardKey {
  526. oneof key {
  527. string keyword = 1; // String key
  528. uint64 number = 2; // Number key
  529. }
  530. }
  531. message LocalShardInfo {
  532. uint32 shard_id = 1; // Local shard id
  533. uint64 points_count = 2; // Number of points in the shard
  534. ReplicaState state = 3; // Is replica active
  535. optional ShardKey shard_key = 4; // User-defined shard key
  536. }
  537. message RemoteShardInfo {
  538. uint32 shard_id = 1; // Local shard id
  539. uint64 peer_id = 2; // Remote peer id
  540. ReplicaState state = 3; // Is replica active
  541. optional ShardKey shard_key = 4; // User-defined shard key
  542. }
  543. message ShardTransferInfo {
  544. uint32 shard_id = 1; // Local shard id
  545. optional uint32 to_shard_id = 5;
  546. uint64 from = 2;
  547. uint64 to = 3;
  548. bool sync = 4; // If `true` transfer is a synchronization of a replicas; If `false` transfer is a moving of a shard from one peer to another
  549. }
  550. message ReshardingInfo {
  551. uint32 shard_id = 1;
  552. uint64 peer_id = 2;
  553. optional ShardKey shard_key = 3;
  554. ReshardingDirection direction = 4;
  555. }
  556. /*
  557. Resharding direction, scale up or down in number of shards
  558. */
  559. enum ReshardingDirection {
  560. Up = 0; // Scale up, add a new shard
  561. Down = 1; // Scale down, remove a shard
  562. }
  563. message CollectionClusterInfoResponse {
  564. uint64 peer_id = 1; // ID of this peer
  565. uint64 shard_count = 2; // Total number of shards
  566. repeated LocalShardInfo local_shards = 3; // Local shards
  567. repeated RemoteShardInfo remote_shards = 4; // Remote shards
  568. repeated ShardTransferInfo shard_transfers = 5; // Shard transfers
  569. repeated ReshardingInfo resharding_operations = 6; // Resharding operations
  570. }
  571. message MoveShard {
  572. uint32 shard_id = 1; // Local shard id
  573. optional uint32 to_shard_id = 5;
  574. uint64 from_peer_id = 2;
  575. uint64 to_peer_id = 3;
  576. optional ShardTransferMethod method = 4;
  577. }
  578. message ReplicateShard {
  579. uint32 shard_id = 1; // Local shard id
  580. optional uint32 to_shard_id = 5;
  581. uint64 from_peer_id = 2;
  582. uint64 to_peer_id = 3;
  583. optional ShardTransferMethod method = 4;
  584. }
  585. message AbortShardTransfer {
  586. uint32 shard_id = 1; // Local shard id
  587. optional uint32 to_shard_id = 4;
  588. uint64 from_peer_id = 2;
  589. uint64 to_peer_id = 3;
  590. }
  591. message RestartTransfer {
  592. uint32 shard_id = 1; // Local shard id
  593. optional uint32 to_shard_id = 5;
  594. uint64 from_peer_id = 2;
  595. uint64 to_peer_id = 3;
  596. ShardTransferMethod method = 4;
  597. }
  598. enum ShardTransferMethod {
  599. StreamRecords = 0; // Stream shard records in batches
  600. Snapshot = 1; // Snapshot the shard and recover it on the target peer
  601. WalDelta = 2; // Resolve WAL delta between peers and transfer the difference
  602. ReshardingStreamRecords = 3; // Stream shard records in batches for resharding
  603. }
  604. message Replica {
  605. uint32 shard_id = 1;
  606. uint64 peer_id = 2;
  607. }
  608. message CreateShardKey {
  609. ShardKey shard_key = 1; // User-defined shard key
  610. optional uint32 shards_number = 2; // Number of shards to create per shard key
  611. optional uint32 replication_factor = 3; // Number of replicas of each shard to create
  612. repeated uint64 placement = 4; // List of peer ids, allowed to create shards. If empty - all peers are allowed
  613. }
  614. message DeleteShardKey {
  615. ShardKey shard_key = 1; // Shard key to delete
  616. }
  617. message UpdateCollectionClusterSetupRequest {
  618. string collection_name = 1; // Name of the collection
  619. oneof operation {
  620. MoveShard move_shard = 2;
  621. ReplicateShard replicate_shard = 3;
  622. AbortShardTransfer abort_transfer = 4;
  623. Replica drop_replica = 5;
  624. CreateShardKey create_shard_key = 7;
  625. DeleteShardKey delete_shard_key = 8;
  626. RestartTransfer restart_transfer = 9;
  627. }
  628. optional uint64 timeout = 6; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
  629. }
  630. message UpdateCollectionClusterSetupResponse {
  631. bool result = 1;
  632. }
  633. message CreateShardKeyRequest {
  634. string collection_name = 1; // Name of the collection
  635. CreateShardKey request = 2; // Request to create shard key
  636. optional uint64 timeout = 3; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
  637. }
  638. message DeleteShardKeyRequest {
  639. string collection_name = 1; // Name of the collection
  640. DeleteShardKey request = 2; // Request to delete shard key
  641. optional uint64 timeout = 3; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
  642. }
  643. message CreateShardKeyResponse {
  644. bool result = 1;
  645. }
  646. message DeleteShardKeyResponse {
  647. bool result = 1;
  648. }