333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843 | class DIAdminClient(DIClient):
"""
DIAdminClient
The `DIAdminClient` class is an extension of the `DIClient` class, designed specifically for administrative operations
within the Data Intelligence (DI) platform. This class provides methods to manage collections, pipelines, schemas,
and their associated resources. It is intended for use cases where administrative privileges are required to perform
operations such as creating, deleting, or modifying collections, pipelines, and schemas.
Purpose:
---------
The `DIAdminClient` is tailored for scenarios where administrative-level API interactions are necessary.
It provides a higher level of control over the DI platform's resources, enabling administrators to configure
and manage the system effectively.
When to Use:
------------
- Use this class when you need to perform administrative tasks such as:
- Creating or deleting collections.
- Assigning or unassigning buckets to/from collections.
- Creating or deleting pipelines.
- Creating or deleting schemas.
- This class is specifically designed for admin APIs. For non-admin APIs, use the `DIClient` class instead.
Initialization:
---------------
The `DIAdminClient` requires authentication credentials (username and password) to establish an authenticated session
with the DI platform. Upon initialization, it creates an authenticated session that is used for all subsequent API calls.
--------
"""
def __init__(self, *, uri: str, username: str, password: str) -> None:
super().__init__(uri=uri)
# create session with auth
self._authenticated_session = AuthAPI.login(
uri=uri, username=username, password=password
)
@property
def authenticated_session(self) -> AuthenticatedSession:
"""
Property to get the authenticated session object.
Returns:
AuthenticatedSession: The authenticated session object used for making API requests.This session is initialized with the provided URI, username, password, and token.
"""
return self._authenticated_session
def create_collection(
self,
*,
name: str,
pipeline: str,
buckets: Optional[List[str]] = None,
output_store: Optional[str] = None,
indexing_mode: Optional[str] = None,
) -> V1CollectionResponse:
"""
Creates a new collection using the specified pipeline.
If buckets are provided, they will be associated with the collection and inline embedding generation will be triggered.
If buckets are not provided, the collection will be created but embedding generation will be deferred until buckets are assigned to the collection.
Args:
name (str): The name of the collection to be created. This should be unique.
pipeline (str): The name of the pipeline to be associated with the collection.
buckets (Optional[List[str]], optional): A list of bucket names. Defaults to None.
output_store (Optional[str], optional): Output bucket for transcription results. Defaults to None.
indexing_mode (Optional[str], optional): Indexing mode for RAG collections (supported values: 'HNSW', 'GPU_CAGRA'). Auto-detected if omitted. Defaults to None.
Returns:
V1CollectionResponse: The created collection object.
Example usage:
```python
# Example usage of create_collection
client = DIAdminClient(uri="https://example.com", username="admin", password="password")
collection = client.create_collection(
name="example_collection",
pipeline="data_ingestion_pipeline",
buckets=["bucket1", "bucket2"],
output_store="output-bucket"
)
print(collection)
# Sample Output:
# V1CollectionResponse(
# name="example_collection",
# pipeline="data_ingestion_pipeline",
# buckets=["bucket1", "bucket2"],
# outputStore="output-bucket"
# )
```
"""
if buckets is None:
buckets = []
return CollectionAPI(session=self.authenticated_session).create_collection(
name=name,
buckets=buckets,
pipeline=pipeline,
output_store=output_store,
indexing_mode=indexing_mode,
)
def delete_collection(self, *, name: str) -> V1DeleteCollectionResponse:
"""
Deletes a collection by its name.
Args:
name (str): The name of the collection to be deleted.
Returns:
V1DeleteCollectionResponse: The response object containing details about the deleted collection.
Example Usage:
```python
client = DIAdminClient(uri="https://example.com", username="admin", password="password")
resp = client.delete_collection(name="example_collection")
print(resp)
# Output:
# V1DeleteCollectionResponse(
# success=True,
# message="Collection 'example_collection' has been deleted."
# )
```
"""
return CollectionAPI(session=self.authenticated_session).delete_collection(
name=name
)
def assign_buckets_to_collection(
self, *, collection_name: str, buckets: List[str]
) -> BucketUpdateResponse:
"""
Assigns a list of buckets to a specified collection.
This enables inline embedding generation for the specified buckets.
Args:
collection_name (str): The name of the collection to which the buckets
will be assigned.
buckets (List[str]): A list of bucket names to be assigned to the
specified collection.
Returns:
BucketUpdateResponse: The response object containing details about the
updated collection and assigned buckets.
Example Usage:
```python
# Initialize the DIAdminClient
client = DIAdminClient(uri="http://example.com", username="admin", password="password")
# Define the collection name and buckets
collection_name = "my_collection"
buckets = ["bucket1", "bucket2", "bucket3"]
# Assign buckets to the collection
response = client.assign_buckets_to_collection(
collection_name=collection_name,
buckets=buckets
)
print(response)
# Output:
# BucketUpdateResponse(
# success=True,
# message="Buckets assigned successfully to collection 'my_collection'."
# )
```
Notes:
- This method is typically used for enabling the user buckets for intelligence using an existing collection.
"""
return CollectionAPI(
session=self.authenticated_session
).assign_buckets_to_collection(collection_name=collection_name, buckets=buckets)
def unassign_buckets_from_collection(
self, *, collection_name: str, buckets: List[str]
) -> BucketUpdateResponse:
"""
Unassigns one or more buckets from a specified collection.
Args:
collection_name (str): The name of the collection from which the buckets will be unassigned.
buckets (List[str]): A list of bucket names to be unassigned.
Returns:
BucketUpdateResponse: The response object containing details about the updated collection
after the buckets have been unassigned.
Example usage:
```python
# Example usage of unassign_buckets_from_collection
client = DIAdminClient(uri="http://example.com", username="admin", password="password")
# Unassign multiple buckets
response = client.unassign_buckets_from_collection(
collection_name="example_collection",
buckets=["bucket_1", "bucket_2", "bucket_3"]
)
print(response)
# Output:
# BucketUpdateResponse(
# success=True,
# message="Buckets unassigned successfully from collection 'example_collection'."
# )
```
"""
return CollectionAPI(
session=self.authenticated_session
).unassign_buckets_from_collection(
collection_name=collection_name, buckets=buckets
)
@normalize_pipeline_argument_errors
def create_pipeline(
self,
*,
name: str,
pipeline_type: str,
event_filter_object_suffix: List[str],
schema: str,
event_filter_max_object_size: Optional[int] = None,
model: Optional[str] = None,
custom_func: Optional[str] = None,
prompt: Optional[str] = None,
chunk_size: Optional[int] = None,
chunk_overlap: Optional[int] = None,
) -> V1CreatePipelineResponse:
"""
Creates a new pipeline with the specified configuration.
Args:
name (str): The name of the pipeline to be created.
pipeline_type (str): The type of the pipeline ("rag", "metadata", "transcribe-metadata" or "custom-function").
event_filter_object_suffix (List[str]): A list of file suffixes to filter events. Ex - ["*.txt", "*.pdf"]
schema (str): The schema definition for the pipeline.
event_filter_max_object_size Optional (int): The maximum object size for event filtering. Ex - 10485760
model Optional (str): The model associated with the pipeline. Required for "rag" and "custom-function" pipelines.
custom_func Optional (str): The custom function to be used in the pipeline. Required for "metadata" pipelines.
prompt Optional (str): The prompt for transcribe pipelines (e.g., "Transcribe the image content into text.").
chunk_size Optional (int): Chunk size for RAG pipelines.
chunk_overlap Optional (int): Chunk overlap for RAG pipelines.
Returns:
V1CreatePipelineResponse: The response object containing details of the created pipeline.
Raises:
PipelineValidationError: If the arguments are invalid or incomplete, or if the server rejects
the request with an HTTP 422 validation error.
Example usage:
```python
client = DIAdminClient(
uri="http://example.com",
username="admin",
password="password"
)
pipeline_data = client.create_pipeline(
name="example_pipeline",
pipeline_type="rag",
model="example_model",
event_filter_object_suffix=["*.txt", "*.pdf"],
event_filter_max_object_size=10485760,
schema="example_schema"
)
print(pipeline_data)
# Output: V1CreatePipelineResponse(
# success=true,
# message="Pipeline 'example_pipeline' created successfully."
# )
```
"""
return PipelineAPI(session=self.authenticated_session).create_pipeline(
name=name,
pipeline_type=pipeline_type,
model=model,
custom_func=custom_func,
event_filter_object_suffix=event_filter_object_suffix,
event_filter_max_object_size=event_filter_max_object_size,
schema=schema,
prompt=prompt,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
)
def delete_pipeline(self, *, name: str) -> V1DeletePipelineResponse:
"""
Deletes a pipeline with the specified name.
Args:
name (str): The name of the pipeline to be deleted.
Returns:
V1DeletePipelineResponse: The response object containing details about the deleted pipeline.
Example usage:
```python
# Initialize the DIAdminClient
client = DIAdminClient(uri="http://example.com", username="admin", password="password")
# Delete a pipeline by name
response = client.delete_pipeline(name="example_pipeline")
print(response)
# Output:
# V1DeletePipelineResponse(
# message="Pipeline successfully deleted",
# success=True,
# )
```
"""
return PipelineAPI(session=self.authenticated_session).delete_pipeline(
name=name
)
def get_schema(self, *, name: str) -> V1SchemasResponse:
"""
Retrieve a schema by its name.
This method fetches a schema object from the SchemaAPI using the provided name.
Args:
name (str): The name of the schema to retrieve. This is a required keyword-only argument.
Returns:
V1SchemasResponse: The response object containing details about the schema.
Example usage:
```python
client = DIAdminClient(uri="https://example.com", username="admin", password="password")
schema = client.get_schema(name="example_schema")
print(schema)
# Output: V1SchemasResponse(
# name="example_schema",
# type="...",
# schema_fields=[SchemaItem(name="id", type="varchar", nullable=False)]
# )
```
Note:
The schema fields are exposed as the ``schema_fields`` attribute because ``schema`` collides
with a reserved Pydantic attribute. They are serialized and deserialized as ``schema``.
"""
return SchemaAPI(session=self.authenticated_session).get_schema(name=name)
def get_all_schemas(self) -> V1ListSchemasResponse:
"""
Retrieves all schemas available in the system.
Returns:
V1ListSchemasResponse: A response object containing a list of
schemas available in the system.
Example usage:
```python
client = DIAdminClient(uri="https://example.com", username="admin", password="password")
schemas = client.get_all_schemas()
print(schemas)
# Output: V1ListSchemasResponse(
# schemas=[SchemaListItem(id="1", name="example_schema")]
# )
```
"""
return SchemaAPI(session=self.authenticated_session).get_schemas()
@deprecated(
message="This method is deprecated and will be removed in future versions. Please use get_model() instead."
)
def get_embedding_model(self, *, name: str) -> V1ModelsResponse:
"""
Retrieve an embedding model by its name.
This method fetches an embedding model object from the ModelAPI using the provided name.
.. Deprecated::
This method is deprecated and will be removed in future versions. Please use `get_model` instead.
Args:
name (str): The name of the embedding model to retrieve. This is a required keyword-only argument.
Returns:
V1ModelsResponse: The response object containing details about the embedding model.
Example usage:
```python
client = DIAdminClient(uri="https://example.com", username="admin", password="password")
model = client.get_embedding_model(name="example_model")
print(model)
# Output: V1ModelsResponse(
# name="example_model",
# modelName="...",
# capabilities=["Sentence-Similarity"],
# version="...",
# dimension=..., # e.g., 768
# maximumTokens=..., # e.g., 512
# )
```
"""
return ModelAPI(self.authenticated_session).get_model(name=name)
@deprecated(
message="This method is deprecated and will be removed in future versions. Please use get_all_models() instead."
)
def get_all_embedding_models(self) -> V1ListModelsResponse:
"""
Retrieves all embedding models available in the system.
.. Deprecated::
This method is deprecated and will be removed in future versions. Please use `get_all_models` instead.
Returns:
V1ListModelsResponse: A response object containing a list of
embedding models available in the system.
Example usage:
```python
client = DIAdminClient(uri="https://example.com", username="admin", password="password")
models = client.get_all_embedding_models()
print(models)
# Output: V1ListModelsResponse(
# models=[ModelRecordSummary(id="1", name="example_embedding_model")]
# )
```
"""
try:
model_api = ModelAPI(self.authenticated_session)
models = model_api.get_models()
embedding_models_list = list()
for model in models.models:
model_details = model_api.get_model(name=model.name)
if any(
capability.lower() == ModelTags.SENTENCE_SIMILARITY.value.lower()
for capability in model_details.capabilities
):
embedding_models_list.append(model)
except (UnexpectedResponse, UnexpectedStatus) as e:
raise e
return V1ListModelsResponse(models=embedding_models_list)
def create_schema(
self,
*,
name: str,
schema_type: str,
schema: List[SchemaItem],
) -> V1CreateSchemaResponse:
"""
Create a new schema.
Args:
name (str): The name of the schema to create.
schema_type (str): The schema type (e.g., 'custom-function').
schema (List[SchemaItem]): List of schema fields. Each field has a
``name`` and ``type``, and may optionally include a ``nullable``
flag (defaults to ``True``). Set ``nullable`` to ``False`` to
mark a field as required: the underlying database column is created
as ``NOT NULL`` and schema validation is strict for that field,
while fields left as ``nullable=True`` are validated non-strictly.
Returns:
V1CreateSchemaResponse: Response indicating success or failure.
Example usage:
```python
schema_response = client.create_schema(
name="yolo-detection-schema",
schema_type="custom-function",
schema=[{"name": "id", "type": "varchar", "nullable": False},
{"name": "content", "type": "varchar"},
{"name": "embedding", "type": "array(real)", "nullable": True},
]
)
# "id" is required (NOT NULL, strict validation); "content" and
# "embedding" allow nulls (nullable defaults to True when omitted).
```
"""
return SchemaAPI(session=self.authenticated_session).create_schema(
name=name,
schema_type=schema_type,
schema=schema,
)
def delete_schema(self, *, name: str) -> V1DeleteSchemaResponse:
"""
Deletes a schema with the specified name.
Args:
name (str): The name of the schema to be deleted.
Returns:
V1DeleteSchemaResponse: The response object containing details about the deleted schema.
Example usage:
```python
# Initialize the DIAdminClient
client = DIAdminClient(uri="http://example.com", username="admin", password="password")
# Delete a schema by name
response = client.delete_schema(name="example_schema")
print(response)
# Output:
# V1DeleteSchemaResponse(
# message="Schema successfully deleted"
# success=True,
# )
```
"""
return SchemaAPI(session=self.authenticated_session).delete_schema(name=name)
|