Use Snapshot as Data Source

You can create an external collection from a Milvus snapshot by using the snapshot metadata JSON path as external_source and setting external_spec.format to "milvus-table".

After you refresh the external collection, Milvus maps the source segment manifests into target external segments. The main column data remains referenced from the snapshot source, while target-owned data such as generated function outputs and converted delete logs is written under the target collection.

Before you start

Before you create the external collection, make sure that

• the source snapshot was created from a collection in a Zilliz Cloud cluster compatible with Milvus v3.0.x server;

• the source collection is not itself an external collection;

• external_source points to the concrete snapshot metadata JSON file, for example s3://bucket/snapshots/{source_collection_id}/metadata/{snapshot_id}.json;

• each target data field sets external_field to the matching source field name; and

• the target schema matches the source snapshot schema for mapped data fields.

If the source snapshot is not from a collection created on a Zilliz Cloud cluster compatible with Milvus instance of the v3.0.x release, the source schema does not match the target schema, the snapshot metadata JSON is missing manifest information, or the source is another external collection, creation or refresh fails.

Create an external collection from a Milvus snapshot

In the following example, you will create a milvus-table external collection from an existing snapshot and refreshes it.

Step 1: Get the snapshot metadata path

Create or choose a snapshot from a normal Milvus collection, and then describe it to get its object-storage location.

Python

python

from pymilvus import DataType, MilvusClient

client = MilvusClient(
    uri="YOUR_CLUSTER_ENDPOINT",
    token="YOUR_CLUSTER_TOKEN",
)

snapshot_info = client.describe_snapshot(
    snapshot_name="analytics_snapshot_20260321",
    collection_name="analytics",
)
external_source = f"s3://bucket/{snapshot_info.s3_location}"

Java

java

import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.service.snapshot.request.DescribeSnapshotReq;
import io.milvus.v2.service.snapshot.response.DescribeSnapshotResp;

ConnectConfig connectConfig = ConnectConfig.builder()
        .uri("YOUR_CLUSTER_ENDPOINT")
        .token("YOUR_CLUSTER_TOKEN")
        .build();
MilvusClientV2 client = new MilvusClientV2(connectConfig);

DescribeSnapshotResp snapshotInfo = client.describeSnapshot(DescribeSnapshotReq.builder()
        .snapshotName("analytics_snapshot_20260321")
        .collectionName("analytics")
        .build());

String externalSource = "s3://bucket/" + snapshotInfo.getS3Location();

Go

go

import (
    "context"
    "fmt"
    "log"

    "github.com/milvus-io/milvus/client/v3/milvusclient"
)

client, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
    Address: "YOUR_CLUSTER_ENDPOINT",
    APIKey:  "YOUR_CLUSTER_TOKEN",
})
if err != nil {
    log.Fatal("failed to connect to milvus: ", err.Error())
}
defer client.Close(ctx)

snapshot, err := client.DescribeSnapshot(ctx, milvusclient.NewDescribeSnapshotOption("analytics_snapshot_20260321", "analytics"))
if err != nil {
    log.Fatal("failed to describe snapshot: ", err.Error())
}

externalSource := fmt.Sprintf("s3://bucket/%s", snapshot.GetS3Location())
fmt.Println("External source:", externalSource)

Rust

rust

use milvus::v2::prelude::*;

let client = ClientV2::new(&ConnectConfig::new().uri("YOUR_CLUSTER_ENDPOINT").token("YOUR_CLUSTER_TOKEN")).await?;

let snapshot_info = client
    .describe_snapshot(
        DescribeSnapshotRequest::builder()
            .collection_name("analytics")
            .snapshot_name("analytics_snapshot_20260321")
            .build()?,
    )
    .await?;

let external_source = format!("s3://bucket/{}", snapshot_info.s3_location());
println!("External source: {external_source}");

C++

c++

#include <iostream>
#include <stdexcept>
#include <string>

#include "milvus/MilvusClientV2.h"

auto client = milvus::MilvusClientV2::Create();
milvus::ConnectParam connect_param{"YOUR_CLUSTER_ENDPOINT", "YOUR_CLUSTER_TOKEN"};
auto status = client->Connect(connect_param);
if (!status.IsOk()) {
    throw std::runtime_error(status.Message());
}

milvus::DescribeSnapshotResponse response;
status = client->DescribeSnapshot(milvus::DescribeSnapshotRequest()
                                      .WithSnapshotName("analytics_snapshot_20260321")
                                      .WithCollectionName("analytics"),
                                  response);
if (!status.IsOk()) {
    throw std::runtime_error(status.Message());
}

std::string external_source = "s3://bucket/" + response.S3Location();
std::cout << "External source: " << external_source << std::endl;

NodeJS

javascript

import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";

const client = new MilvusClient({ address: "YOUR_CLUSTER_ENDPOINT", token: "YOUR_CLUSTER_TOKEN" });

const snapshotInfo = await client.describeSnapshot({
  collection_name: "analytics",
  snapshot_name: "analytics_snapshot_20260321",
});

const externalSource = `s3://bucket/${snapshotInfo.s3_location}`;

cURL

bash

curl 'YOUR_CLUSTER_ENDPOINT/v2/vectordb/snapshots/describe' \
  -H 'Authorization: Bearer YOUR_CLUSTER_TOKEN' \
  -H 'Content-Type: application/json' \
  -d '{
    "collectionName": "analytics",
    "snapshotName": "analytics_snapshot_20260321"
  }'

Step 2: Create and refresh a milvus-table external collection

Create an external collection whose schema matches the snapshot source collection. Set external_spec.format to "milvus-table", and set each target data field's external_field to the corresponding source field name.

Python

python

schema = client.create_schema(
    external_source=external_source,
    external_spec="""{
        "format": "milvus-table",
        "extfs": {
            "cloud_provider": "aws",
            "region": "us-west-2",
            "access_key_id": "YOUR_ACCESS_KEY",
            "access_key_value": "YOUR_SECRET_KEY"
        }
    }""",
)
schema.add_field(
    field_name="id",
    datatype=DataType.INT64,
    is_primary=True,
    external_field="id",
)
schema.add_field(
    field_name="embedding",
    datatype=DataType.FLOAT_VECTOR,
    dim=768,
    external_field="embedding",
)
client.create_collection(
    collection_name="snapshot_external_collection",
    schema=schema,
)

Java

java

import com.google.gson.Gson;
import com.google.gson.JsonObject;
import io.milvus.v2.common.DataType;
import io.milvus.v2.service.collection.request.AddFieldReq;
import io.milvus.v2.service.collection.request.CreateCollectionReq;

// The snapshot metadata path obtained in Step 1
String externalSource = "s3://bucket/snapshots/{source_collection_id}/metadata/{snapshot_id}.json";

CreateCollectionReq.CollectionSchema schema = CreateCollectionReq.CollectionSchema.builder()
        .externalSource(externalSource)
        .externalSpec(new Gson().fromJson(
                "{\"format\":\"milvus-table\",\"extfs\":{\"cloud_provider\":\"aws\",\"region\":\"us-west-2\",\"access_key_id\":\"YOUR_ACCESS_KEY\",\"access_key_value\":\"YOUR_SECRET_KEY\"}}",
                JsonObject.class))
        .build();
schema.addField(AddFieldReq.builder().fieldName("id").dataType(DataType.Int64).isPrimaryKey(true).externalField("id").build());
schema.addField(AddFieldReq.builder().fieldName("embedding").dataType(DataType.FloatVector).dimension(768).externalField("embedding").build());

client.createCollection(CreateCollectionReq.builder()
        .collectionName("snapshot_external_collection")
        .collectionSchema(schema)
        .build());

Go

go

import (
    "context"
    "log"

    "github.com/milvus-io/milvus/client/v3/entity"
    "github.com/milvus-io/milvus/client/v3/milvusclient"
)

client, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
    Address: "YOUR_CLUSTER_ENDPOINT",
    APIKey:  "YOUR_CLUSTER_TOKEN",
})
if err != nil {
    log.Fatal("failed to connect to milvus: ", err.Error())
}
defer client.Close(ctx)

// The snapshot metadata path obtained in Step 1
externalSource := "s3://bucket/snapshots/{source_collection_id}/metadata/{snapshot_id}.json"

schema := entity.NewSchema().
    WithExternalSource(externalSource).
    WithExternalSpec(`{
  "format": "milvus-table",
  "extfs": {
    "cloud_provider": "aws",
    "region": "us-west-2",
    "access_key_id": "YOUR_ACCESS_KEY",
    "access_key_value": "YOUR_SECRET_KEY"
  }
}`).
    WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true).WithExternalField("id")).
    WithField(entity.NewField().WithName("embedding").WithDataType(entity.FieldTypeFloatVector).WithDim(768).WithExternalField("embedding"))

err = client.CreateCollection(ctx, milvusclient.NewCreateCollectionOption("snapshot_external_collection", schema))
if err != nil {
    log.Fatal("failed to create external collection: ", err.Error())
}

Rust

rust

use milvus::v2::prelude::*;

let client = ClientV2::new(&ConnectConfig::new().uri("YOUR_CLUSTER_ENDPOINT").token("YOUR_CLUSTER_TOKEN")).await?;

// The snapshot metadata path obtained in Step 1
let external_source = "s3://bucket/snapshots/{source_collection_id}/metadata/{snapshot_id}.json";

let schema = CollectionSchema::new()
    .enable_dynamic_field(false) // external collections do not support dynamic fields
    .external_source(external_source)
    .external_spec(serde_json::json!({
        "format": "milvus-table",
        "extfs": {
            "cloud_provider": "aws",
            "region": "us-west-2",
            "access_key_id": "YOUR_ACCESS_KEY",
            "access_key_value": "YOUR_SECRET_KEY"
        }
    }))
    .add_field(FieldSchema::new().name("id").data_type(DataType::Int64).primary_key(true).external_field("id"))
    .add_field(FieldSchema::new().name("embedding").data_type(DataType::FloatVector).dimension(768).external_field("embedding"));

client
    .create_collection(
        CreateCollectionRequest::builder()
            .collection_name("snapshot_external_collection")
            .schema(schema)
            .build()?,
    )
    .await?;

C++

c++

#include <iostream>
#include <memory>
#include <stdexcept>
#include <string>

#include "milvus/MilvusClientV2.h"
#include <milvus/thirdparty/nlohmann/json.hpp>

auto client = milvus::MilvusClientV2::Create();
milvus::ConnectParam connect_param{"YOUR_CLUSTER_ENDPOINT", "YOUR_CLUSTER_TOKEN"};
auto status = client->Connect(connect_param);
if (!status.IsOk()) {
    throw std::runtime_error(status.Message());
}

// The snapshot metadata path obtained in Step 1
std::string external_source = "s3://bucket/snapshots/{source_collection_id}/metadata/{snapshot_id}.json";

auto schema = std::make_shared<milvus::CollectionSchema>();
schema->SetEnableDynamicField(false); // external collections do not support dynamic fields
schema->WithExternalSource(external_source);
schema->WithExternalSpec(nlohmann::json::parse(R"({
        "format": "milvus-table",
        "extfs": {
            "cloud_provider": "aws",
            "region": "us-west-2",
            "access_key_id": "YOUR_ACCESS_KEY",
            "access_key_value": "YOUR_SECRET_KEY"
        }
    })"));
schema->AddField(milvus::FieldSchema("id", milvus::DataType::INT64, "", true).WithExternalField("id"));
schema->AddField(milvus::FieldSchema("embedding", milvus::DataType::FLOAT_VECTOR).WithDimension(768).WithExternalField("embedding"));

status = client->CreateCollection(milvus::CreateCollectionRequest()
                                      .WithCollectionName("snapshot_external_collection")
                                      .WithCollectionSchema(schema));
if (!status.IsOk()) {
    throw std::runtime_error(status.Message());
}

NodeJS

javascript

import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";

const client = new MilvusClient({ address: "YOUR_CLUSTER_ENDPOINT", token: "YOUR_CLUSTER_TOKEN" });

// The snapshot metadata path obtained in Step 1
const externalSource = "s3://bucket/snapshots/{source_collection_id}/metadata/{snapshot_id}.json";

await client.createCollection({
  collection_name: "snapshot_external_collection",
  schema: [
    // Note: milvus-sdk-node does not allow is_primary_key on external collection fields.
    { name: "id", data_type: DataType.Int64, external_field: "id" },
    { name: "embedding", data_type: DataType.FloatVector, dim: 768, external_field: "embedding" },
  ],
  external_source: externalSource,
  external_spec: JSON.stringify({
    format: "milvus-table",
    extfs: {
      cloud_provider: "aws",
      region: "us-west-2",
      access_key_id: "YOUR_ACCESS_KEY",
      access_key_value: "YOUR_SECRET_KEY",
    },
  }),
});

cURL

bash

curl 'YOUR_CLUSTER_ENDPOINT/v2/vectordb/collections/create' \
  -H 'Authorization: Bearer YOUR_CLUSTER_TOKEN' \
  -H 'Content-Type: application/json' \
  -d '{
    "collectionName": "snapshot_external_collection",
    "schema": {
      "externalSource": "s3://bucket/snapshots/{source_collection_id}/metadata/{snapshot_id}.json",
      "externalSpec": "{\"format\": \"milvus-table\", \"extfs\": {\"cloud_provider\": \"aws\", \"region\": \"us-west-2\", \"access_key_id\": \"YOUR_ACCESS_KEY\", \"access_key_value\": \"YOUR_SECRET_KEY\"}}",
      "enableDynamicField": false,
      "fields": [
        {"fieldName": "id", "dataType": "Int64", "isPrimary": true, "externalField": "id"},
        {"fieldName": "embedding", "dataType": "FloatVector", "elementTypeParams": {"dim": "768"}, "externalField": "embedding"}
      ]
    }
  }'

Refresh data

Once the collection is ready, refresh it to create the metadata and indexes for your data. For details, refer to Create an External Collection.