Standard Analyzer

The standard analyzer is the default analyzer in Zilliz Cloud, which is automatically applied to text fields if no analyzer is specified. It combines the standard tokenizer with the lowercase filter.

Notes

The standard analyzer is suitable for languages that rely on separators (such as spaces, punctuation) for word boundaries. However, languages like Chinese, Japanese, and Korean require dictionary-based tokenizations. In such cases, using a language-specific analyzer like chinese or custom analyzers with specialized tokenizers (such as lindera, icu) and filters is highly recommended to ensure accurate tokenization and better search results.

Definition

The standard analyzer consists of:

• Tokenizer: Uses the standard tokenizer to keep consecutive Unicode letters and numeric characters in tokens and split at other characters. For the exact character rules, refer to Standard Tokenizer.

• Filter: Uses the lowercase filter to convert all tokens to lowercase, enabling case-insensitive searches. For more information, refer to Lowercase.

The functionality of the standard analyzer is equivalent to the following custom analyzer configuration:

Python

python

analyzer_params = {
    "tokenizer": "standard",
    "filter": ["lowercase"]
}

Java

java

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", "standard");
analyzerParams.put("filter", Collections.singletonList("lowercase"));

Go

go

analyzerParams := map[string]any{"tokenizer": "standard", "filter": []any{"lowercase"}}

Rust

rust

let analyzer_params = serde_json::json!({
    "tokenizer": "standard",
    "filter": ["lowercase"]
});

C++

c++

nlohmann::json analyzer_params = {
    {"tokenizer", "standard"},
    {"filter", {"lowercase"}},
};

NodeJS

javascript

const analyzer_params = {
    "tokenizer": "standard",
    "filter": ["lowercase"]
};

cURL

bash

# restful
analyzerParams='{
  "tokenizer": "standard",
  "filter": [
    "lowercase"
  ]
}'

Configuration

To apply the standard analyzer to a field, simply set type to standard in analyzer_params, and include optional parameters as needed.

Python

python

analyzer_params = {
    "type": "standard", # Specifies the standard analyzer type
}

Java

java

import java.util.HashMap;
import java.util.Map;

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("type", "standard");

Go

go

analyzerParams = map[string]any{"type": "standard"}

Rust

rust

let analyzer_params = serde_json::json!({
    "type": "standard"
});

C++

c++

nlohmann::json analyzer_params = {
    {"type", "standard"},
};

NodeJS

javascript

const analyzer_params = {
    "type": "standard", // Specifies the standard analyzer type
}

cURL

bash

# restful
analyzerParams='{
  "type": "standard"
}'

The standard analyzer accepts the following optional parameters:

Parameter	Description
stop_words	An array containing a list of stop words, which will be removed from tokenization.

Example configuration of custom stop words:

Python

python

analyzer_params = {
    "type": "standard", # Specifies the standard analyzer type
    "stop_words": ["of"] # Optional: List of words to exclude from tokenization
}

Java

java

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("type", "standard");
analyzerParams.put("stop_words", Collections.singletonList("of"));

Go

go

analyzerParams = map[string]any{"type": "standard", "stop_words": []string{"of"}}

Rust

rust

let analyzer_params = serde_json::json!({
    "type": "standard",
    "stop_words": ["of"]
});

C++

c++

nlohmann::json analyzer_params = {
    {"type", "standard"},
    {"stop_words", {"of"}},
};

NodeJS

javascript

analyzer_params = {
    "type": "standard", // Specifies the standard analyzer type
    "stop_words": ["of"] // Optional: List of words to exclude from tokenization
}

cURL

bash

# restful
analyzerParams='{
  "type": "standard",
  "stop_words": [
    "of"
  ]
}'

After defining analyzer_params, you can apply them to a VARCHAR field when defining a collection schema. This allows Zilliz Cloud to process the text in that field using the specified analyzer for efficient tokenization and filtering. For more information, refer to Example use.

Examples

Before applying the analyzer configuration to your collection schema, verify its behavior using the run_analyzer method.

Analyzer configuration

Python

python

analyzer_params = {
    "type": "standard",  # Standard analyzer configuration
    "stop_words": ["for"] # Optional: Custom stop words parameter
}

Java

java

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("type", "standard");
analyzerParams.put("stop_words", Collections.singletonList("for"));

Go

go

analyzerParams = map[string]any{"type": "standard", "stop_words": []string{"for"}}

Rust

rust

let analyzer_params = serde_json::json!({
    "type": "standard",
    "stop_words": ["for"]
});

C++

c++

nlohmann::json analyzer_params = {
    {"type", "standard"},
    {"stop_words", {"for"}},
};

NodeJS

javascript

// javascript
analyzer_params = {
    "type": "standard", // Specifies the standard analyzer type
    "stop_words": ["for"] // Optional: List of words to exclude from tokenization
}

cURL

bash

# restful
analyzerParams='{
  "type": "standard",
  "stop_words": [
    "for"
  ]
}'

Verification using run_analyzer

Python

python

from pymilvus import (
    MilvusClient,
)

client = MilvusClient(
    uri="YOUR_CLUSTER_ENDPOINT",
    token="YOUR_CLUSTER_TOKEN"
)

# Sample text to analyze
sample_text = "The Milvus vector database is built for scale!"

# Run the standard analyzer with the defined configuration
result = client.run_analyzer(sample_text, analyzer_params)
print("Standard analyzer output:", result)

Java

java

import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.service.vector.request.RunAnalyzerReq;
import io.milvus.v2.service.vector.response.RunAnalyzerResp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

ConnectConfig config = ConnectConfig.builder()
        .uri("YOUR_CLUSTER_ENDPOINT")
        .token("YOUR_CLUSTER_TOKEN")
        .build();
MilvusClientV2 client = new MilvusClientV2(config);

List<String> texts = new ArrayList<>();
texts.add("The Milvus vector database is built for scale!");

RunAnalyzerResp resp = client.runAnalyzer(RunAnalyzerReq.builder()
        .texts(texts)
        .analyzerParams(analyzerParams)
        .build());
List<RunAnalyzerResp.AnalyzerResult> results = resp.getResults();

Go

go

import (
    "context"
    "encoding/json"
    "fmt"

    "github.com/milvus-io/milvus/client/v3/milvusclient"
)

client, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
    Address: "YOUR_CLUSTER_ENDPOINT",
    APIKey:  "YOUR_CLUSTER_TOKEN",
})
if err != nil {
    fmt.Println(err.Error())
    // handle error
}

bs, _ := json.Marshal(analyzerParams)
texts := []string{"The Milvus vector database is built for scale!"}
option := milvusclient.NewRunAnalyzerOption(texts...).
    WithAnalyzerParamsStr(string(bs))

result, err := client.RunAnalyzer(ctx, option)
if err != nil {
    fmt.Println(err.Error())
    // handle error
}

Rust

rust

use milvus::v2::error::Result;
use milvus::v2::prelude::*;

#[tokio::main]
async fn main() -> Result<()> {
    let config = ConnectConfig::new().uri("YOUR_CLUSTER_ENDPOINT").token("YOUR_CLUSTER_TOKEN");
    let client = ClientV2::new(&config).await?;

    let texts = ["The Milvus vector database is built for scale!".to_string()];
    let analyzer_params = serde_json::json!({
        "type": "standard",
        "stop_words": ["for"]
    });

    let result = client.run_analyzer(
        RunAnalyzerRequest::builder()
            .texts(texts)
            .analyzer_params(analyzer_params)
            .build()?,
    ).await?;

    println!("{:?}", result.results());
    Ok(())
}

C++

c++

#include "milvus/MilvusClientV2.h"
#include <iostream>
#include <string>

auto client = milvus::MilvusClientV2::Create();

milvus::ConnectParam connect_param{"YOUR_CLUSTER_ENDPOINT", "YOUR_CLUSTER_TOKEN"};
auto status = client->Connect(connect_param);
if (!status.IsOk()) {
    std::cout << status.Message() << std::endl;
}

std::string text = "The Milvus vector database is built for scale!";
auto request = milvus::RunAnalyzerRequest()
                       .AddText(text)
                       .WithAnalyzerParams(analyzer_params);

milvus::RunAnalyzerResponse response;
status = client->RunAnalyzer(request, response);
if (!status.IsOk()) {
    std::cout << status.Message() << std::endl;
}

NodeJS

javascript

// javascript
const res = await client.runAnalyzer({
    text: 'The Milvus vector database is built for scale!',
    analyzer_params: analyzer_params
});

console.log(res);

cURL

bash

# restful
curl --request POST \
  --url "${CLUSTER_ENDPOINT}/v2/vectordb/common/run_analyzer" \
  --header "Authorization: Bearer ${TOKEN}" \
  --header "Content-Type: application/json" \
  --data '{
    "analyzerParams": "{\"type\":\"standard\",\"stop_words\":[\"for\"]}",
    "text": ["The Milvus vector database is built for scale!"]
  }'

Expected output

sql

Standard analyzer output: ['the', 'milvus', 'vector', 'database', 'is', 'built', 'for', 'scale']