Chinese
The chinese analyzer is designed specifically to handle Chinese text, providing effective segmentation and tokenization.
Definition
The chinese analyzer consists of:
-
Tokenizer: Uses the
jiebatokenizer to segment Chinese text into tokens based on vocabulary and context. For more information, refer to Jieba. -
Filter: Uses the
cnalphanumonlyfilter to remove tokens that contain any characters other than Chinese characters, English letters, or digits. For more information, refer to Cnalphanumonly.
The functionality of the chinese analyzer is equivalent to the following custom analyzer configuration:
The built-in chinese analyzer does not emit Pinyin tokens. To match Chinese text with Pinyin query terms, use a custom analyzer with the jieba tokenizer and the pinyin filter.
- Python
- Java
- Go
- Rust
- C++
- NodeJS
- CSharp
- cURL
analyzer_params = {
"tokenizer": "jieba",
"filter": ["cnalphanumonly"]
}
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", "jieba");
analyzerParams.put("filter", Collections.singletonList("cnalphanumonly"));
analyzerParams := map[string]any{"tokenizer": "jieba", "filter": []any{"cnalphanumonly"}}
use serde_json::json;
let analyzer_params = json!({
"tokenizer": "jieba",
"filter": ["cnalphanumonly"]
});
nlohmann::json analyzer_params = {
{"tokenizer", "jieba"},
{"filter", {"cnalphanumonly"}}
};
const analyzer_params = {
"tokenizer": "jieba",
"filter": ["cnalphanumonly"]
};
using System.Collections.Generic;
var analyzerParams = new Dictionary<string, object>
{
["tokenizer"] = "jieba",
["filter"] = new[] { "cnalphanumonly" }
};
# restful
analyzerParams='{
"tokenizer": "jieba",
"filter": [
"cnalphanumonly"
]
}'
Configuration
To apply the chinese analyzer to a field, simply set type to chinese in analyzer_params.
- Python
- Java
- Go
- Rust
- C++
- NodeJS
- CSharp
- cURL
analyzer_params = {
"type": "chinese",
}
Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("type", "chinese");
analyzerParams := map[string]any{"type": "chinese"}
use serde_json::json;
let analyzer_params = json!({
"type": "chinese"
});
nlohmann::json analyzer_params = {
{"type", "chinese"}
};
const analyzer_params = {
"type": "chinese",
};
var analyzerParams = new Dictionary<string, object>
{
["type"] = "chinese"
};
# restful
analyzerParams='{
"type": "chinese"
}'
The chinese analyzer does not accept any optional parameters.
Examples
Before applying the analyzer configuration to your collection schema, verify its behavior using the run_analyzer method.
Analyzer configuration
- Python
- Java
- Go
- Rust
- C++
- NodeJS
- CSharp
- cURL
analyzer_params = {
"type": "chinese",
}
Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("type", "chinese");
analyzerParams := map[string]any{"type": "chinese"}
use serde_json::json;
let analyzer_params = json!({
"type": "chinese"
});
nlohmann::json analyzer_params = {
{"type", "chinese"}
};
const analyzer_params = {
"type": "chinese",
};
var analyzerParams = new Dictionary<string, object>
{
["type"] = "chinese"
};
# restful
analyzerParams='{
"type": "chinese"
}'
Verification using run_analyzer
- Python
- Java
- Go
- Rust
- C++
- NodeJS
- CSharp
- cURL
from pymilvus import (
MilvusClient,
)
client = MilvusClient(
uri="YOUR_CLUSTER_ENDPOINT",
token="YOUR_CLUSTER_TOKEN"
)
# Sample text to analyze
sample_text = "Milvus 是一个高性能、可扩展的向量数据库!"
# Run the chinese analyzer with the defined configuration
result = client.run_analyzer(sample_text, analyzer_params)
print("Chinese analyzer output:", result)
import java.util.*;
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.service.vector.request.RunAnalyzerReq;
import io.milvus.v2.service.vector.response.RunAnalyzerResp;
ConnectConfig config = ConnectConfig.builder()
.uri("YOUR_CLUSTER_ENDPOINT")
.token("YOUR_CLUSTER_TOKEN")
.build();
MilvusClientV2 client = new MilvusClientV2(config);
List<String> texts = new ArrayList<>();
texts.add("Milvus 是一个高性能、可扩展的向量数据库!");
RunAnalyzerResp resp = client.runAnalyzer(RunAnalyzerReq.builder()
.texts(texts)
.analyzerParams(analyzerParams)
.build());
List<RunAnalyzerResp.AnalyzerResult> results = resp.getResults();
for (RunAnalyzerResp.AnalyzerResult result : results) {
for (RunAnalyzerResp.AnalyzerToken token : result.getTokens()) {
System.out.println("Chinese analyzer output: " + token.getToken());
}
}
import (
"context"
"fmt"
"github.com/milvus-io/milvus/client/v3/milvusclient"
)
ctx := context.Background()
client, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
Address: "YOUR_CLUSTER_ENDPOINT",
APIKey: "YOUR_CLUSTER_TOKEN",
})
if err != nil {
fmt.Println(err.Error())
// handle error
}
texts := []string{"Milvus 是一个高性能、可扩展的向量数据库!"}
option := milvusclient.NewRunAnalyzerOption(texts...).
WithAnalyzerParams(analyzerParams)
result, err := client.RunAnalyzer(ctx, option)
if err != nil {
fmt.Println(err.Error())
// handle error
}
for _, r := range result {
for _, token := range r.Tokens {
fmt.Println("Chinese analyzer output:", token.Text)
}
}
use milvus::v2::prelude::*;
let config = ConnectConfig::new().uri("YOUR_CLUSTER_ENDPOINT").token("YOUR_CLUSTER_TOKEN");
let client = ClientV2::new(&config).await?;
let response = client
.run_analyzer(
RunAnalyzerRequest::builder()
.texts(["Milvus 是一个高性能、可扩展的向量数据库!"])
.analyzer_params(analyzer_params)
.build()?,
)
.await?;
for result in response.results() {
for token in result.get_tokens() {
println!("Chinese analyzer output: {}", token.get_text());
}
}
#include "milvus/MilvusClientV2.h"
#include <iostream>
#include <string>
auto client = milvus::MilvusClientV2::Create();
milvus::ConnectParam connect_param{"YOUR_CLUSTER_ENDPOINT", "YOUR_CLUSTER_TOKEN"};
auto status = client->Connect(connect_param);
if (!status.IsOk()) {
std::cout << status.Message() << std::endl;
}
std::string text = "Milvus 是一个高性能、可扩展的向量数据库!";
auto request = milvus::RunAnalyzerRequest()
.AddText(text)
.WithAnalyzerParams(analyzer_params);
milvus::RunAnalyzerResponse response;
status = client->RunAnalyzer(request, response);
if (!status.IsOk()) {
std::cout << status.Message() << std::endl;
}
for (const auto& result : response.Results()) {
for (const auto& token : result.Tokens()) {
std::cout << "Chinese analyzer output: " << token.token_ << std::endl;
}
}
import { MilvusClient } from '@zilliz/milvus2-sdk-node';
const client = new MilvusClient({
address: 'YOUR_CLUSTER_ENDPOINT',
token: 'YOUR_CLUSTER_TOKEN',
});
const sample_text = 'Milvus 是一个高性能、可扩展的向量数据库!';
const result = await client.runAnalyzer({
text: sample_text,
analyzer_params,
});
const tokens = result.results.flatMap(r => r.tokens.map(t => t.token));
console.log('Chinese analyzer output:', tokens);
using Milvus.Client.V2;
using Milvus.Client.V2.Requests.Utility;
using Milvus.Client.V2.Responses.Utility;
using Milvus.Client.V2.Types;
using var client = new MilvusClientV2(new ConnectConfig
{
Uri = "YOUR_CLUSTER_ENDPOINT",
ApiKey = "YOUR_CLUSTER_TOKEN"
});
await client.ConnectAsync();
// Sample text to analyze
var sampleText = "Milvus 是一个高性能、可扩展的向量数据库!";
// Run the chinese analyzer with the defined configuration
RunAnalyzerResp result = await client.RunAnalyzerAsync(new RunAnalyzerReq
{
AnalyzerParams = analyzerParams,
Texts = new[] { sampleText }
});
foreach (AnalyzerResult r in result.Results)
{
foreach (AnalyzerToken t in r.Tokens)
{
Console.WriteLine($"Chinese analyzer output: {t.Token}");
}
}
curl --request POST \
--url "YOUR_CLUSTER_ENDPOINT/v2/vectordb/common/run_analyzer" \
--header "Authorization: Bearer YOUR_CLUSTER_TOKEN" \
--header "Content-Type: application/json" \
--data-raw '{
"analyzerParams": "{\"type\": \"chinese\"}",
"text": ["Milvus 是一个高性能、可扩展的向量数据库!"]
}'
Expected output
For the sample text, the analyzer produces the following tokens in order. SDK response objects and printed formatting vary by language; the REST response contains these tokens at data.results[].tokens[].token.
['Milvus', '是', '一个', '高性', '性能', '高性能', '可', '扩展', '的', '向量', '数据', '据库', '数据库']