Jieba

The jieba tokenizer processes Chinese text by breaking it down into its component words.

📘Notes

The jieba tokenizer preserves punctuation marks as separate tokens in the output. For example, "你好！世界。" becomes ["你好", "！", "世界", "。"]. To remove these standalone punctuation tokens, use the removepunct filter.

Configuration

Zilliz Cloud supports two configuration approaches for the jieba tokenizer: a simple configuration and a custom configuration.

Simple configuration

With the simple configuration, you only need to set the tokenizer to "jieba". For example:

Python

python

# Simple configuration: only specifying the tokenizer name
analyzer_params = {
    "tokenizer": "jieba",  # Use the default settings: dict=["_default_"], mode="search", hmm=True
}

Java

java

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", "jieba");

Go

go

analyzerParams := map[string]any{"tokenizer": "jieba"}

Rust

rust

use serde_json::json;

let analyzer_params = json!({"tokenizer": "jieba"});

C++

c++

nlohmann::json analyzer_params = {
    {"tokenizer", "jieba"}
};

NodeJS

javascript

const analyzer_params = {
    "tokenizer": "jieba",
};

CSharp

csharp

// Simple configuration: only specifying the tokenizer name
var analyzerParams = new Dictionary<string, object>
{
    ["tokenizer"] = "jieba" // Use the default settings: dict=["_default_"], mode="search", hmm=True
};

cURL

bash

# restful
analyzerParams='{
  "tokenizer": "jieba"
}'

This simple configuration is equivalent to the following custom configuration:

Python

python

# Custom configuration equivalent to the simple configuration above
analyzer_params = {
    "tokenizer": {                 # Tokenizer configuration
        "type": "jieba",           # Tokenizer type, fixed as "jieba"
        "dict": ["_default_"],     # Use the default dictionary
        "mode": "search",          # Use search mode for improved recall (see mode details below)
        "hmm": True                # Enable HMM for probabilistic segmentation
    }
}

Java

java

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", new HashMap<String, Object>() {{
  put("type", "jieba");
  put("dict", Collections.singletonList("_default_"));
  put("mode", "search");
  put("hmm", true);
}});

Go

go

analyzerParams := map[string]any{
  "tokenizer": map[string]any{
    "type": "jieba",
    "dict": []any{"_default_"},
    "mode": "search",
    "hmm":  true,
  },
}

Rust

rust

use serde_json::json;

let analyzer_params = json!({
    "tokenizer": {
        "type": "jieba",
        "dict": ["_default_"],
        "mode": "search",
        "hmm": true
    }
});

C++

c++

nlohmann::json analyzer_params = {
    {"tokenizer", {
        {"type", "jieba"},
        {"dict", {"_default_"}},
        {"mode", "search"},
        {"hmm", true}
    }}
};

NodeJS

javascript

const analyzer_params = {
  tokenizer: {
    type: "jieba",
    dict: ["_default_"],
    mode: "search",
    hmm: true,
  },
};

CSharp

csharp

// Custom configuration equivalent to the simple configuration above
var analyzerParams = new Dictionary<string, object>
{
    ["tokenizer"] = new Dictionary<string, object> // Tokenizer configuration
    {
        ["type"] = "jieba",               // Tokenizer type, fixed as "jieba"
        ["dict"] = new[] { "_default_" }, // Use the default dictionary
        ["mode"] = "search",              // Use search mode for improved recall (see mode details below)
        ["hmm"] = true                    // Enable HMM for probabilistic segmentation
    }
};

cURL

bash

# restful
analyzerParams='{
  "tokenizer": {
    "type": "jieba",
    "dict": ["_default_"],
    "mode": "search",
    "hmm": true
  }
}'

For details on parameters, refer to Custom configuration.

Custom configuration

For more control, you can provide a custom configuration that allows you to specify a custom dictionary, select the segmentation mode, and enable or disable the Hidden Markov Model (HMM). For example:

Python

python

# Custom configuration with user-defined settings
analyzer_params = {
    "tokenizer": {
        "type": "jieba",           # Fixed tokenizer type
        "dict": ["customDictionary"],  # Custom dictionary list; replace with your own terms
        "mode": "exact",           # Use exact mode (non-overlapping tokens)
        "hmm": False               # Disable HMM; unmatched text will be split into individual characters
    }
}

Java

java

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", new HashMap<String, Object>() {{
  put("type", "jieba");
  put("dict", Arrays.asList("customDictionary"));
  put("mode", "exact");
  put("hmm", false);
}});

Go

go

analyzerParams := map[string]any{
  "tokenizer": map[string]any{
      "type": "jieba",
      "dict": []string{"customDictionary"},
      "mode": "exact",
      "hmm":  false,
  },
}

Rust

rust

use serde_json::json;

let analyzer_params = json!({
    "tokenizer": {
        "type": "jieba",
        "dict": ["customDictionary"],
        "mode": "exact",
        "hmm": false
    }
});

C++

c++

nlohmann::json analyzer_params = {
  {"tokenizer", {
      {"type", "jieba"},
      {"dict", {"customDictionary"}},
      {"mode", "exact"},
      {"hmm", false}
  }}
};

NodeJS

javascript

const analyzer_params = {
  tokenizer: {
    type: "jieba",
    dict: ["customDictionary"],
    mode: "exact",
    hmm: false,
  },
};

CSharp

csharp

// Custom configuration with user-defined settings
var analyzerParams = new Dictionary<string, object>
{
    ["tokenizer"] = new Dictionary<string, object>
    {
        ["type"] = "jieba",                  // Fixed tokenizer type
        ["dict"] = new[] { "customDictionary" }, // Custom dictionary list; replace with your own terms
        ["mode"] = "exact",                  // Use exact mode (non-overlapping tokens)
        ["hmm"] = false                      // Disable HMM; unmatched text will be split into individual characters
    }
};

cURL

bash

# restful
analyzerParams='{
  "tokenizer": {
    "type": "jieba",
    "dict": ["customDictionary"],
    "mode": "exact",
    "hmm": false
  }
}'

Parameter	Description	Default Value
type	The type of tokenizer. This is fixed to "jieba".	"jieba"
dict	A list of dictionaries that the analyzer will load as its vocabulary source. Built-in options: "default": Loads the engine's built‑in Simplified‑Chinese dictionary. For details, refer to dict.txt. "extend_default": Loads everything in "default" plus an additional Traditional‑Chinese supplement. For details, refer to dict.txt.big. You can also mix the built‑in dictionary with any number of custom dictionaries. Example: ["default", "结巴分词器"].	["default"]
mode	The segmentation mode. Possible values: "exact": Tries to segment the sentence in the most precise manner, making it ideal for text analysis. "search": Builds on exact mode by further breaking down long words to improve recall, making it suitable for search engine tokenization. For more information, refer to Jieba GitHub Project.	"search"
hmm	A boolean flag indicating whether to enable the Hidden Markov Model (HMM) for probabilistic segmentation of words not found in the dictionary.	true

After defining analyzer_params, you can apply them to a VARCHAR field when defining a collection schema. This allows Zilliz Cloud to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to Example use.

Examples

Before applying the analyzer configuration to your collection schema, verify its behavior using the run_analyzer method.

Analyzer configuration

Python

python

analyzer_params = {
    "tokenizer": {
        "type": "jieba",
        "dict": ["结巴分词器"],
        "mode": "exact",
        "hmm": False
    }
}

Java

java

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", new HashMap<String, Object>() {{
  put("type", "jieba");
  put("dict", Arrays.asList("结巴分词器"));
  put("mode", "exact");
  put("hmm", false);
}});

Go

go

analyzerParams := map[string]any{
  "tokenizer": map[string]any{
      "type": "jieba",
      "dict": []string{"结巴分词器"},
      "mode": "exact",
      "hmm":  false,
  },
}

Rust

rust

use serde_json::json;

let analyzer_params = json!({
    "tokenizer": {
        "type": "jieba",
        "dict": ["结巴分词器"],
        "mode": "exact",
        "hmm": false
    }
});

C++

c++

nlohmann::json analyzer_params = {
  {"tokenizer", {
      {"type", "jieba"},
      {"dict", {"结巴分词器"}},
      {"mode", "exact"},
      {"hmm", false}
  }}
};

NodeJS

javascript

const analyzer_params = {
  tokenizer: {
    type: "jieba",
    dict: ["结巴分词器"],
    mode: "exact",
    hmm: false,
  },
};

CSharp

csharp

var analyzerParams = new Dictionary<string, object>
{
    ["tokenizer"] = new Dictionary<string, object>
    {
        ["type"] = "jieba",
        ["dict"] = new[] { "结巴分词器" },
        ["mode"] = "exact",
        ["hmm"] = false
    }
};

cURL

bash

# restful
analyzerParams='{
  "tokenizer": {
    "type": "jieba",
    "dict": ["结巴分词器"],
    "mode": "exact",
    "hmm": false
  }
}'

Verification using run_analyzer

Python

python

from pymilvus import (
    MilvusClient,
)

client = MilvusClient(
    uri="YOUR_CLUSTER_ENDPOINT",
    token="YOUR_CLUSTER_TOKEN"
)

# Sample text to analyze
sample_text = "milvus结巴分词器中文测试"

# Run the jieba tokenizer with the defined configuration
result = client.run_analyzer(sample_text, analyzer_params)
print("Jieba analyzer output:", result)

Java

java

import java.util.*;
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.service.vector.request.RunAnalyzerReq;
import io.milvus.v2.service.vector.response.RunAnalyzerResp;

ConnectConfig config = ConnectConfig.builder()
        .uri("YOUR_CLUSTER_ENDPOINT")
        .token("YOUR_CLUSTER_TOKEN")
        .build();
MilvusClientV2 client = new MilvusClientV2(config);

List<String> texts = new ArrayList<>();
texts.add("milvus结巴分词器中文测试");

RunAnalyzerResp resp = client.runAnalyzer(RunAnalyzerReq.builder()
        .texts(texts)
        .analyzerParams(analyzerParams)
        .build());
List<RunAnalyzerResp.AnalyzerResult> results = resp.getResults();

for (RunAnalyzerResp.AnalyzerResult result : results) {
    for (RunAnalyzerResp.AnalyzerToken token : result.getTokens()) {
        System.out.println("Jieba analyzer output: " + token.getToken());
    }
}

Go

go

import (
    "context"
    "fmt"

    "github.com/milvus-io/milvus/client/v3/milvusclient"
)

ctx := context.Background()

client, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
    Address: "YOUR_CLUSTER_ENDPOINT",
    APIKey:  "YOUR_CLUSTER_TOKEN",
})
if err != nil {
    fmt.Println(err.Error())
    // handle error
}

texts := []string{"milvus结巴分词器中文测试"}
option := milvusclient.NewRunAnalyzerOption(texts...).
    WithAnalyzerParams(analyzerParams)

result, err := client.RunAnalyzer(ctx, option)
if err != nil {
    fmt.Println(err.Error())
    // handle error
}

for _, r := range result {
    for _, token := range r.Tokens {
        fmt.Println("Jieba analyzer output:", token.Text)
    }
}

Rust

rust

use milvus::v2::prelude::*;

let config = ConnectConfig::new().uri("YOUR_CLUSTER_ENDPOINT").token("YOUR_CLUSTER_TOKEN");
let client = ClientV2::new(&config).await?;

let response = client
    .run_analyzer(
        RunAnalyzerRequest::builder()
            .texts(["milvus结巴分词器中文测试"])
            .analyzer_params(analyzer_params)
            .build()?,
    )
    .await?;

for result in response.results() {
    for token in result.get_tokens() {
        println!("Jieba analyzer output: {}", token.get_text());
    }
}

C++

c++

#include "milvus/MilvusClientV2.h"
#include <iostream>
#include <string>

auto client = milvus::MilvusClientV2::Create();

milvus::ConnectParam connect_param{"YOUR_CLUSTER_ENDPOINT", "YOUR_CLUSTER_TOKEN"};
auto status = client->Connect(connect_param);
if (!status.IsOk()) {
    std::cout << status.Message() << std::endl;
}

std::string text = "milvus结巴分词器中文测试";
auto request = milvus::RunAnalyzerRequest()
                       .AddText(text)
                       .WithAnalyzerParams(analyzer_params);

milvus::RunAnalyzerResponse response;
status = client->RunAnalyzer(request, response);
if (!status.IsOk()) {
    std::cout << status.Message() << std::endl;
}

for (const auto& result : response.Results()) {
    for (const auto& token : result.Tokens()) {
        std::cout << "Jieba analyzer output: " << token.token_ << std::endl;
    }
}

NodeJS

javascript

import { MilvusClient } from '@zilliz/milvus2-sdk-node';

const client = new MilvusClient({
  address: 'YOUR_CLUSTER_ENDPOINT',
  token: 'YOUR_CLUSTER_TOKEN',
});

const sample_text = 'milvus结巴分词器中文测试';
const result = await client.runAnalyzer({
  text: sample_text,
  analyzer_params,
});
const tokens = result.results.flatMap(r => r.tokens.map(t => t.token));
console.log('Jieba analyzer output:', tokens);

CSharp

csharp

using Milvus.Client.V2;
using Milvus.Client.V2.Requests.Utility;
using Milvus.Client.V2.Responses.Utility;
using Milvus.Client.V2.Types;

using var client = new MilvusClientV2(new ConnectConfig
{
    Uri = "YOUR_CLUSTER_ENDPOINT",
    ApiKey = "YOUR_CLUSTER_TOKEN"
});
await client.ConnectAsync();

// Sample text to analyze
var sampleText = "milvus结巴分词器中文测试";

// Run the jieba tokenizer with the defined configuration
RunAnalyzerResp result = await client.RunAnalyzerAsync(new RunAnalyzerReq
{
    AnalyzerParams = analyzerParams,
    Texts = new[] { sampleText }
});

foreach (AnalyzerResult r in result.Results)
{
    foreach (AnalyzerToken t in r.Tokens)
    {
        Console.WriteLine($"Jieba analyzer output: {t.Token}");
    }
}

cURL

bash

curl --request POST \
  --url "YOUR_CLUSTER_ENDPOINT/v2/vectordb/common/run_analyzer" \
  --header "Authorization: Bearer YOUR_CLUSTER_TOKEN" \
  --header "Content-Type: application/json" \
  --data-raw '{
    "text": ["milvus结巴分词器中文测试"],
    "analyzerParams": "{\"tokenizer\":{\"type\":\"jieba\",\"dict\":[\"结巴分词器\"],\"mode\":\"exact\",\"hmm\":false}}"
  }'

Expected output

For the sample text, the analyzer produces the following tokens in order. SDK response objects and printed formatting vary by language; the REST response contains these tokens at data.results[].tokens[].token.

plaintext

['milvus', '结巴分词器', '中', '文', '测', '试']