Regex

The regex filter is a regular expression filter: any token produced by the tokenizer is kept only if it matches the expression you provide; everything else is discarded.

Note

This page describes the regex filter in the analyzer pipeline. It filters tokens produced by a tokenizer and affects the terms generated during text analysis. To filter entities with scalar expressions such as field =~ "pattern" or field !~ "pattern" in query, search, or hybrid search, refer to Pattern Matching.

Configuration

The regex filter is a custom filter in Zilliz Cloud. To use it, specify "type": "regex" in the filter configuration, along with an expr parameter to specify the desired regular expressions.

Python

python

analyzer_params = {
    "tokenizer": "standard",
    "filter": [{
        "type": "regex",
        "expr": "^(?!test)" # keep tokens that do NOT start with "test"
    }]
}

Java

java

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", "standard");
analyzerParams.put("filter",
        Arrays.asList(new HashMap<String, Object>() {{
            put("type", "regex");
            put("expr", "^(?!test)");
        }})
);

Go

go

analyzerParams := map[string]any{"tokenizer": "standard",
    "filter": []any{map[string]any{
        "type": "regex",
        "expr": "^(?!test)",
    }}}

Rust

rust

let analyzer_params = serde_json::json!({
    "tokenizer": "standard",
    "filter": [{
        "type": "regex",
        "expr": "^(?!test)"
    }]
});

C++

c++

nlohmann::json analyzer_params = {
    {"tokenizer", "standard"},
    {"filter", {
        {{"type", "regex"}, {"expr", "^(?!test)"}}
    }}
};

NodeJS

javascript

const analyzer_params = {
    "tokenizer": "standard",
    "filter": [
        {
            "type": "regex",
            "expr": "^(?!test)"
        }
    ],
};

cURL

bash

# restful
analyzerParams='{
  "tokenizer": "standard",
  "filter": [
    {
      "type": "regex",
      "expr": "^(?!test)"
    }
  ]
}'

The regex filter accepts the following configurable parameters.

Parameter	Description
expr	A regular‑expression pattern applied to each token. Tokens that match are retained; non‑matches are dropped. For details on regex syntax, refer to Syntax.

The regex filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer.

After defining analyzer_params, you can apply them to a VARCHAR field when defining a collection schema. This allows Zilliz Cloud to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to Example use.

Examples

Before applying the analyzer configuration to your collection schema, verify its behavior using the run_analyzer method.

Analyzer configuration

Python

python

analyzer_params = {
    "tokenizer": "standard",
    "filter": [{
        "type": "regex",
        "expr": "^(?!test)"
    }]
}

Java

java

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", "standard");
analyzerParams.put("filter",
        Collections.singletonList(new HashMap<String, Object>() {{
            put("type", "regex");
            put("expr", "^(?!test)");
        }}));

Go

go

analyzerParams := map[string]any{"tokenizer": "standard",
    "filter": []any{map[string]any{
        "type": "regex",
        "expr": "^(?!test)",
    }}}

Rust

rust

let analyzer_params = serde_json::json!({
    "tokenizer": "standard",
    "filter": [{
        "type": "regex",
        "expr": "^(?!test)"
    }]
});

C++

c++

nlohmann::json analyzer_params = {
    {"tokenizer", "standard"},
    {"filter", {
        {{"type", "regex"}, {"expr", "^(?!test)"}}
    }}
};

NodeJS

javascript

const analyzer_params = {
    "tokenizer": "standard",
    "filter": [
        {
            "type": "regex",
            "expr": "^(?!test)"
        }
    ],
};

cURL

bash

# restful
analyzerParams='{
  "tokenizer": "standard",
  "filter": [
    {
      "type": "regex",
      "expr": "^(?!test)"
    }
  ]
}'

Verification using run_analyzer

Python

python

from pymilvus import (
    MilvusClient,
)

client = MilvusClient(uri="YOUR_CLUSTER_ENDPOINT")

# Sample text to analyze
sample_text = "testItem apple testCase banana"

# Run the standard analyzer with the defined configuration
result = client.run_analyzer(sample_text, analyzer_params)
print("Standard analyzer output:", result)

Java

java

import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.service.vector.request.RunAnalyzerReq;
import io.milvus.v2.service.vector.response.RunAnalyzerResp;
import java.util.ArrayList;
import java.util.List;

ConnectConfig config = ConnectConfig.builder()
        .uri("YOUR_CLUSTER_ENDPOINT")
        .build();
MilvusClientV2 client = new MilvusClientV2(config);

List<String> texts = new ArrayList<>();
texts.add("testItem apple testCase banana");

RunAnalyzerResp resp = client.runAnalyzer(RunAnalyzerReq.builder()
        .texts(texts)
        .analyzerParams(analyzerParams)
        .build());
List<RunAnalyzerResp.AnalyzerResult> results = resp.getResults();

Go

go

import (
    "context"
    "fmt"

    "github.com/milvus-io/milvus/client/v3/milvusclient"
)

ctx := context.Background()
client, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
    Address: "YOUR_CLUSTER_ENDPOINT",
    APIKey:  "YOUR_CLUSTER_TOKEN",
})
if err != nil {
    fmt.Println(err.Error())
    // handle error
}

texts := []string{"testItem apple testCase banana"}
option := milvusclient.NewRunAnalyzerOption(texts...).
    WithAnalyzerParams(analyzerParams)

result, err := client.RunAnalyzer(ctx, option)
if err != nil {
    fmt.Println(err.Error())
    // handle error
}

for _, r := range result {
    for _, token := range r.Tokens {
        fmt.Println("Standard analyzer output:", token.Text)
    }
}

Rust

rust

use milvus::v2::error::Result;
use milvus::v2::prelude::*;

#[tokio::main]
async fn main() -> Result<()> {
    let config = ConnectConfig::new().uri("YOUR_CLUSTER_ENDPOINT");
    let client = ClientV2::new(&config).await?;

    let analyzer_params = serde_json::json!({
        "tokenizer": "standard",
        "filter": [{
            "type": "regex",
            "expr": "^(?!test)"
        }]
    });

    let sample_text = "testItem apple testCase banana";
    let result = client
        .run_analyzer(
            RunAnalyzerRequest::builder()
                .texts(vec![sample_text])
                .analyzer_params(analyzer_params)
                .build()?,
        )
        .await?;

    println!("Standard analyzer output: {:?}", result);
    Ok(())
}

C++

c++

#include "milvus/MilvusClientV2.h"
#include <iostream>
#include <string>

auto client = milvus::MilvusClientV2::Create();

milvus::ConnectParam connect_param{"YOUR_CLUSTER_ENDPOINT"};
auto status = client->Connect(connect_param);
if (!status.IsOk()) {
    std::cout << status.Message() << std::endl;
}

std::string text = "testItem apple testCase banana";
auto request = milvus::RunAnalyzerRequest()
                       .AddText(text)
                       .WithAnalyzerParams(analyzer_params);

milvus::RunAnalyzerResponse response;
status = client->RunAnalyzer(request, response);
if (!status.IsOk()) {
    std::cout << status.Message() << std::endl;
}

NodeJS

javascript

import { MilvusClient } from "@zilliz/milvus2-sdk-node";

const client = new MilvusClient({ address: "YOUR_CLUSTER_ENDPOINT" });

const sampleText = "testItem apple testCase banana";
const result = await client.runAnalyzer({
    text: sampleText,
    analyzer_params,
});
console.log("Standard analyzer output:", result);

cURL

bash

export CLUSTER_ENDPOINT="YOUR_CLUSTER_ENDPOINT"
export TOKEN="YOUR_CLUSTER_TOKEN"

curl --request POST \
--url "${CLUSTER_ENDPOINT}/v2/vectordb/common/run_analyzer" \
--header "Authorization: Bearer ${TOKEN}" \
--header "Content-Type: application/json" \
--header "Request-Timeout: 10" \
-d '{
    "text": ["testItem apple testCase banana"],
    "analyzerParams": "{\"tokenizer\": \"standard\", \"filter\": [{\"type\": \"regex\", \"expr\": \"^(?!test)\"}]}"
}'

Expected output

python

['apple', 'banana']