curl --request POST \
--url https://api.example.com/api/kb/ingest-web \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data 'collection=<string>' \
--data 'start_url=<string>' \
--data max_pages=100 \
--data max_depth=3 \
--data 'url_patterns=<string>' \
--data 'exclude_patterns=<string>' \
--data same_domain_only=true \
--data 'content_selector=<string>' \
--data 'remove_selectors=<string>' \
--data concurrent_requests=3 \
--data request_delay=1 \
--data timeout=30 \
--data respect_robots_txt=true \
--data chunk_size=1 \
--data chunk_overlap=1 \
--data 'separators=<string>' \
--data embedding_model_id=text-embedding-v4 \
--data embedding_batch_size=1 \
--data max_retries=1 \
--data retry_delay=1import requests
url = "https://api.example.com/api/kb/ingest-web"
payload = {
"collection": "<string>",
"start_url": "<string>",
"max_pages": "100",
"max_depth": "3",
"url_patterns": "<string>",
"exclude_patterns": "<string>",
"same_domain_only": "true",
"content_selector": "<string>",
"remove_selectors": "<string>",
"concurrent_requests": "3",
"request_delay": "1",
"timeout": "30",
"respect_robots_txt": "true",
"chunk_size": "1",
"chunk_overlap": "1",
"separators": "<string>",
"embedding_model_id": "text-embedding-v4",
"embedding_batch_size": "1",
"max_retries": "1",
"retry_delay": "1"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/x-www-form-urlencoded"
}
response = requests.post(url, data=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {
Authorization: 'Bearer <token>',
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams({
collection: '<string>',
start_url: '<string>',
max_pages: '100',
max_depth: '3',
url_patterns: '<string>',
exclude_patterns: '<string>',
same_domain_only: 'true',
content_selector: '<string>',
remove_selectors: '<string>',
concurrent_requests: '3',
request_delay: '1',
timeout: '30',
respect_robots_txt: 'true',
chunk_size: '1',
chunk_overlap: '1',
separators: '<string>',
embedding_model_id: 'text-embedding-v4',
embedding_batch_size: '1',
max_retries: '1',
retry_delay: '1'
})
};
fetch('https://api.example.com/api/kb/ingest-web', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/kb/ingest-web",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => "collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/x-www-form-urlencoded"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/kb/ingest-web"
payload := strings.NewReader("collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/kb/ingest-web")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/x-www-form-urlencoded")
.body("collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/kb/ingest-web")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/x-www-form-urlencoded'
request.body = "collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1"
response = http.request(request)
puts response.read_body{
"status": "<string>",
"collection": "<string>",
"total_urls_found": 1,
"pages_crawled": 1,
"pages_failed": 1,
"documents_created": 1,
"chunks_created": 1,
"embeddings_created": 1,
"message": "<string>",
"elapsed_time_ms": 1,
"crawled_urls": [
"<string>"
],
"failed_urls": {},
"warnings": [
"<string>"
],
"side_effects_may_remain": false
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Ingest Web
Ingest website content into the knowledge base.
Args: collection: Target collection name start_url: Starting URL for crawling max_pages: Maximum number of pages to crawl max_depth: Maximum crawl depth url_patterns: Comma-separated URL match patterns (regex) exclude_patterns: Comma-separated exclusion patterns (regex) same_domain_only: Only crawl same domain content_selector: CSS selector for main content area remove_selectors: Comma-separated CSS selectors to remove concurrent_requests: Number of concurrent requests request_delay: Delay between requests in seconds timeout: Request timeout in seconds respect_robots_txt: Respect robots.txt rules parse_method: Parser for document ingestion chunk_strategy: Chunking strategy chunk_size: Chunk size in characters chunk_overlap: Chunk overlap in characters embedding_model_id: Embedding model ID embedding_batch_size: Batch size for embedding max_retries: Maximum retry attempts retry_delay: Delay between retries
curl --request POST \
--url https://api.example.com/api/kb/ingest-web \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data 'collection=<string>' \
--data 'start_url=<string>' \
--data max_pages=100 \
--data max_depth=3 \
--data 'url_patterns=<string>' \
--data 'exclude_patterns=<string>' \
--data same_domain_only=true \
--data 'content_selector=<string>' \
--data 'remove_selectors=<string>' \
--data concurrent_requests=3 \
--data request_delay=1 \
--data timeout=30 \
--data respect_robots_txt=true \
--data chunk_size=1 \
--data chunk_overlap=1 \
--data 'separators=<string>' \
--data embedding_model_id=text-embedding-v4 \
--data embedding_batch_size=1 \
--data max_retries=1 \
--data retry_delay=1import requests
url = "https://api.example.com/api/kb/ingest-web"
payload = {
"collection": "<string>",
"start_url": "<string>",
"max_pages": "100",
"max_depth": "3",
"url_patterns": "<string>",
"exclude_patterns": "<string>",
"same_domain_only": "true",
"content_selector": "<string>",
"remove_selectors": "<string>",
"concurrent_requests": "3",
"request_delay": "1",
"timeout": "30",
"respect_robots_txt": "true",
"chunk_size": "1",
"chunk_overlap": "1",
"separators": "<string>",
"embedding_model_id": "text-embedding-v4",
"embedding_batch_size": "1",
"max_retries": "1",
"retry_delay": "1"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/x-www-form-urlencoded"
}
response = requests.post(url, data=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {
Authorization: 'Bearer <token>',
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams({
collection: '<string>',
start_url: '<string>',
max_pages: '100',
max_depth: '3',
url_patterns: '<string>',
exclude_patterns: '<string>',
same_domain_only: 'true',
content_selector: '<string>',
remove_selectors: '<string>',
concurrent_requests: '3',
request_delay: '1',
timeout: '30',
respect_robots_txt: 'true',
chunk_size: '1',
chunk_overlap: '1',
separators: '<string>',
embedding_model_id: 'text-embedding-v4',
embedding_batch_size: '1',
max_retries: '1',
retry_delay: '1'
})
};
fetch('https://api.example.com/api/kb/ingest-web', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/kb/ingest-web",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => "collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/x-www-form-urlencoded"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/kb/ingest-web"
payload := strings.NewReader("collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/kb/ingest-web")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/x-www-form-urlencoded")
.body("collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/kb/ingest-web")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/x-www-form-urlencoded'
request.body = "collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1"
response = http.request(request)
puts response.read_body{
"status": "<string>",
"collection": "<string>",
"total_urls_found": 1,
"pages_crawled": 1,
"pages_failed": 1,
"documents_created": 1,
"chunks_created": 1,
"embeddings_created": 1,
"message": "<string>",
"elapsed_time_ms": 1,
"crawled_urls": [
"<string>"
],
"failed_urls": {},
"warnings": [
"<string>"
],
"side_effects_may_remain": false
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
Target collection name
Starting URL for crawling
Maximum number of pages to crawl (default: 100)
Maximum crawl depth (default: 3)
Comma-separated URL match patterns (regex)
Comma-separated exclusion patterns (regex)
Only crawl same domain (default: True)
CSS selector for main content area
Comma-separated CSS selectors to remove
Concurrent requests (default: 3, max: 10)
1 <= x <= 10Delay between requests in seconds (default: 1.0)
x >= 0Request timeout in seconds (default: 30)
x >= 1Respect robots.txt (default: True)
Parser used during ingestion
default, pypdf, pdfplumber, unstructured, pymupdf, deepdoc Chunking strategy
recursive, fixed_size, markdown Chunk size in characters (default: 1000)
x > 0Chunk overlap (default: 200)
x >= 0Custom chunk separators as JSON array of strings; only used when chunk_strategy is recursive.
Embedding model ID
Batch size for embedding (default: 10)
x > 0Maximum retries for embedding failures (default: 3)
x >= 0Delay between retries in seconds (default: 1.0)
x >= 0Response
Successful Response
Result of website crawling and knowledge base ingestion.
This model provides comprehensive statistics about the crawling and ingestion process, including success/failure counts and timing.
Overall status: success|error|partial
Target collection name
Total number of unique URLs discovered
x >= 0Number of successfully crawled pages
x >= 0Number of pages that failed to crawl
x >= 0Number of documents created in collection
x >= 0Total number of chunks created
x >= 0Total number of embeddings generated
x >= 0Human-readable summary message
Total elapsed time in milliseconds
x >= 0List of successfully crawled URLs
Map of failed URLs to error messages
Show child attributes
Show child attributes
Non-critical warnings
Whether rollback failed after an ingestion error and persisted side effects may still exist

