Create Ingest Web Job
curl --request POST \
--url https://api.example.com/api/kb/ingest-web/jobs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data 'collection=<string>' \
--data 'start_url=<string>' \
--data max_pages=100 \
--data max_depth=3 \
--data 'url_patterns=<string>' \
--data 'exclude_patterns=<string>' \
--data same_domain_only=true \
--data 'content_selector=<string>' \
--data 'remove_selectors=<string>' \
--data concurrent_requests=3 \
--data request_delay=1 \
--data timeout=30 \
--data respect_robots_txt=true \
--data chunk_size=1 \
--data chunk_overlap=1 \
--data 'separators=<string>' \
--data embedding_model_id=text-embedding-v4 \
--data embedding_batch_size=1 \
--data max_retries=1 \
--data retry_delay=1import requests
url = "https://api.example.com/api/kb/ingest-web/jobs"
payload = {
"collection": "<string>",
"start_url": "<string>",
"max_pages": "100",
"max_depth": "3",
"url_patterns": "<string>",
"exclude_patterns": "<string>",
"same_domain_only": "true",
"content_selector": "<string>",
"remove_selectors": "<string>",
"concurrent_requests": "3",
"request_delay": "1",
"timeout": "30",
"respect_robots_txt": "true",
"chunk_size": "1",
"chunk_overlap": "1",
"separators": "<string>",
"embedding_model_id": "text-embedding-v4",
"embedding_batch_size": "1",
"max_retries": "1",
"retry_delay": "1"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/x-www-form-urlencoded"
}
response = requests.post(url, data=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {
Authorization: 'Bearer <token>',
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams({
collection: '<string>',
start_url: '<string>',
max_pages: '100',
max_depth: '3',
url_patterns: '<string>',
exclude_patterns: '<string>',
same_domain_only: 'true',
content_selector: '<string>',
remove_selectors: '<string>',
concurrent_requests: '3',
request_delay: '1',
timeout: '30',
respect_robots_txt: 'true',
chunk_size: '1',
chunk_overlap: '1',
separators: '<string>',
embedding_model_id: 'text-embedding-v4',
embedding_batch_size: '1',
max_retries: '1',
retry_delay: '1'
})
};
fetch('https://api.example.com/api/kb/ingest-web/jobs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/kb/ingest-web/jobs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => "collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/x-www-form-urlencoded"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/kb/ingest-web/jobs"
payload := strings.NewReader("collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/kb/ingest-web/jobs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/x-www-form-urlencoded")
.body("collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/kb/ingest-web/jobs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/x-www-form-urlencoded'
request.body = "collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"user_id": 123,
"job_type": "<string>",
"queue": "<string>",
"status": "<string>",
"attempts": 123,
"max_attempts": 123,
"progress": {},
"result": {},
"error_message": "<string>",
"celery_task_id": "<string>",
"started_at": "2023-11-07T05:31:56Z",
"finished_at": "2023-11-07T05:31:56Z",
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Knowledge Base
Create Ingest Web Job
Enqueue durable website ingestion into the knowledge base.
POST
/
api
/
kb
/
ingest-web
/
jobs
Create Ingest Web Job
curl --request POST \
--url https://api.example.com/api/kb/ingest-web/jobs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data 'collection=<string>' \
--data 'start_url=<string>' \
--data max_pages=100 \
--data max_depth=3 \
--data 'url_patterns=<string>' \
--data 'exclude_patterns=<string>' \
--data same_domain_only=true \
--data 'content_selector=<string>' \
--data 'remove_selectors=<string>' \
--data concurrent_requests=3 \
--data request_delay=1 \
--data timeout=30 \
--data respect_robots_txt=true \
--data chunk_size=1 \
--data chunk_overlap=1 \
--data 'separators=<string>' \
--data embedding_model_id=text-embedding-v4 \
--data embedding_batch_size=1 \
--data max_retries=1 \
--data retry_delay=1import requests
url = "https://api.example.com/api/kb/ingest-web/jobs"
payload = {
"collection": "<string>",
"start_url": "<string>",
"max_pages": "100",
"max_depth": "3",
"url_patterns": "<string>",
"exclude_patterns": "<string>",
"same_domain_only": "true",
"content_selector": "<string>",
"remove_selectors": "<string>",
"concurrent_requests": "3",
"request_delay": "1",
"timeout": "30",
"respect_robots_txt": "true",
"chunk_size": "1",
"chunk_overlap": "1",
"separators": "<string>",
"embedding_model_id": "text-embedding-v4",
"embedding_batch_size": "1",
"max_retries": "1",
"retry_delay": "1"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/x-www-form-urlencoded"
}
response = requests.post(url, data=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {
Authorization: 'Bearer <token>',
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams({
collection: '<string>',
start_url: '<string>',
max_pages: '100',
max_depth: '3',
url_patterns: '<string>',
exclude_patterns: '<string>',
same_domain_only: 'true',
content_selector: '<string>',
remove_selectors: '<string>',
concurrent_requests: '3',
request_delay: '1',
timeout: '30',
respect_robots_txt: 'true',
chunk_size: '1',
chunk_overlap: '1',
separators: '<string>',
embedding_model_id: 'text-embedding-v4',
embedding_batch_size: '1',
max_retries: '1',
retry_delay: '1'
})
};
fetch('https://api.example.com/api/kb/ingest-web/jobs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/kb/ingest-web/jobs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => "collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/x-www-form-urlencoded"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/kb/ingest-web/jobs"
payload := strings.NewReader("collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/kb/ingest-web/jobs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/x-www-form-urlencoded")
.body("collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/kb/ingest-web/jobs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/x-www-form-urlencoded'
request.body = "collection=%3Cstring%3E&start_url=%3Cstring%3E&max_pages=100&max_depth=3&url_patterns=%3Cstring%3E&exclude_patterns=%3Cstring%3E&same_domain_only=true&content_selector=%3Cstring%3E&remove_selectors=%3Cstring%3E&concurrent_requests=3&request_delay=1&timeout=30&respect_robots_txt=true&chunk_size=1&chunk_overlap=1&separators=%3Cstring%3E&embedding_model_id=text-embedding-v4&embedding_batch_size=1&max_retries=1&retry_delay=1"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"user_id": 123,
"job_type": "<string>",
"queue": "<string>",
"status": "<string>",
"attempts": 123,
"max_attempts": 123,
"progress": {},
"result": {},
"error_message": "<string>",
"celery_task_id": "<string>",
"started_at": "2023-11-07T05:31:56Z",
"finished_at": "2023-11-07T05:31:56Z",
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
application/x-www-form-urlencoded
Target collection name
Starting URL for crawling
Required range:
1 <= x <= 10Required range:
x >= 0Required range:
x >= 1Available parsing methods
Available options:
default, pypdf, pdfplumber, unstructured, pymupdf, deepdoc Available chunk strategies
Available options:
recursive, fixed_size, markdown Required range:
x > 0Required range:
x >= 0Required range:
x > 0Required range:
x >= 0Required range:
x >= 0Response
Successful Response
⌘I

