FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fix(elasticsearch): resolve Cloud ID to the real ES host and stop the timeout param collision by waleedlatif1 · Pull Request #7260 · simstudioai/sim · GitHub

This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ Retrieve index information including settings, mappings, and aliases.

| Parameter | Type | Description |
| --------- | ---- | ----------- |
| `index` | json | Index information including aliases, mappings, and settings |
| `indices` | json | Matched indices keyed by index name, each with its aliases, mappings, and settings |

### Elasticsearch Cluster Health

Expand All @@ -324,7 +324,7 @@ Get the health status of the Elasticsearch cluster.
| `username` | string | No | Username for basic auth |
| `password` | string | No | Password for basic auth |
| `waitForStatus` | string | No | Wait until cluster reaches this status: green, yellow, or red |
| `timeout` | string | No | Timeout for the wait operation \(e.g., 30s, 1m\) |
| `clusterTimeout` | string | No | Server-side wait timeout passed to Elasticsearch as the `timeout` query parameter \(e.g., 30s, 1m\) |

#### Output

Expand Down
16 changes: 12 additions & 4 deletions apps/sim/blocks/blocks/elasticsearch.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -523,13 +523,21 @@ Return ONLY valid JSON - no explanations, no markdown code blocks.`,
// Return the operation as the tool ID
return params.operation || 'elasticsearch_search'
},
/**
* The `timeout` subBlock id is retained so saved workflow state keeps
* resolving, but it is remapped onto `clusterTimeout` and cleared here:
* `params.timeout` is read by the request transport as the outbound HTTP
* deadline in milliseconds, which would abort the request long before
* Elasticsearch finished its own server-side wait.
*/
params: (params) => {
const result: Record<string, unknown> = {}
const result: Record<string, unknown> = { timeout: undefined }
if (params.size) result.size = Number(params.size)
if (params.from) result.from = Number(params.from)
if (params.retryOnConflict) result.retryOnConflict = Number(params.retryOnConflict)
if (params.timeout && typeof params.timeout === 'string') {
result.timeout = params.timeout.endsWith('s') ? params.timeout : `${params.timeout}s`
if (typeof params.timeout === 'string' && params.timeout.trim()) {
const timeout = params.timeout.trim()
result.clusterTimeout = /^\d+$/.test(timeout) ? `${timeout}s` : timeout
}
return result
},
Expand Down Expand Up @@ -559,7 +567,7 @@ Return ONLY valid JSON - no explanations, no markdown code blocks.`,
mappings: { type: 'string', description: 'Index mappings as JSON' },
refresh: { type: 'string', description: 'Refresh policy' },
waitForStatus: { type: 'string', description: 'Wait for cluster status' },
timeout: { type: 'string', description: 'Timeout for wait operations' },
timeout: { type: 'string', description: 'Server-side wait timeout for cluster health' },
Comment thread
waleedlatif1 marked this conversation as resolved.
retryOnConflict: { type: 'number', description: 'Retry attempts on conflict' },
},

Expand Down
57 changes: 15 additions & 42 deletions apps/sim/tools/elasticsearch/bulk.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,9 @@ import type {
ElasticsearchBulkParams,
ElasticsearchBulkResponse,
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
Comment thread
waleedlatif1 marked this conversation as resolved.
import type { ToolConfig } from '@/tools/types'

function buildBaseUrl(params: ElasticsearchBulkParams): string {
if (params.deploymentType === 'cloud' && params.cloudId) {
const parts = params.cloudId.split(':')
if (parts.length >= 2) {
try {
const decoded = Buffer.from(parts[1], 'base64').toString('utf-8')
const [esHost] = decoded.split('$')
if (esHost) {
return `https://${parts[0]}.${esHost}`
}
} catch {
// Fallback
}
}
throw new Error('Invalid Cloud ID format')
}

if (!params.host) {
throw new Error('Host is required for self-hosted deployments')
}

return params.host.replace(/\/$/, '')
}

function buildAuthHeaders(params: ElasticsearchBulkParams): Record<string, string> {
const headers: Record<string, string> = {
'Content-Type': 'application/x-ndjson',
}

if (params.authMethod === 'api_key' && params.apiKey) {
headers.Authorization = `ApiKey ${params.apiKey}`
} else if (params.authMethod === 'basic_auth' && params.username && params.password) {
const credentials = Buffer.from(`${params.username}:${params.password}`).toString('base64')
headers.Authorization = `Basic ${credentials}`
} else {
throw new Error('Invalid authentication configuration')
}

return headers
}

export const bulkTool: ToolConfig<ElasticsearchBulkParams, ElasticsearchBulkResponse> = {
id: 'elasticsearch_bulk',
name: 'Elasticsearch Bulk Operations',
Expand Down Expand Up @@ -127,7 +87,20 @@ export const bulkTool: ToolConfig<ElasticsearchBulkParams, ElasticsearchBulkResp
return url
},
method: 'POST',
headers: (params) => buildAuthHeaders(params),
headers: (params) => buildAuthHeaders(params, 'application/x-ndjson'),
/**
* `host` is a user-supplied origin, so a redirect to a *different* origin
* must not carry the API key or Basic credentials with it. Credentials are
* kept on a same-origin hop: a reverse proxy in front of Elasticsearch can
* legitimately redirect within its own origin, and dropping the header
* there would turn a valid request into a 401.
*
* `mode: 'legacy'` preserves the existing method and body replay semantics,
* so the only behavior change is the cross-origin credential strip. Under
* `'standard'` a 301/302 would rewrite POST to GET, breaking the `_search`,
* `_count` and `_bulk` calls.
*/
redirectPolicy: () => ({ mode: 'legacy', sendCredentialsOnCrossOriginRedirect: false }),
body: (params) => {
// The body should be NDJSON format - we pass it as raw string
// Ensure it ends with a newline
Expand Down
104 changes: 104 additions & 0 deletions apps/sim/tools/elasticsearch/cluster_health.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { ElasticsearchBlock } from '@/blocks/blocks/elasticsearch'
import { clusterHealthTool } from '@/tools/elasticsearch/cluster_health'
import { getIndexTool } from '@/tools/elasticsearch/get_index'
import type { ElasticsearchClusterHealthParams } from '@/tools/elasticsearch/types'
import { prepareToolRequest } from '@/tools/request-transport'
import type { ToolConfig } from '@/tools/types'

const CONNECTION = {
deploymentType: 'self_hosted',
host: 'https://es.example.com',
authMethod: 'api_key',
apiKey: 'test-key',
} as const

function clusterHealthUrl(overrides: Partial<ElasticsearchClusterHealthParams>): string {
const build = clusterHealthTool.request.url
if (typeof build !== 'function')
throw new Error('clusterHealthTool.request.url is not a function')
return build({ ...CONNECTION, ...overrides } as ElasticsearchClusterHealthParams)
}

describe('elasticsearch_cluster_health timeout param', () => {
it('does not declare a param named timeout, which the transport reads as an HTTP deadline', () => {
expect(clusterHealthTool.params).not.toHaveProperty('timeout')
expect(clusterHealthTool.params).toHaveProperty('clusterTimeout')
})

it('sends clusterTimeout as the Elasticsearch timeout query parameter', () => {
const url = new URL(clusterHealthUrl({ clusterTimeout: '30s' }))
expect(url.searchParams.get('timeout')).toBe('30s')
})

it('omits the query parameter when no timeout is supplied', () => {
expect(new URL(clusterHealthUrl({})).searchParams.get('timeout')).toBeNull()
})

it('never sets an outbound HTTP deadline from the wait timeout', () => {
const mapped = ElasticsearchBlock.tools.config?.params?.({
operation: 'elasticsearch_cluster_health',
timeout: '30',
})
expect(mapped?.clusterTimeout).toBe('30s')

const prepared = prepareToolRequest(clusterHealthTool as ToolConfig, {
...CONNECTION,
...mapped,
})
expect(prepared.timeout).toBeUndefined()
expect(new URL(prepared.url).searchParams.get('timeout')).toBe('30s')
})

it('preserves a unit-bearing timeout instead of appending a second unit', () => {
const mapped = ElasticsearchBlock.tools.config?.params?.({
operation: 'elasticsearch_cluster_health',
timeout: '1m',
})
expect(mapped?.clusterTimeout).toBe('1m')
})
})

describe('elasticsearch_get_index response shape', () => {
async function transform(body: unknown) {
const response = new Response(JSON.stringify(body), {
status: 200,
headers: { 'Content-Type': 'application/json' },
})
return getIndexTool.transformResponse?.(response, {})
}

it('declares only output keys that the transform actually returns', async () => {
const result = await transform({ 'logs-2024': { aliases: {}, mappings: {}, settings: {} } })
const returned = Object.keys(result?.output ?? {})
for (const declared of Object.keys(getIndexTool.outputs ?? {})) {
expect(returned).toContain(declared)
}
})

it('lets a real index named "indices" keep its own saved reference', async () => {
const result = await transform({ indices: { mappings: { properties: { sku: {} } } } })
const output = result?.output as { indices: { mappings?: Record<string, unknown> } }
expect(output.indices.mappings).toEqual({ properties: { sku: {} } })
})

it('keeps the previous top-level index keys so saved references still resolve', async () => {
const result = await transform({ products: { mappings: { properties: { sku: {} } } } })
const output = result?.output as Record<string, { mappings?: Record<string, unknown> }>
expect(output.products.mappings).toEqual({ properties: { sku: {} } })
})

it('keeps every index a wildcard request matched', async () => {
const result = await transform({
'logs-2024.01': { aliases: {}, mappings: {}, settings: {} },
'logs-2024.02': { aliases: {}, mappings: {}, settings: {} },
})
expect(Object.keys((result?.output as { indices: Record<string, unknown> }).indices)).toEqual([
'logs-2024.01',
'logs-2024.02',
])
})
})
64 changes: 19 additions & 45 deletions apps/sim/tools/elasticsearch/cluster_health.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,9 @@ import type {
ElasticsearchClusterHealthParams,
ElasticsearchClusterHealthResponse,
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'

function buildBaseUrl(params: ElasticsearchClusterHealthParams): string {
if (params.deploymentType === 'cloud' && params.cloudId) {
const parts = params.cloudId.split(':')
if (parts.length >= 2) {
try {
const decoded = Buffer.from(parts[1], 'base64').toString('utf-8')
const [esHost] = decoded.split('$')
if (esHost) {
return `https://${parts[0]}.${esHost}`
}
} catch {
// Fallback
}
}
throw new Error('Invalid Cloud ID format')
}

if (!params.host) {
throw new Error('Host is required for self-hosted deployments')
}

return params.host.replace(/\/$/, '')
}

function buildAuthHeaders(params: ElasticsearchClusterHealthParams): Record<string, string> {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
}

if (params.authMethod === 'api_key' && params.apiKey) {
headers.Authorization = `ApiKey ${params.apiKey}`
} else if (params.authMethod === 'basic_auth' && params.username && params.password) {
const credentials = Buffer.from(`${params.username}:${params.password}`).toString('base64')
headers.Authorization = `Basic ${credentials}`
} else {
throw new Error('Invalid authentication configuration')
}

return headers
}

export const clusterHealthTool: ToolConfig<
ElasticsearchClusterHealthParams,
ElasticsearchClusterHealthResponse
Expand Down Expand Up @@ -100,10 +60,11 @@ export const clusterHealthTool: ToolConfig<
required: false,
description: 'Wait until cluster reaches this status: green, yellow, or red',
},
timeout: {
clusterTimeout: {
type: 'string',
required: false,
description: 'Timeout for the wait operation (e.g., 30s, 1m)',
description:
'Server-side wait timeout passed to Elasticsearch as the `timeout` query parameter (e.g., 30s, 1m)',
},
},

Expand All @@ -116,8 +77,8 @@ export const clusterHealthTool: ToolConfig<
if (params.waitForStatus) {
queryParams.push(`wait_for_status=${params.waitForStatus}`)
}
if (params.timeout) {
queryParams.push(`timeout=${encodeURIComponent(params.timeout)}`)
if (params.clusterTimeout) {
queryParams.push(`timeout=${encodeURIComponent(params.clusterTimeout)}`)
}
if (queryParams.length > 0) {
url += `?${queryParams.join('&')}`
Expand All @@ -127,6 +88,19 @@ export const clusterHealthTool: ToolConfig<
},
method: 'GET',
headers: (params) => buildAuthHeaders(params),
/**
* `host` is a user-supplied origin, so a redirect to a *different* origin
* must not carry the API key or Basic credentials with it. Credentials are
* kept on a same-origin hop: a reverse proxy in front of Elasticsearch can
* legitimately redirect within its own origin, and dropping the header
* there would turn a valid request into a 401.
*
* `mode: 'legacy'` preserves the existing method and body replay semantics,
* so the only behavior change is the cross-origin credential strip. Under
* `'standard'` a 301/302 would rewrite POST to GET, breaking the `_search`,
* `_count` and `_bulk` calls.
*/
redirectPolicy: () => ({ mode: 'legacy', sendCredentialsOnCrossOriginRedirect: false }),
},

transformResponse: async (response: Response) => {
Expand Down
55 changes: 14 additions & 41 deletions apps/sim/tools/elasticsearch/cluster_stats.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,9 @@ import type {
ElasticsearchClusterStatsParams,
ElasticsearchClusterStatsResponse,
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'

function buildBaseUrl(params: ElasticsearchClusterStatsParams): string {
if (params.deploymentType === 'cloud' && params.cloudId) {
const parts = params.cloudId.split(':')
if (parts.length >= 2) {
try {
const decoded = Buffer.from(parts[1], 'base64').toString('utf-8')
const [esHost] = decoded.split('$')
if (esHost) {
return `https://${parts[0]}.${esHost}`
}
} catch {
// Fallback
}
}
throw new Error('Invalid Cloud ID format')
}

if (!params.host) {
throw new Error('Host is required for self-hosted deployments')
}

return params.host.replace(/\/$/, '')
}

function buildAuthHeaders(params: ElasticsearchClusterStatsParams): Record<string, string> {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
}

if (params.authMethod === 'api_key' && params.apiKey) {
headers.Authorization = `ApiKey ${params.apiKey}`
} else if (params.authMethod === 'basic_auth' && params.username && params.password) {
const credentials = Buffer.from(`${params.username}:${params.password}`).toString('base64')
headers.Authorization = `Basic ${credentials}`
} else {
throw new Error('Invalid authentication configuration')
}

return headers
}

export const clusterStatsTool: ToolConfig<
ElasticsearchClusterStatsParams,
ElasticsearchClusterStatsResponse
Expand Down Expand Up @@ -104,6 +64,19 @@ export const clusterStatsTool: ToolConfig<
},
method: 'GET',
headers: (params) => buildAuthHeaders(params),
/**
* `host` is a user-supplied origin, so a redirect to a *different* origin
* must not carry the API key or Basic credentials with it. Credentials are
* kept on a same-origin hop: a reverse proxy in front of Elasticsearch can
* legitimately redirect within its own origin, and dropping the header
* there would turn a valid request into a 401.
*
* `mode: 'legacy'` preserves the existing method and body replay semantics,
* so the only behavior change is the cross-origin credential strip. Under
* `'standard'` a 301/302 would rewrite POST to GET, breaking the `_search`,
* `_count` and `_bulk` calls.
*/
redirectPolicy: () => ({ mode: 'legacy', sendCredentialsOnCrossOriginRedirect: false }),
},

transformResponse: async (response: Response) => {
Expand Down
Loading
Loading

Back | FazBrowse Home | New Git URL