FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

feat:Adapt outgoing images to the formats supported by the provider, with support for animation strategies and local conversion caching. by piexian · Pull Request #9703 · AstrBotDevs/AstrBot · GitHub

39 changes: 39 additions & 0 deletions astrbot/core/config/default.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -2129,6 +2129,45 @@
"labels": ["文本", "图像", "音频", "工具使用"],
"render_type": "checkbox",
"hint": "模型支持的模态及能力。",
"default": ["text", "image", "audio", "tool_use"],
},
"image_formats": {
"description": "图片格式支持",
"type": "list",
"items": {"type": "string"},
"options": ["jpeg", "png", "webp", "gif", "bmp", "heic", "*"],
"labels": [
"JPEG",
"PNG",
"WebP",
"GIF",
"BMP",
"HEIC",
"不限制",
],
"render_type": "checkbox",
"hint": "发送给该提供商前允许的图片格式,不兼容的格式会自动转换。留空使用内置默认值(官方 API 按其文档,未知第三方默认仅 jpeg/png)。勾选「不限制」时其他选项不生效。",
"default": [],
"condition": {"modalities": "image"},
},
"animated_image_strategy": {
"description": "动图处理策略",
"type": "string",
"options": ["first_frame", "multi_frame"],
"labels": ["仅首帧", "多帧抽取"],
"hint": "GIF 等动图发送给模型时的处理方式:仅取首帧(省 token),或按时长均匀抽帧后作为多张图片发送。",
"default": "first_frame",
"condition": {"modalities": "image"},
},
"animated_image_max_frames": {
"description": "动图最大抽帧数",
"type": "int",
"hint": "多帧抽取策略下最多发送的帧数(1-16),默认 4。帧数越多 token 消耗越大。",
"default": 4,
"condition": {
"modalities": "image",
"animated_image_strategy": "multi_frame",
},
},
"custom_headers": {
"description": "自定义请求头",
Expand Down
22 changes: 10 additions & 12 deletions astrbot/core/provider/entities.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from astrbot.core.agent.tool import ToolSet
from astrbot.core.db.po import Conversation
from astrbot.core.message.message_event_result import MessageChain
from astrbot.core.utils.media_utils import MediaResolver
from astrbot.core.utils.media_utils import MediaResolver, resolve_image_ref_to_images


class ProviderType(enum.Enum):
Expand Down Expand Up @@ -208,19 +208,17 @@ async def assemble_context(self) -> dict:
# 3. 图片内容
if self.image_urls:
for image_url in self.image_urls:
image_data = await MediaResolver(
image_url,
media_type="image",
).to_base64_data()
if not image_data:
image_datas = await resolve_image_ref_to_images(image_url)
if not image_datas:
logger.warning("图片预处理结果为空,将忽略。")
continue
content_blocks.append(
{
"type": "image_url",
"image_url": {"url": image_data.to_data_url()},
},
)
for image_data in image_datas:
content_blocks.append(
{
"type": "image_url",
"image_url": {"url": image_data.to_data_url()},
},
)

# 4. 音频内容
if self.audio_urls:
Expand Down
84 changes: 83 additions & 1 deletion astrbot/core/provider/provider.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@
import asyncio
import os
from collections.abc import AsyncGenerator
from typing import Literal, TypeAlias, Union
from typing import ClassVar, Literal, TypeAlias, Union

from astrbot import logger
from astrbot.core.agent.message import ContentPart, Message, is_checkpoint_message
from astrbot.core.agent.tool import ToolSet
from astrbot.core.provider.entities import (
Expand All @@ -14,6 +15,16 @@
)
from astrbot.core.provider.register import provider_cls_map
from astrbot.core.utils.astrbot_path import get_astrbot_path
from astrbot.core.utils.media_utils import (
ANIMATED_DEFAULT_MAX_FRAMES,
ANIMATED_MAX_FRAMES_LIMIT,
ANIMATED_STRATEGY_FIRST_FRAME,
ANIMATED_STRATEGY_MULTI_FRAME,
IMAGE_SHORT_MIME_TYPES,
)

DEFAULT_FALLBACK_IMAGE_FORMATS = frozenset({"image/jpeg", "image/png"})
"""Conservative image formats assumed for providers without a declared set."""

Providers: TypeAlias = Union[
"Provider",
Expand Down Expand Up @@ -66,6 +77,14 @@ async def test(self) -> None:
class Provider(AbstractProvider):
"""Chat Provider"""

supported_image_formats: ClassVar[frozenset[str] | None] = None
"""Image MIME types the provider officially accepts.

``None`` means undeclared and falls back to DEFAULT_FALLBACK_IMAGE_FORMATS.
Aggregator subclasses (e.g. OpenRouter) must set this back to ``None``
explicitly so they do not inherit an official vendor's format set.
"""

def __init__(
self,
provider_config: dict,
Expand All @@ -74,6 +93,69 @@ def __init__(
super().__init__(provider_config)
self.provider_settings = provider_settings

def resolve_allowed_image_formats(self) -> frozenset[str] | None:
Comment thread
piexian marked this conversation as resolved.
"""Resolve the image MIME types allowed for this provider instance.

Priority: ``provider_config["image_formats"]`` (per-instance override,
short names like ``jpeg`` or MIME types, ``*`` disables restriction) >
the class-level ``supported_image_formats`` > the conservative
DEFAULT_FALLBACK_IMAGE_FORMATS (jpeg/png).

Returns:
The allowed MIME types, or ``None`` when unrestricted.
"""
configured = self.provider_config.get("image_formats")
if configured:
normalized = {
str(value).strip().lower() for value in configured if str(value).strip()
}
if "*" in normalized:
return None
mapped = {
IMAGE_SHORT_MIME_TYPES.get(value, value)
for value in normalized
if value.startswith("image/") or value in IMAGE_SHORT_MIME_TYPES
}
if mapped:
return frozenset(mapped)
logger.warning(
"Provider %s: image_formats %s contains no valid entries; "
"falling back to the default format set.",
self.provider_config.get("id"),
sorted(normalized),
)
if self.supported_image_formats is not None:
return self.supported_image_formats
return DEFAULT_FALLBACK_IMAGE_FORMATS

def get_animated_image_strategy(self) -> tuple[str, int]:
"""Read the animated image handling strategy from the provider config.

Returns:
Tuple of ``(strategy, max_frames)`` where strategy is
``first_frame`` or ``multi_frame`` and max_frames is clamped to
``[1, 16]``.
"""
strategy = str(
self.provider_config.get("animated_image_strategy")
or ANIMATED_STRATEGY_FIRST_FRAME
)
if strategy not in (
ANIMATED_STRATEGY_FIRST_FRAME,
ANIMATED_STRATEGY_MULTI_FRAME,
):
strategy = ANIMATED_STRATEGY_FIRST_FRAME
raw_max_frames = self.provider_config.get("animated_image_max_frames")
try:
max_frames = (
ANIMATED_DEFAULT_MAX_FRAMES
if raw_max_frames is None
else int(raw_max_frames)
)
except (TypeError, ValueError):
max_frames = ANIMATED_DEFAULT_MAX_FRAMES
return strategy, min(max(max_frames, 1), ANIMATED_MAX_FRAMES_LIMIT)

@abc.abstractmethod
def get_current_key(self) -> str:
raise NotImplementedError
Expand Down
113 changes: 62 additions & 51 deletions astrbot/core/provider/sources/anthropic_source.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
from astrbot.core.provider.func_tool_manager import ToolSet
from astrbot.core.utils.media_utils import (
describe_media_ref,
resolve_media_ref_to_base64_data,
detect_image_mime_type,
resolve_image_ref_to_images,
)
from astrbot.core.utils.network_utils import (
create_proxy_client,
Expand All @@ -37,6 +38,11 @@
class ProviderAnthropic(Provider):
_PROMPT_CACHE_CONTROL = {"type": "ephemeral"}

supported_image_formats = frozenset(
{"image/jpeg", "image/png", "image/gif", "image/webp"}
)
"""Formats accepted by the official Anthropic vision API."""

@staticmethod
def _ensure_usable_response(
llm_response: LLMResponse,
Expand Down Expand Up @@ -264,19 +270,32 @@ def _prepare_payload(self, messages: list[dict]):
_, base64_data = url.split(",", 1)
# Detect actual image format from binary data
image_bytes = base64.b64decode(base64_data)
media_type = self._detect_image_mime_type(
image_bytes
media_type = detect_image_mime_type(
image_bytes,
default_mime_type=None,
)
converted_content.append(
{
"type": "image",
"source": {
"type": "base64",
"media_type": media_type,
"data": base64_data,
},
}
allowed_formats = (
self.resolve_allowed_image_formats()
)
if media_type and (
allowed_formats is None
or media_type in allowed_formats
):
converted_content.append(
{
"type": "image",
"source": {
"type": "base64",
"media_type": media_type,
"data": base64_data,
},
}
)
else:
logger.warning(
"Skipping context image with unsupported or undetectable format: %s...",
url[:50],
)
except ValueError:
logger.warning(
f"Failed to parse image data URI: {url[:50]}..."
Expand Down Expand Up @@ -884,17 +903,16 @@ async def text_chat_stream(
):
yield llm_response

def _detect_image_mime_type(self, data: bytes) -> str:
"""根据图片二进制数据的 magic bytes 检测 MIME 类型"""
if data[:8] == b"\x89PNG\r\n\x1a\n":
return "image/png"
if data[:2] == b"\xff\xd8":
return "image/jpeg"
if data[:6] in (b"GIF87a", b"GIF89a"):
return "image/gif"
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
return "image/webp"
return "image/jpeg"
async def _image_ref_to_images(self, image_ref: str, *, strict: bool = False):
"""Resolve an image ref with this provider's format adaptation applied."""
strategy, max_frames = self.get_animated_image_strategy()
return await resolve_image_ref_to_images(
image_ref,
allowed_mime_types=self.resolve_allowed_image_formats(),
animated_strategy=strategy,
animated_max_frames=max_frames,
strict=strict,
)

async def assemble_context(
self,
Expand All @@ -905,23 +923,23 @@ async def assemble_context(
):
"""组装上下文,支持文本和图片"""

async def resolve_image_url(image_url: str) -> dict | None:
image_data = await resolve_media_ref_to_base64_data(
image_url,
media_type="image",
)
if not image_data:
async def resolve_image_url(image_url: str) -> list[dict]:
image_datas = await self._image_ref_to_images(image_url)
if not image_datas:
logger.warning("图片预处理结果为空,将忽略。")
return None

return {
"type": "image",
"source": {
"type": "base64",
"media_type": image_data.mime_type,
"data": image_data.base64_data,
},
}
return []

return [
{
"type": "image",
"source": {
"type": "base64",
"media_type": image_data.mime_type,
"data": image_data.base64_data,
},
}
for image_data in image_datas
]

content = []

Expand All @@ -943,9 +961,7 @@ async def resolve_image_url(image_url: str) -> dict | None:
if isinstance(block, TextPart):
content.append({"type": "text", "text": block.text})
elif isinstance(block, ImageURLPart):
image_dict = await resolve_image_url(block.image_url.url)
if image_dict:
content.append(image_dict)
content.extend(await resolve_image_url(block.image_url.url))
elif isinstance(block, AudioURLPart):
content.append({"type": "text", "text": "[Audio]"})
else:
Expand All @@ -954,9 +970,7 @@ async def resolve_image_url(image_url: str) -> dict | None:
# 3. 图片内容
if image_urls:
for image_url in image_urls:
image_dict = await resolve_image_url(image_url)
if image_dict:
content.append(image_dict)
content.extend(await resolve_image_url(image_url))
if audio_urls:
for _audio_path in audio_urls:
content.append({"type": "text", "text": "[Audio]"})
Expand All @@ -977,15 +991,12 @@ async def resolve_image_url(image_url: str) -> dict | None:

async def encode_image_bs64(self, image_url: str) -> tuple[str, str]:
"""将图片转换为 base64,同时检测实际 MIME 类型"""
image_data = await resolve_media_ref_to_base64_data(
image_url,
media_type="image",
strict=True,
)
if image_data is None:
image_datas = await self._image_ref_to_images(image_url, strict=True)
if not image_datas:
raise RuntimeError(
f"Failed to encode image data: {describe_media_ref(image_url)}"
)
image_data = image_datas[0]
return image_data.to_data_url(), image_data.mime_type

def get_current_key(self) -> str:
Expand Down
Loading
Loading

Back | FazBrowse Home | New Git URL