feat: configurable SubQuestions via Step 1.2 system prompt page
- Split 'Step 1: Query Decomposition' into Step 1.1 (prompt template) and Step 1.2 (format config with description + max_length) - Add create_subquestions_model() and parse_decompose_format() to decompose.py - QueryDecomposer reads decompose_format from DB, creates dynamic Pydantic model at runtime - PromptEditor renders Step 1.2 as textarea (description) + number input (max_length 1-5) - Graceful fallback to static SubQuestions when decompose_format unavailable
This commit is contained in:
Woody
parent
40b338d3ca
commit
76c3bec2ab
|
|
@ -11,12 +11,18 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
_SEED_DECOMPOSE = (
|
||||
"Given this question: '{question}'\n\n"
|
||||
"Break it down into 2-5 simplified sub-questions that would help "
|
||||
"Break it down into 1-3 simplified sub-questions that would help "
|
||||
"search for relevant information. Each sub-question should be short "
|
||||
"and focused on one aspect.\n\n"
|
||||
'Return a JSON array of strings: ["sub-question 1", "sub-question 2", ...]'
|
||||
)
|
||||
|
||||
_SEED_DECOMPOSE_FORMAT = (
|
||||
'{"description": "請將問題/任務拆解成 1-3 個簡化子問題，標籤式主題必須清楚、簡潔、具體，'
|
||||
'一看就明白（建議 3-8 個字），若涉及地點、地區、人物、時間、金額/財政 等關鍵資訊，必須包含在標籤中 。'
|
||||
'具體提問/要求要精準、完整 並全部轉換成以下固定格式：\\n「標籤式主題：具體提問/要求」", "max_length": 3}'
|
||||
)
|
||||
|
||||
_SEED_FILTER = (
|
||||
"Given question '{question}' and these document chunks, rate each 0-10 for relevance. "
|
||||
"Return JSON array of scores.\n{chunks}\n"
|
||||
|
|
@ -68,6 +74,7 @@ _SEED_PROFILES = [
|
|||
|
||||
_SEED_STEPS = [
|
||||
"decompose",
|
||||
"decompose_format",
|
||||
"filter",
|
||||
"generate",
|
||||
"generate_per_subq",
|
||||
|
|
@ -77,6 +84,7 @@ _SEED_STEPS = [
|
|||
]
|
||||
_SEED_TEMPLATES = {
|
||||
"decompose": _SEED_DECOMPOSE,
|
||||
"decompose_format": _SEED_DECOMPOSE_FORMAT,
|
||||
"filter": _SEED_FILTER,
|
||||
"generate": _SEED_GENERATE,
|
||||
"generate_per_subq": _SEED_GENERATE_PER_SUBQ,
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
from pydantic import BaseModel, Field
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, create_model
|
||||
|
||||
|
||||
class SubQuestions(BaseModel):
|
||||
"""Structured output model for query decomposition.
|
||||
"""Structured output model for query decomposition — static fallback.
|
||||
|
||||
Used with LangChain's with_structured_output() to guarantee
|
||||
the LLM returns a valid list of sub-questions.
|
||||
When ``decompose_format`` is available from PromptService, a dynamic
|
||||
model with DB-configured description and max_length is used instead.
|
||||
See ``create_subquestions_model()``.
|
||||
"""
|
||||
|
||||
questions: list[str] = Field(
|
||||
|
|
@ -13,3 +16,54 @@ class SubQuestions(BaseModel):
|
|||
min_length=1,
|
||||
max_length=3,
|
||||
)
|
||||
|
||||
|
||||
def create_subquestions_model(description: str, max_length: int) -> type[BaseModel]:
|
||||
"""Create a dynamic SubQuestions model with configurable field constraints.
|
||||
|
||||
Args:
|
||||
description: Field description text injected into the JSON format
|
||||
instruction sent to the LLM.
|
||||
max_length: Maximum number of sub-questions (1-5).
|
||||
|
||||
Returns:
|
||||
A Pydantic BaseModel subclass with the configured constraints.
|
||||
"""
|
||||
return create_model(
|
||||
"SubQuestions",
|
||||
questions=(
|
||||
list[str],
|
||||
Field(
|
||||
description=description,
|
||||
min_length=1,
|
||||
max_length=max_length,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def parse_decompose_format(raw: str) -> dict[str, Any]:
|
||||
"""Parse the decompose_format JSON string into a validated config dict.
|
||||
|
||||
Returns a dict with ``description`` (str) and ``max_length`` (int).
|
||||
Raises ``ValueError`` if the JSON is invalid or fields are missing/malformed.
|
||||
"""
|
||||
import json
|
||||
|
||||
try:
|
||||
config = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"decompose_format is not valid JSON: {exc}") from exc
|
||||
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError("decompose_format must be a JSON object")
|
||||
|
||||
description = config.get("description")
|
||||
if not isinstance(description, str) or not description.strip():
|
||||
raise ValueError("decompose_format.description must be a non-empty string")
|
||||
|
||||
max_length = config.get("max_length")
|
||||
if not isinstance(max_length, int) or max_length < 1 or max_length > 5:
|
||||
raise ValueError("decompose_format.max_length must be an integer between 1 and 5")
|
||||
|
||||
return {"description": description.strip(), "max_length": max_length}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ router = APIRouter(prefix="/api/v1/prompts", tags=["prompts"])
|
|||
_VALID_NAMES = {"A", "B", "C"}
|
||||
_VALID_STEPS = {
|
||||
"decompose",
|
||||
"decompose_format",
|
||||
"filter",
|
||||
"generate",
|
||||
"generate_per_subq",
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ logger = logging.getLogger(__name__)
|
|||
_VALID_NAMES = {"A", "B", "C"}
|
||||
_VALID_STEPS = {
|
||||
"decompose",
|
||||
"decompose_format",
|
||||
"filter",
|
||||
"generate",
|
||||
"generate_per_subq",
|
||||
|
|
|
|||
|
|
@ -24,11 +24,21 @@ logger = logging.getLogger(__name__)
|
|||
# Fallback template used when prompt_service is not provided (tests, standalone).
|
||||
_BUILTIN_DECOMPOSE_TEMPLATE = (
|
||||
"Given this question: '{question}'\n\n"
|
||||
"Break it down into 2-5 simplified sub-questions that would help "
|
||||
"Break it down into 1-3 simplified sub-questions that would help "
|
||||
"search for relevant information. Each sub-question should be short "
|
||||
"and focused on one aspect. Return as a JSON array of strings."
|
||||
)
|
||||
|
||||
_DEFAULT_DECOMPOSE_FORMAT = {
|
||||
"description": (
|
||||
"請將問題/任務拆解成 1-3 個簡化子問題，標籤式主題必須清楚、簡潔、具體，"
|
||||
"一看就明白（建議 3-8 個字），若涉及地點、地區、人物、時間、金額/財政 等關鍵資訊，"
|
||||
"必須包含在標籤中 。具體提問/要求要精準、完整 並全部轉換成以下固定格式：\n"
|
||||
"「標籤式主題：具體提問/要求」"
|
||||
),
|
||||
"max_length": 3,
|
||||
}
|
||||
|
||||
|
||||
def _extract_json_from_markdown(response: str) -> str:
|
||||
if not isinstance(response, str):
|
||||
|
|
@ -98,12 +108,32 @@ class QueryDecomposer:
|
|||
|
||||
prompt = template.replace("{question}", question)
|
||||
|
||||
from app.models.decompose import SubQuestions
|
||||
from app.models.decompose import SubQuestions, create_subquestions_model, parse_decompose_format
|
||||
|
||||
pydantic_model = SubQuestions
|
||||
|
||||
if self._prompt_service is not None:
|
||||
try:
|
||||
format_raw = self._prompt_service.get_prompt_template("decompose_format")
|
||||
format_config = parse_decompose_format(format_raw)
|
||||
pydantic_model = create_subquestions_model(
|
||||
description=format_config["description"],
|
||||
max_length=format_config["max_length"],
|
||||
)
|
||||
logger.info(
|
||||
"Decompose format loaded from DB: max_length=%d",
|
||||
format_config["max_length"],
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to load decompose_format from DB: %s. Falling back to static SubQuestions.",
|
||||
exc,
|
||||
)
|
||||
|
||||
try:
|
||||
result = await self.llm_client.complete_structured(
|
||||
prompt=prompt,
|
||||
pydantic_model=SubQuestions,
|
||||
pydantic_model=pydantic_model,
|
||||
step_name="QueryDecomposer",
|
||||
)
|
||||
return result.questions, prompt
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ export interface PromptEditorProps {
|
|||
}
|
||||
|
||||
const STEPS = [
|
||||
{ key: 'decompose', label: 'Step 1: Query Decomposition', placeholders: ['question'] },
|
||||
{ key: 'decompose', label: 'Step 1.1: Query Decomposition', placeholders: ['question'] },
|
||||
{ key: 'decompose_format', label: 'Step 1.2: Query Decomposition Format', placeholders: [], type: 'format_config' },
|
||||
{ key: 'filter_intro', label: 'Step 2.1: Filter Intro (Preamble)', placeholders: [] },
|
||||
{ key: 'filter_section', label: 'Step 2.2: Filter Section (Per Sub-Q)', placeholders: ['subq_idx', 'subq_question', 'chunks'] },
|
||||
{ key: 'filter_outro', label: 'Step 2.3: Filter Outro (Format)', placeholders: [] },
|
||||
|
|
@ -23,6 +24,7 @@ const STEPS = [
|
|||
|
||||
const VALID_PLACEHOLDERS: Record<string, string[]> = {
|
||||
decompose: ['{question}'],
|
||||
decompose_format: [],
|
||||
filter: ['{question}', '{chunks}'],
|
||||
generate: ['{question}', '{context}'],
|
||||
generate_per_subq: ['{context_sections}'],
|
||||
|
|
@ -37,6 +39,68 @@ const findUnknownPlaceholders = (template: string, stepKey: string): string[] =>
|
|||
return found.filter((p) => !valid.includes(p))
|
||||
}
|
||||
|
||||
const DEFAULT_FORMAT_CONFIG = { description: '', max_length: 3 }
|
||||
|
||||
function parseFormatConfig(raw: string): { description: string; max_length: number } {
|
||||
try {
|
||||
const parsed = JSON.parse(raw)
|
||||
return {
|
||||
description: typeof parsed.description === 'string' ? parsed.description : '',
|
||||
max_length: typeof parsed.max_length === 'number' ? parsed.max_length : 3,
|
||||
}
|
||||
} catch {
|
||||
return { ...DEFAULT_FORMAT_CONFIG }
|
||||
}
|
||||
}
|
||||
|
||||
function serializeFormatConfig(description: string, max_length: number): string {
|
||||
return JSON.stringify({ description, max_length })
|
||||
}
|
||||
|
||||
interface FormatConfigFieldsProps {
|
||||
value: string
|
||||
disabled: boolean
|
||||
onChange: (description: string, max_length: number) => void
|
||||
}
|
||||
|
||||
const FormatConfigFields: React.FC<FormatConfigFieldsProps> = ({ value, disabled, onChange }) => {
|
||||
const config = parseFormatConfig(value)
|
||||
|
||||
return (
|
||||
<div className="space-y-3 pl-1">
|
||||
<div className="space-y-1">
|
||||
<label className="text-xs font-medium text-gray-600">Description (field description for sub-question format)</label>
|
||||
<textarea
|
||||
value={config.description}
|
||||
onChange={(e) => onChange(e.target.value, config.max_length)}
|
||||
disabled={disabled}
|
||||
rows={5}
|
||||
className="w-full rounded border border-gray-300 px-3 py-2 font-mono text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent disabled:bg-gray-100 disabled:cursor-not-allowed resize-y"
|
||||
/>
|
||||
<span className="text-xs text-gray-500">{config.description.length} characters</span>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
<label className="text-xs font-medium text-gray-600">Max Sub-Questions (1–5)</label>
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={5}
|
||||
value={config.max_length}
|
||||
onChange={(e) => {
|
||||
const v = parseInt(e.target.value, 10)
|
||||
if (!isNaN(v) && v >= 1 && v <= 5) {
|
||||
onChange(config.description, v)
|
||||
}
|
||||
}}
|
||||
disabled={disabled}
|
||||
className="w-24 rounded border border-gray-300 px-3 py-1.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent disabled:bg-gray-100 disabled:cursor-not-allowed"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export const PromptEditor: React.FC<PromptEditorProps> = ({
|
||||
profileName,
|
||||
prompts,
|
||||
|
|
@ -80,7 +144,9 @@ export const PromptEditor: React.FC<PromptEditorProps> = ({
|
|||
|
||||
{STEPS.map((step) => {
|
||||
const value = localPrompts[step.key] ?? ''
|
||||
const unknownPlaceholders = findUnknownPlaceholders(value, step.key)
|
||||
const unknownPlaceholders = step.type === 'format_config'
|
||||
? []
|
||||
: findUnknownPlaceholders(value, step.key)
|
||||
const placeholderBadges = step.placeholders.map((p) => `{${p}}`).join(', ')
|
||||
|
||||
return (
|
||||
|
|
@ -99,39 +165,49 @@ export const PromptEditor: React.FC<PromptEditorProps> = ({
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="text-xs text-gray-500">Placeholders:</span>
|
||||
{step.placeholders.map((p) => (
|
||||
<code
|
||||
key={p}
|
||||
className="bg-gray-100 px-1.5 py-0.5 rounded text-xs font-mono text-gray-700"
|
||||
>
|
||||
{'{'}{p}{'}'}
|
||||
</code>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<textarea
|
||||
value={value}
|
||||
onChange={(e) => handleChange(step.key, e.target.value)}
|
||||
disabled={isSaving}
|
||||
rows={6}
|
||||
className="w-full min-h-[8rem] rounded border border-gray-300 px-3 py-2 font-mono text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent disabled:bg-gray-100 disabled:cursor-not-allowed resize-y"
|
||||
/>
|
||||
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs text-gray-500">{value.length} characters</span>
|
||||
|
||||
{unknownPlaceholders.length > 0 && (
|
||||
<div className="flex items-center gap-1.5 bg-yellow-50 border border-yellow-200 rounded px-2 py-1">
|
||||
<AlertTriangle className="w-3.5 h-3.5 text-yellow-600" />
|
||||
<span className="text-xs text-yellow-700">
|
||||
Unknown placeholder{unknownPlaceholders.length > 1 ? 's' : ''}:{' '}
|
||||
{unknownPlaceholders.join(', ')}
|
||||
</span>
|
||||
{step.type === 'format_config' ? (
|
||||
<FormatConfigFields
|
||||
value={value}
|
||||
disabled={isSaving}
|
||||
onChange={(desc, ml) => handleChange(step.key, serializeFormatConfig(desc, ml))}
|
||||
/>
|
||||
) : (
|
||||
<>
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="text-xs text-gray-500">Placeholders:</span>
|
||||
{step.placeholders.map((p) => (
|
||||
<code
|
||||
key={p}
|
||||
className="bg-gray-100 px-1.5 py-0.5 rounded text-xs font-mono text-gray-700"
|
||||
>
|
||||
{'{'}{p}{'}'}
|
||||
</code>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<textarea
|
||||
value={value}
|
||||
onChange={(e) => handleChange(step.key, e.target.value)}
|
||||
disabled={isSaving}
|
||||
rows={6}
|
||||
className="w-full min-h-[8rem] rounded border border-gray-300 px-3 py-2 font-mono text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent disabled:bg-gray-100 disabled:cursor-not-allowed resize-y"
|
||||
/>
|
||||
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs text-gray-500">{value.length} characters</span>
|
||||
|
||||
{unknownPlaceholders.length > 0 && (
|
||||
<div className="flex items-center gap-1.5 bg-yellow-50 border border-yellow-200 rounded px-2 py-1">
|
||||
<AlertTriangle className="w-3.5 h-3.5 text-yellow-600" />
|
||||
<span className="text-xs text-yellow-700">
|
||||
Unknown placeholder{unknownPlaceholders.length > 1 ? 's' : ''}:{' '}
|
||||
{unknownPlaceholders.join(', ')}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { PromptEditor } from '../../components/PromptEditor'
|
|||
|
||||
const mockPrompts: Record<string, string> = {
|
||||
decompose: 'Decompose template with {question}',
|
||||
decompose_format: '{"description": "test description", "max_length": 3}',
|
||||
filter_intro: 'Evaluate each chunk for relevance',
|
||||
filter_section: 'Sub-question {subq_idx}: "{subq_question}"\n{chunks}',
|
||||
filter_outro: 'Return JSON object mapping sub-question indices to scores.',
|
||||
|
|
@ -31,15 +32,16 @@ describe('PromptEditor', () => {
|
|||
defaultProps.onCancel.mockClear()
|
||||
})
|
||||
|
||||
it('renders 5 textareas for the active steps', () => {
|
||||
it('renders 6 textareas for the active steps (5 regular + 1 format description)', () => {
|
||||
render(<PromptEditor {...defaultProps} />)
|
||||
const textareas = screen.getAllByRole('textbox')
|
||||
expect(textareas).toHaveLength(5)
|
||||
expect(textareas).toHaveLength(6)
|
||||
})
|
||||
|
||||
it('renders labels for each step', () => {
|
||||
render(<PromptEditor {...defaultProps} />)
|
||||
expect(screen.getByText('Step 1: Query Decomposition')).toBeInTheDocument()
|
||||
expect(screen.getByText('Step 1.1: Query Decomposition')).toBeInTheDocument()
|
||||
expect(screen.getByText('Step 1.2: Query Decomposition Format')).toBeInTheDocument()
|
||||
expect(screen.getByText('Step 2.1: Filter Intro (Preamble)')).toBeInTheDocument()
|
||||
expect(screen.getByText('Step 2.2: Filter Section (Per Sub-Q)')).toBeInTheDocument()
|
||||
expect(screen.getByText('Step 2.3: Filter Outro (Format)')).toBeInTheDocument()
|
||||
|
|
@ -66,18 +68,21 @@ describe('PromptEditor', () => {
|
|||
|
||||
render(<PromptEditor {...defaultProps} />)
|
||||
const resetButtons = screen.getAllByTitle(/reset.*to default/i)
|
||||
expect(resetButtons).toHaveLength(5)
|
||||
expect(resetButtons).toHaveLength(6)
|
||||
|
||||
fireEvent.click(resetButtons[0])
|
||||
expect(defaultProps.onResetStep).toHaveBeenCalledWith('decompose')
|
||||
|
||||
fireEvent.click(resetButtons[1])
|
||||
expect(defaultProps.onResetStep).toHaveBeenCalledWith('decompose_format')
|
||||
|
||||
fireEvent.click(resetButtons[2])
|
||||
expect(defaultProps.onResetStep).toHaveBeenCalledWith('filter_intro')
|
||||
|
||||
fireEvent.click(resetButtons[3])
|
||||
fireEvent.click(resetButtons[4])
|
||||
expect(defaultProps.onResetStep).toHaveBeenCalledWith('filter_outro')
|
||||
|
||||
fireEvent.click(resetButtons[4])
|
||||
fireEvent.click(resetButtons[5])
|
||||
expect(defaultProps.onResetStep).toHaveBeenCalledWith('generate_per_subq')
|
||||
|
||||
confirmSpy.mockRestore()
|
||||
|
|
@ -143,10 +148,11 @@ describe('PromptEditor', () => {
|
|||
const textareas = screen.getAllByRole('textbox')
|
||||
|
||||
expect(textareas[0]).toHaveValue('Decompose template with {question}')
|
||||
expect(textareas[1]).toHaveValue('Evaluate each chunk for relevance')
|
||||
expect(textareas[2]).toHaveValue('Sub-question {subq_idx}: "{subq_question}"\n{chunks}')
|
||||
expect(textareas[3]).toHaveValue('Return JSON object mapping sub-question indices to scores.')
|
||||
expect(textareas[4]).toHaveValue('Generate per-subq with {context_sections}')
|
||||
expect(textareas[1]).toHaveValue('test description')
|
||||
expect(textareas[2]).toHaveValue('Evaluate each chunk for relevance')
|
||||
expect(textareas[3]).toHaveValue('Sub-question {subq_idx}: "{subq_question}"\n{chunks}')
|
||||
expect(textareas[4]).toHaveValue('Return JSON object mapping sub-question indices to scores.')
|
||||
expect(textareas[5]).toHaveValue('Generate per-subq with {context_sections}')
|
||||
})
|
||||
|
||||
it('calls onUpdate when typing in a textarea', () => {
|
||||
|
|
|
|||
Loading…
Reference in New Issue