2347 lines
97 KiB
Diff
2347 lines
97 KiB
Diff
--- a/core/app/apps/advanced_chat/app_runner.py
|
|
+++ b/core/app/apps/advanced_chat/app_runner.py
|
|
@@ -1,5 +1,3 @@
|
|
-import threading
|
|
-from core.workflow.graph_engine.local_teardown import close_owned_producer
|
|
import logging
|
|
import time
|
|
from collections.abc import Mapping, Sequence
|
|
@@ -10,7 +8,6 @@
|
|
|
|
from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfig
|
|
from core.app.apps.base_app_queue_manager import AppQueueManager
|
|
-from core.app.apps.exc import GenerateTaskStoppedError
|
|
from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner
|
|
from core.app.entities.app_invoke_entities import (
|
|
AdvancedChatAppGenerateEntity,
|
|
@@ -207,23 +204,9 @@
|
|
workflow_entry.graph_engine.layer(layer)
|
|
|
|
generator = workflow_entry.run()
|
|
- producer_owner = threading.current_thread()
|
|
- primary_error = None
|
|
- try:
|
|
- presentation_stopped = False
|
|
- for event in generator:
|
|
- if presentation_stopped:
|
|
- continue
|
|
- try:
|
|
- self._handle_event(workflow_entry, event)
|
|
- except GenerateTaskStoppedError:
|
|
- presentation_stopped = True
|
|
- workflow_entry.graph_engine.request_abort("Application presentation stopped")
|
|
- except BaseException as error:
|
|
- primary_error = error
|
|
- raise
|
|
- finally:
|
|
- close_owned_producer(generator, workflow_entry.graph_engine, producer_owner, primary_error)
|
|
+
|
|
+ for event in generator:
|
|
+ self._handle_event(workflow_entry, event)
|
|
|
|
def handle_input_moderation(
|
|
self,
|
|
--- a/core/app/apps/workflow/app_runner.py
|
|
+++ b/core/app/apps/workflow/app_runner.py
|
|
@@ -1,12 +1,9 @@
|
|
-import threading
|
|
-from core.workflow.graph_engine.local_teardown import close_owned_producer
|
|
import logging
|
|
import time
|
|
from collections.abc import Sequence
|
|
from typing import cast
|
|
|
|
from core.app.apps.base_app_queue_manager import AppQueueManager
|
|
-from core.app.apps.exc import GenerateTaskStoppedError
|
|
from core.app.apps.workflow.app_config_manager import WorkflowAppConfig
|
|
from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner
|
|
from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity
|
|
@@ -153,20 +150,6 @@
|
|
workflow_entry.graph_engine.layer(layer)
|
|
|
|
generator = workflow_entry.run()
|
|
- producer_owner = threading.current_thread()
|
|
- primary_error = None
|
|
- try:
|
|
- presentation_stopped = False
|
|
- for event in generator:
|
|
- if presentation_stopped:
|
|
- continue
|
|
- try:
|
|
- self._handle_event(workflow_entry, event)
|
|
- except GenerateTaskStoppedError:
|
|
- presentation_stopped = True
|
|
- workflow_entry.graph_engine.request_abort("Application presentation stopped")
|
|
- except BaseException as error:
|
|
- primary_error = error
|
|
- raise
|
|
- finally:
|
|
- close_owned_producer(generator, workflow_entry.graph_engine, producer_owner, primary_error)
|
|
+
|
|
+ for event in generator:
|
|
+ self._handle_event(workflow_entry, event)
|
|
--- a/core/model_runtime/model_providers/__base/large_language_model.py
|
|
+++ b/core/model_runtime/model_providers/__base/large_language_model.py
|
|
@@ -1,11 +1,3 @@
|
|
-
|
|
-from core.workflow.iterator_cancellation import (
|
|
- remember_primary_exception,
|
|
- IteratorCancellationRequested,
|
|
- cancellation_checkpoint,
|
|
- owned_iterator,
|
|
- owned_result,
|
|
-)
|
|
import logging
|
|
import time
|
|
import uuid
|
|
@@ -122,7 +114,6 @@
|
|
:param callbacks: callbacks
|
|
:return: full response or stream response chunk generator result
|
|
"""
|
|
- cancellation_checkpoint()
|
|
# validate and filter model parameters
|
|
if model_parameters is None:
|
|
model_parameters = {}
|
|
@@ -153,7 +144,6 @@
|
|
from core.plugin.impl.model import PluginModelClient
|
|
|
|
plugin_model_manager = PluginModelClient()
|
|
- cancellation_checkpoint()
|
|
result = plugin_model_manager.invoke_llm(
|
|
tenant_id=self.tenant_id,
|
|
user_id=user or "unknown",
|
|
@@ -175,20 +165,17 @@
|
|
system_fingerprint = None
|
|
tools_calls: list[AssistantPromptMessage.ToolCall] = []
|
|
|
|
- with owned_iterator(
|
|
- result
|
|
- ) as owned_llm_chunks:
|
|
- for chunk in owned_llm_chunks:
|
|
- if isinstance(chunk.delta.message.content, str):
|
|
- content += chunk.delta.message.content
|
|
- elif isinstance(chunk.delta.message.content, list):
|
|
- content_list.extend(chunk.delta.message.content)
|
|
- if chunk.delta.message.tool_calls:
|
|
- _increase_tool_call(chunk.delta.message.tool_calls, tools_calls)
|
|
-
|
|
- usage = chunk.delta.usage or LLMUsage.empty_usage()
|
|
- system_fingerprint = chunk.system_fingerprint
|
|
- break
|
|
+ for chunk in result:
|
|
+ if isinstance(chunk.delta.message.content, str):
|
|
+ content += chunk.delta.message.content
|
|
+ elif isinstance(chunk.delta.message.content, list):
|
|
+ content_list.extend(chunk.delta.message.content)
|
|
+ if chunk.delta.message.tool_calls:
|
|
+ _increase_tool_call(chunk.delta.message.tool_calls, tools_calls)
|
|
+
|
|
+ usage = chunk.delta.usage or LLMUsage.empty_usage()
|
|
+ system_fingerprint = chunk.system_fingerprint
|
|
+ break
|
|
|
|
result = LLMResult(
|
|
model=model,
|
|
@@ -200,10 +187,7 @@
|
|
usage=usage,
|
|
system_fingerprint=system_fingerprint,
|
|
)
|
|
- except IteratorCancellationRequested:
|
|
- raise
|
|
except Exception as e:
|
|
- remember_primary_exception(e)
|
|
self._trigger_invoke_error_callbacks(
|
|
model=model,
|
|
ex=e,
|
|
@@ -221,9 +205,9 @@
|
|
raise self._transform_invoke_error(e)
|
|
|
|
if stream and isinstance(result, Generator):
|
|
- return owned_result(result, lambda owned_plugin_result: self._invoke_result_generator(
|
|
+ return self._invoke_result_generator(
|
|
model=model,
|
|
- result=owned_plugin_result,
|
|
+ result=result,
|
|
credentials=credentials,
|
|
prompt_messages=prompt_messages,
|
|
model_parameters=model_parameters,
|
|
@@ -232,7 +216,7 @@
|
|
stream=stream,
|
|
user=user,
|
|
callbacks=callbacks,
|
|
- ))
|
|
+ )
|
|
elif isinstance(result, LLMResult):
|
|
self._trigger_after_invoke_callbacks(
|
|
model=model,
|
|
@@ -272,7 +256,6 @@
|
|
:param result: result generator
|
|
:return: result generator
|
|
"""
|
|
- cancellation_checkpoint()
|
|
callbacks = callbacks or []
|
|
message_content: list[PromptMessageContentUnionTypes] = []
|
|
usage = None
|
|
@@ -290,41 +273,35 @@
|
|
return
|
|
|
|
try:
|
|
- with owned_iterator(
|
|
- result
|
|
- ) as owned_llm_chunks:
|
|
- for chunk in owned_llm_chunks:
|
|
- # Following https://github.com/langgenius/dify/issues/17799,
|
|
- # we removed the prompt_messages from the chunk on the plugin daemon side.
|
|
- # To ensure compatibility, we add the prompt_messages back here.
|
|
- chunk.prompt_messages = prompt_messages
|
|
- yield chunk
|
|
-
|
|
- self._trigger_new_chunk_callbacks(
|
|
- chunk=chunk,
|
|
- model=model,
|
|
- credentials=credentials,
|
|
- prompt_messages=prompt_messages,
|
|
- model_parameters=model_parameters,
|
|
- tools=tools,
|
|
- stop=stop,
|
|
- stream=stream,
|
|
- user=user,
|
|
- callbacks=callbacks,
|
|
- )
|
|
-
|
|
- _update_message_content(chunk.delta.message.content)
|
|
-
|
|
- real_model = chunk.model
|
|
- if chunk.delta.usage:
|
|
- usage = chunk.delta.usage
|
|
-
|
|
- if chunk.system_fingerprint:
|
|
- system_fingerprint = chunk.system_fingerprint
|
|
- except IteratorCancellationRequested:
|
|
- raise
|
|
+ for chunk in result:
|
|
+ # Following https://github.com/langgenius/dify/issues/17799,
|
|
+ # we removed the prompt_messages from the chunk on the plugin daemon side.
|
|
+ # To ensure compatibility, we add the prompt_messages back here.
|
|
+ chunk.prompt_messages = prompt_messages
|
|
+ yield chunk
|
|
+
|
|
+ self._trigger_new_chunk_callbacks(
|
|
+ chunk=chunk,
|
|
+ model=model,
|
|
+ credentials=credentials,
|
|
+ prompt_messages=prompt_messages,
|
|
+ model_parameters=model_parameters,
|
|
+ tools=tools,
|
|
+ stop=stop,
|
|
+ stream=stream,
|
|
+ user=user,
|
|
+ callbacks=callbacks,
|
|
+ )
|
|
+
|
|
+ _update_message_content(chunk.delta.message.content)
|
|
+
|
|
+ real_model = chunk.model
|
|
+ if chunk.delta.usage:
|
|
+ usage = chunk.delta.usage
|
|
+
|
|
+ if chunk.system_fingerprint:
|
|
+ system_fingerprint = chunk.system_fingerprint
|
|
except Exception as e:
|
|
- remember_primary_exception(e)
|
|
raise self._transform_invoke_error(e)
|
|
|
|
assistant_message = AssistantPromptMessage(content=message_content)
|
|
--- a/core/plugin/impl/base.py
|
|
+++ b/core/plugin/impl/base.py
|
|
@@ -1,9 +1,3 @@
|
|
-
|
|
-from core.workflow.iterator_cancellation import (
|
|
- remember_primary_exception,
|
|
- cancellation_checkpoint,
|
|
- owned_iterator,
|
|
-)
|
|
import inspect
|
|
import json
|
|
import logging
|
|
@@ -73,7 +67,6 @@
|
|
"""
|
|
Make a request to the plugin daemon inner API.
|
|
"""
|
|
- cancellation_checkpoint()
|
|
url, headers, prepared_data, params, files = self._prepare_request(path, headers, data, params, files)
|
|
|
|
request_kwargs: dict[str, Any] = {
|
|
@@ -90,10 +83,8 @@
|
|
request_kwargs["content"] = prepared_data
|
|
|
|
try:
|
|
- cancellation_checkpoint()
|
|
response = httpx.request(**request_kwargs)
|
|
- except httpx.RequestError as error:
|
|
- remember_primary_exception(error)
|
|
+ except httpx.RequestError:
|
|
logger.exception("Request to Plugin Daemon Service failed")
|
|
raise PluginDaemonInnerError(code=-500, message="Request to Plugin Daemon Service failed")
|
|
|
|
@@ -135,7 +126,6 @@
|
|
"""
|
|
Make a stream request to the plugin daemon inner API
|
|
"""
|
|
- cancellation_checkpoint()
|
|
url, headers, prepared_data, params, files = self._prepare_request(path, headers, data, params, files)
|
|
|
|
stream_kwargs: dict[str, Any] = {
|
|
@@ -152,22 +142,17 @@
|
|
stream_kwargs["content"] = prepared_data
|
|
|
|
try:
|
|
- cancellation_checkpoint()
|
|
with httpx.stream(**stream_kwargs) as response:
|
|
- with owned_iterator(
|
|
- response.iter_lines()
|
|
- ) as owned_response_lines:
|
|
- for raw_line in owned_response_lines:
|
|
- if not raw_line:
|
|
- continue
|
|
- line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else raw_line
|
|
- line = line.strip()
|
|
- if line.startswith("data:"):
|
|
- line = line[5:].strip()
|
|
- if line:
|
|
- yield line
|
|
- except httpx.RequestError as error:
|
|
- remember_primary_exception(error)
|
|
+ for raw_line in response.iter_lines():
|
|
+ if not raw_line:
|
|
+ continue
|
|
+ line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else raw_line
|
|
+ line = line.strip()
|
|
+ if line.startswith("data:"):
|
|
+ line = line[5:].strip()
|
|
+ if line:
|
|
+ yield line
|
|
+ except httpx.RequestError:
|
|
logger.exception("Stream request to Plugin Daemon Service failed")
|
|
raise PluginDaemonInnerError(code=-500, message="Request to Plugin Daemon Service failed")
|
|
|
|
@@ -184,12 +169,8 @@
|
|
"""
|
|
Make a stream request to the plugin daemon inner API and yield the response as a model.
|
|
"""
|
|
- cancellation_checkpoint()
|
|
- with owned_iterator(
|
|
- self._stream_request(method, path, params, headers, data, files)
|
|
- ) as owned_plugin_lines:
|
|
- for line in owned_plugin_lines:
|
|
- yield type_(**json.loads(line)) # type: ignore
|
|
+ for line in self._stream_request(method, path, params, headers, data, files):
|
|
+ yield type_(**json.loads(line)) # type: ignore
|
|
|
|
def _request_with_model(
|
|
self,
|
|
@@ -272,38 +253,34 @@
|
|
"""
|
|
Make a stream request to the plugin daemon inner API and yield the response as a model.
|
|
"""
|
|
- cancellation_checkpoint()
|
|
- with owned_iterator(
|
|
- self._stream_request(method, path, params, headers, data, files)
|
|
- ) as owned_plugin_lines:
|
|
- for line in owned_plugin_lines:
|
|
+ for line in self._stream_request(method, path, params, headers, data, files):
|
|
+ try:
|
|
+ rep = PluginDaemonBasicResponse[type_].model_validate_json(line) # type: ignore
|
|
+ except (ValueError, TypeError):
|
|
+ # TODO modify this when line_data has code and message
|
|
try:
|
|
- rep = PluginDaemonBasicResponse[type_].model_validate_json(line) # type: ignore
|
|
+ line_data = json.loads(line)
|
|
except (ValueError, TypeError):
|
|
- # TODO modify this when line_data has code and message
|
|
+ raise ValueError(line)
|
|
+ # If the dictionary contains the `error` key, use its value as the argument
|
|
+ # for `ValueError`.
|
|
+ # Otherwise, use the `line` to provide better contextual information about the error.
|
|
+ raise ValueError(line_data.get("error", line))
|
|
+
|
|
+ if rep.code != 0:
|
|
+ if rep.code == -500:
|
|
try:
|
|
- line_data = json.loads(line)
|
|
- except (ValueError, TypeError):
|
|
- raise ValueError(line)
|
|
- # If the dictionary contains the `error` key, use its value as the argument
|
|
- # for `ValueError`.
|
|
- # Otherwise, use the `line` to provide better contextual information about the error.
|
|
- raise ValueError(line_data.get("error", line))
|
|
-
|
|
- if rep.code != 0:
|
|
- if rep.code == -500:
|
|
- try:
|
|
- error = PluginDaemonError.model_validate(json.loads(rep.message))
|
|
- except Exception:
|
|
- raise PluginDaemonInnerError(code=rep.code, message=rep.message)
|
|
-
|
|
- logger.error("Error in stream response for plugin %s", rep.__dict__)
|
|
- self._handle_plugin_daemon_error(error.error_type, error.message)
|
|
- raise ValueError(f"plugin daemon: {rep.message}, code: {rep.code}")
|
|
- if rep.data is None:
|
|
- frame = inspect.currentframe()
|
|
- raise ValueError(f"got empty data from plugin daemon: {frame.f_lineno if frame else 'unknown'}")
|
|
- yield rep.data
|
|
+ error = PluginDaemonError.model_validate(json.loads(rep.message))
|
|
+ except Exception:
|
|
+ raise PluginDaemonInnerError(code=rep.code, message=rep.message)
|
|
+
|
|
+ logger.error("Error in stream response for plugin %s", rep.__dict__)
|
|
+ self._handle_plugin_daemon_error(error.error_type, error.message)
|
|
+ raise ValueError(f"plugin daemon: {rep.message}, code: {rep.code}")
|
|
+ if rep.data is None:
|
|
+ frame = inspect.currentframe()
|
|
+ raise ValueError(f"got empty data from plugin daemon: {frame.f_lineno if frame else 'unknown'}")
|
|
+ yield rep.data
|
|
|
|
def _handle_plugin_daemon_error(self, error_type: str, message: str):
|
|
"""
|
|
--- a/core/workflow/graph_engine/command_processing/command_handlers.py
|
|
+++ b/core/workflow/graph_engine/command_processing/command_handlers.py
|
|
@@ -17,11 +17,8 @@
|
|
@override
|
|
def handle(self, command: GraphEngineCommand, execution: GraphExecution) -> None:
|
|
assert isinstance(command, AbortCommand)
|
|
+ logger.debug("Aborting workflow %s: %s", execution.workflow_id, command.reason)
|
|
execution.abort(command.reason or "User requested abort")
|
|
- try:
|
|
- logger.debug("Aborting workflow %s: %s", execution.workflow_id, command.reason)
|
|
- except BaseException:
|
|
- pass
|
|
|
|
|
|
@final
|
|
--- a/core/workflow/graph_engine/domain/graph_execution.py
|
|
+++ b/core/workflow/graph_engine/domain/graph_execution.py
|
|
@@ -3,8 +3,6 @@
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
-
|
|
-from ..local_teardown import GraphAbortError
|
|
from importlib import import_module
|
|
from typing import Literal
|
|
|
|
@@ -129,10 +127,9 @@
|
|
self.completed = True
|
|
|
|
def abort(self, reason: str) -> None:
|
|
- """Latch typed abort without overwriting an earlier ordinary failure."""
|
|
+ """Abort the graph execution."""
|
|
self.aborted = True
|
|
- if self.error is None:
|
|
- self.error = GraphAbortError(f"Aborted: {reason}")
|
|
+ self.error = RuntimeError(f"Aborted: {reason}")
|
|
|
|
def pause(self, reason: PauseReason) -> None:
|
|
"""Pause the graph execution without marking it complete."""
|
|
--- a/core/workflow/graph_engine/event_management/event_manager.py
|
|
+++ b/core/workflow/graph_engine/event_management/event_manager.py
|
|
@@ -3,7 +3,6 @@
|
|
"""
|
|
|
|
import logging
|
|
-import inspect
|
|
import threading
|
|
import time
|
|
from collections.abc import Generator
|
|
@@ -13,11 +12,8 @@
|
|
from core.workflow.graph_events import GraphEngineEvent
|
|
|
|
from ..layers.base import GraphEngineLayer
|
|
-from ..layers.persistence_failure import CriticalWorkflowPersistenceError
|
|
-from ..local_teardown import LifecycleCallbackFailure
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
-_MARKER_MISSING = object()
|
|
|
|
|
|
@final
|
|
@@ -94,10 +90,6 @@
|
|
self._lock = ReadWriteLock()
|
|
self._layers: list[GraphEngineLayer] = []
|
|
self._execution_complete = threading.Event()
|
|
- self._critical_failure: CriticalWorkflowPersistenceError | None = None
|
|
- self._critical_failure_lock = threading.Lock()
|
|
- self._critical_failure_phase: str | None = None
|
|
- self._lifecycle_failures: list[LifecycleCallbackFailure] = []
|
|
|
|
def set_layers(self, layers: list[GraphEngineLayer]) -> None:
|
|
"""
|
|
@@ -119,12 +111,15 @@
|
|
Args:
|
|
event: The event to collect
|
|
"""
|
|
- # A critical persistence failure must be visible before buffering this
|
|
- # event. Layer I/O remains outside both event and failure locks.
|
|
+ with self._lock.write_lock():
|
|
+ self._events.append(event)
|
|
+
|
|
+ # NOTE: `_notify_layers` is intentionally called outside the critical section
|
|
+ # to minimize lock contention and avoid blocking other readers or writers.
|
|
+ #
|
|
+ # The public `notify_layers` method also does not use a write lock,
|
|
+ # so protecting `_notify_layers` with a lock here is unnecessary.
|
|
self._notify_layers(event)
|
|
- with self._lock.write_lock():
|
|
- self._raise_if_critical_failure()
|
|
- self._events.append(event)
|
|
|
|
def _get_new_events(self, start_index: int) -> list[GraphEngineEvent]:
|
|
"""
|
|
@@ -163,13 +158,11 @@
|
|
yielded_count = 0
|
|
|
|
while not self._execution_complete.is_set() or yielded_count < self._event_count():
|
|
- self._raise_if_critical_failure()
|
|
# Get new events since last yield
|
|
new_events = self._get_new_events(yielded_count)
|
|
|
|
# Yield any new events
|
|
for event in new_events:
|
|
- self._raise_if_critical_failure()
|
|
yield event
|
|
yielded_count += 1
|
|
|
|
@@ -177,137 +170,17 @@
|
|
if not self._execution_complete.is_set() and not new_events:
|
|
time.sleep(0.001)
|
|
|
|
- self._raise_if_critical_failure()
|
|
+ def _notify_layers(self, event: GraphEngineEvent) -> None:
|
|
+ """
|
|
+ Notify all layers of an event.
|
|
|
|
- @property
|
|
- def critical_failure(self) -> CriticalWorkflowPersistenceError | None:
|
|
- """First latched failure; safe type/message, never a terminal receipt."""
|
|
- with self._critical_failure_lock:
|
|
- return self._critical_failure
|
|
+ Layer exceptions are caught and logged to prevent disrupting collection.
|
|
|
|
- def _raise_if_critical_failure(self) -> None:
|
|
- failure = self.critical_failure
|
|
- if failure is not None:
|
|
- raise failure from None
|
|
-
|
|
- @staticmethod
|
|
- def _read_layer_marker(layer: GraphEngineLayer, marker: str) -> bool:
|
|
- """Read once; declared descriptor failure is not dynamic absence.
|
|
-
|
|
- Declared descriptors use standard object binding without a fallback
|
|
- __getattr__ masking their own AttributeError. Other values retain
|
|
- ordinary dynamic lookup. Structural inspection never calls a getter.
|
|
+ Args:
|
|
+ event: The event to send to layers
|
|
"""
|
|
- declared = inspect.getattr_static(layer, marker, _MARKER_MISSING)
|
|
- if declared is _MARKER_MISSING:
|
|
+ for layer in self._layers:
|
|
try:
|
|
- return getattr(layer, marker) is True
|
|
- except AttributeError:
|
|
- return False
|
|
- class_declared = inspect.getattr_static(type(layer), marker, _MARKER_MISSING)
|
|
- if (declared is class_declared
|
|
- and inspect.getattr_static(type(class_declared), "__get__", _MARKER_MISSING) is not _MARKER_MISSING):
|
|
- return object.__getattribute__(layer, marker) is True
|
|
- return getattr(layer, marker) is True
|
|
-
|
|
- @property
|
|
- def critical_failure_phase(self) -> str | None:
|
|
- with self._critical_failure_lock:
|
|
- return self._critical_failure_phase
|
|
-
|
|
- @property
|
|
- def lifecycle_failures(self) -> tuple[LifecycleCallbackFailure, ...]:
|
|
- """Private originals and finite phase origin; never a safe export."""
|
|
- with self._critical_failure_lock:
|
|
- return tuple(self._lifecycle_failures)
|
|
-
|
|
- @staticmethod
|
|
- def _log_lifecycle_failure(*, critical: bool) -> None:
|
|
- try:
|
|
- if critical:
|
|
- _logger.error("CRITICAL_WORKFLOW_LIFECYCLE_FAILURE")
|
|
- else:
|
|
- _logger.warning("OPTIONAL_WORKFLOW_LIFECYCLE_FAILURE")
|
|
- except BaseException:
|
|
- pass
|
|
-
|
|
- def _retain_lifecycle_failure(self, phase: str, critical: bool, original: BaseException):
|
|
- item = LifecycleCallbackFailure(phase, critical, original)
|
|
- # These fixed, non-layer objects are prepared outside the lock too.
|
|
- proposed = CriticalWorkflowPersistenceError() if critical else None
|
|
- if proposed is not None:
|
|
- proposed._original_error = original
|
|
- with self._critical_failure_lock:
|
|
- self._lifecycle_failures.append(item)
|
|
- created = critical and self._critical_failure is None
|
|
- if created:
|
|
- self._critical_failure = proposed
|
|
- self._critical_failure_phase = phase
|
|
- retained = self._critical_failure if critical else item
|
|
- if critical:
|
|
- assert retained is not None
|
|
- if created:
|
|
- if phase == "on_event":
|
|
- self._log_layer_failure(critical=True)
|
|
- else:
|
|
- self._log_lifecycle_failure(critical=True)
|
|
- return retained
|
|
- if phase == "on_event":
|
|
- self._log_layer_failure(critical=False)
|
|
- else:
|
|
- self._log_lifecycle_failure(critical=False)
|
|
- return item
|
|
-
|
|
- def invoke_lifecycle(self, layer: GraphEngineLayer, phase: str, *args):
|
|
- """Retain the public three-phase contract; share protected dispatch."""
|
|
- arities = {"initialize": 2, "on_graph_start": 0, "on_graph_end": 1}
|
|
- if type(phase) is not str or phase not in arities or len(args) != arities[phase]:
|
|
- raise ValueError("Invalid workflow lifecycle invocation")
|
|
- return self._invoke_layer(layer, phase, *args)
|
|
-
|
|
- def _invoke_layer(self, layer: GraphEngineLayer, phase: str, *args):
|
|
- """Classify once before callback; no marker re-read can replace it."""
|
|
- arities = {"initialize": 2, "on_graph_start": 0, "on_graph_end": 1, "on_event": 1}
|
|
- if type(phase) is not str or phase not in arities or len(args) != arities[phase]:
|
|
- raise ValueError("Invalid workflow layer invocation")
|
|
- # Required end callbacks still run after an earlier latched failure.
|
|
- if phase != "on_graph_end":
|
|
- self._raise_if_critical_failure()
|
|
- marker = "critical_cleanup" if phase == "on_graph_end" else "critical_persistence"
|
|
- try:
|
|
- critical = self._read_layer_marker(layer, marker)
|
|
- except BaseException as original:
|
|
- failure = self._retain_lifecycle_failure(phase, True, original)
|
|
- if not isinstance(original, Exception):
|
|
- raise
|
|
- raise failure from None
|
|
- try:
|
|
- getattr(layer, phase)(*args)
|
|
- except BaseException as original:
|
|
- retained = self._retain_lifecycle_failure(phase, critical, original)
|
|
- if not isinstance(original, Exception):
|
|
- # Control interruptions are not optional observer failures.
|
|
- raise
|
|
- if critical:
|
|
- raise retained from None
|
|
- return retained
|
|
- return None
|
|
-
|
|
- @staticmethod
|
|
- def _log_layer_failure(*, critical: bool) -> None:
|
|
- # No exception rendering, layer repr, SQL, IDs or provider data. A log
|
|
- # sink failure must not replace the latched error or break isolation.
|
|
- try:
|
|
- if critical:
|
|
- _logger.error("CRITICAL_WORKFLOW_PERSISTENCE_FAILURE")
|
|
- else:
|
|
- _logger.warning("OPTIONAL_GRAPH_LAYER_FAILURE")
|
|
- except BaseException:
|
|
- pass
|
|
-
|
|
- def _notify_layers(self, event: GraphEngineEvent) -> None:
|
|
- """Use the same finite, pre-classified protocol as lifecycle hooks."""
|
|
- self._raise_if_critical_failure()
|
|
- for layer in self._layers:
|
|
- self._raise_if_critical_failure()
|
|
- self._invoke_layer(layer, "on_event", event)
|
|
+ layer.on_event(event)
|
|
+ except Exception:
|
|
+ _logger.exception("Error in layer on_event, layer_type=%s", type(layer))
|
|
--- a/core/workflow/graph_engine/graph_engine.py
|
|
+++ b/core/workflow/graph_engine/graph_engine.py
|
|
@@ -8,8 +8,6 @@
|
|
import contextvars
|
|
import logging
|
|
import queue
|
|
-import threading
|
|
-from dataclasses import replace
|
|
from collections.abc import Generator
|
|
from typing import TYPE_CHECKING, cast, final
|
|
|
|
@@ -43,7 +41,6 @@
|
|
from .protocols.command_channel import CommandChannel
|
|
from .ready_queue import ReadyQueue
|
|
from .worker_management import WorkerPool
|
|
-from .local_teardown import (GraphAbortError, LocalTeardownUnconfirmed, ProducerTeardownResult, WorkerLocalResult, OrdinaryPrimaryUnresolved, safe_lifecycle_log)
|
|
|
|
if TYPE_CHECKING:
|
|
from core.workflow.graph_engine.domain.graph_execution import GraphExecution
|
|
@@ -79,17 +76,6 @@
|
|
self._graph_runtime_state = graph_runtime_state
|
|
self._graph_runtime_state.configure(graph=cast("GraphProtocol", graph))
|
|
self._command_channel = command_channel
|
|
- self._abort_request_lock = threading.Lock()
|
|
- self._abort_request_reason: str | None = None
|
|
- self._abort_request_errors: list[Exception] = []
|
|
- self._terminal_selected = False
|
|
- self._run_entered = False
|
|
- self._teardown_result: dict | None = None
|
|
- self._end_layers_notified = False
|
|
- self._producer_owner: threading.Thread | None = None
|
|
- self._primary_error: BaseException | None = None
|
|
- self._local_teardown_result: ProducerTeardownResult | None = None
|
|
- self._local_teardown_history: tuple[ProducerTeardownResult, ...] = ()
|
|
|
|
# Graph execution tracks the overall execution state
|
|
self._graph_execution = cast("GraphExecution", self._graph_runtime_state.graph_execution)
|
|
@@ -231,155 +217,102 @@
|
|
self._layers.append(layer)
|
|
return self
|
|
|
|
- def request_abort(self, reason: str) -> bool:
|
|
- """Latch same-instance cooperative intent; never certify remote cleanup."""
|
|
- if not isinstance(reason, str) or not reason or len(reason) > 512:
|
|
- raise ValueError("Abort reason must be a nonempty bounded string")
|
|
- with self._abort_request_lock:
|
|
- if self._abort_request_reason is not None:
|
|
- return True
|
|
- execution = self._graph_execution
|
|
- if (not execution.started or self._terminal_selected or execution.is_paused
|
|
- or execution.completed or execution.has_error):
|
|
- return False
|
|
- self._abort_request_reason = reason
|
|
+ def run(self) -> Generator[GraphEngineEvent, None, None]:
|
|
+ """
|
|
+ Execute the graph using the modular architecture.
|
|
+
|
|
+ Returns:
|
|
+ Generator yielding GraphEngineEvent instances
|
|
+ """
|
|
try:
|
|
- self._worker_pool.request_stop()
|
|
- except Exception as error:
|
|
- with self._abort_request_lock:
|
|
- self._abort_request_errors.append(error)
|
|
- return True
|
|
-
|
|
- def run(self) -> Generator[GraphEngineEvent, None, None]:
|
|
- """Select no persistable terminal before owner-local teardown confirms."""
|
|
- owner = threading.current_thread()
|
|
- with self._abort_request_lock:
|
|
- if self._run_entered:
|
|
- prior = self._local_teardown_result
|
|
- if (not self._graph_execution.is_paused or prior is None or not prior.confirmed_local
|
|
- or prior.ordinary_primary_unresolved or prior.graph_primary_error is not None
|
|
- or self._event_manager.critical_failure is not None):
|
|
- raise RuntimeError("Prior engine invocation is active or teardown is unconfirmed")
|
|
- self._local_teardown_history = getattr(self, "_local_teardown_history", ()) + (prior,)
|
|
- self._abort_request_reason = None
|
|
- self._abort_request_errors = []
|
|
- self._terminal_selected = False
|
|
- self._teardown_result = None
|
|
- self._local_teardown_result = None
|
|
- self._end_layers_notified = False
|
|
- self._primary_error = None
|
|
- # A paused invocation is closed and fully joined. New emission
|
|
- # state avoids replaying the prior invocation's buffered events.
|
|
- self._event_manager = EventManager()
|
|
- self._dispatcher._event_emitter = self._event_manager
|
|
- self._event_handler_registry._event_collector = self._event_manager
|
|
- self._run_entered = True
|
|
- self._producer_owner = owner
|
|
- try:
|
|
+ # Initialize layers
|
|
self._initialize_layers()
|
|
+
|
|
is_resume = self._graph_execution.started
|
|
if not is_resume:
|
|
self._graph_execution.start()
|
|
else:
|
|
self._graph_execution.paused = False
|
|
self._graph_execution.pause_reasons = []
|
|
+
|
|
start_event = GraphRunStartedEvent()
|
|
self._event_manager.notify_layers(start_event)
|
|
yield start_event
|
|
- if self._abort_request_reason is None and not self._graph_execution.aborted:
|
|
- self._start_execution(resume=is_resume)
|
|
- if self._abort_request_reason is None:
|
|
- events = self._event_manager.emit_events()
|
|
- try:
|
|
- for event in events:
|
|
- yield event
|
|
- if self._abort_request_reason is not None:
|
|
- break
|
|
- finally:
|
|
- events.close()
|
|
- # Original command aborts and local request_abort share this gate.
|
|
- if self._abort_request_reason is not None:
|
|
- self._graph_execution.abort(self._abort_request_reason)
|
|
- execution = self._graph_execution
|
|
- if execution.error is not None and not isinstance(execution.error, GraphAbortError):
|
|
- raise execution.error
|
|
- teardown = self._stop_execution()
|
|
- # Dispatcher/worker errors can arrive during the finite joins.
|
|
- # All producers are now stopped or independently unconfirmed.
|
|
- if execution.error is not None and not isinstance(execution.error, GraphAbortError):
|
|
- raise execution.error
|
|
- critical = self._event_manager.critical_failure
|
|
- if critical is not None:
|
|
- raise critical
|
|
- if not self._local_teardown_result.confirmed_local:
|
|
- raise LocalTeardownUnconfirmed()
|
|
- if self._local_teardown_result.ordinary_primary_unresolved:
|
|
- raise OrdinaryPrimaryUnresolved()
|
|
- if execution.is_paused:
|
|
- assert execution.pause_reasons, "pause_reasons should not be empty when execution is paused."
|
|
- terminal = GraphRunPausedEvent(reasons=execution.pause_reasons, outputs=self._graph_runtime_state.outputs)
|
|
- elif execution.aborted:
|
|
- terminal = GraphRunAbortedEvent(reason=str(execution.error) if execution.error else "Workflow execution aborted by user command",
|
|
- outputs=self._graph_runtime_state.outputs)
|
|
- elif execution.exceptions_count > 0:
|
|
- terminal = GraphRunPartialSucceededEvent(exceptions_count=execution.exceptions_count,
|
|
- outputs=self._graph_runtime_state.outputs)
|
|
+
|
|
+ # Start subsystems
|
|
+ self._start_execution(resume=is_resume)
|
|
+
|
|
+ # Yield events as they occur
|
|
+ yield from self._event_manager.emit_events()
|
|
+
|
|
+ # Handle completion
|
|
+ if self._graph_execution.is_paused:
|
|
+ pause_reasons = self._graph_execution.pause_reasons
|
|
+ assert pause_reasons, "pause_reasons should not be empty when execution is paused."
|
|
+ # Ensure we have a valid PauseReason for the event
|
|
+ paused_event = GraphRunPausedEvent(
|
|
+ reasons=pause_reasons,
|
|
+ outputs=self._graph_runtime_state.outputs,
|
|
+ )
|
|
+ self._event_manager.notify_layers(paused_event)
|
|
+ yield paused_event
|
|
+ elif self._graph_execution.aborted:
|
|
+ abort_reason = "Workflow execution aborted by user command"
|
|
+ if self._graph_execution.error:
|
|
+ abort_reason = str(self._graph_execution.error)
|
|
+ aborted_event = GraphRunAbortedEvent(
|
|
+ reason=abort_reason,
|
|
+ outputs=self._graph_runtime_state.outputs,
|
|
+ )
|
|
+ self._event_manager.notify_layers(aborted_event)
|
|
+ yield aborted_event
|
|
+ elif self._graph_execution.has_error:
|
|
+ if self._graph_execution.error:
|
|
+ raise self._graph_execution.error
|
|
else:
|
|
- terminal = GraphRunSucceededEvent(outputs=self._graph_runtime_state.outputs)
|
|
- self._terminal_selected = True
|
|
- self._event_manager.notify_layers(terminal)
|
|
- yield terminal
|
|
- except BaseException as error:
|
|
- self._remember_primary(error)
|
|
- self._stop_execution()
|
|
- # A generator close, critical persistence failure, or uncertain
|
|
- # local cleanup must never create a substitute FAILED DB receipt.
|
|
- if (not isinstance(error, Exception) or isinstance(error, (LocalTeardownUnconfirmed, OrdinaryPrimaryUnresolved))
|
|
- or not self._local_teardown_result.confirmed_local
|
|
- or self._local_teardown_result.ordinary_primary_unresolved
|
|
- or self._event_manager.critical_failure is not None):
|
|
- raise
|
|
- failed_event = GraphRunFailedEvent(error=str(error), exceptions_count=self._graph_execution.exceptions_count)
|
|
- try:
|
|
- self._terminal_selected = True
|
|
- self._event_manager.notify_layers(failed_event)
|
|
- except BaseException as secondary:
|
|
- self._record_cleanup_error(secondary)
|
|
- raise error
|
|
+ outputs = self._graph_runtime_state.outputs
|
|
+ exceptions_count = self._graph_execution.exceptions_count
|
|
+ if exceptions_count > 0:
|
|
+ partial_event = GraphRunPartialSucceededEvent(
|
|
+ exceptions_count=exceptions_count,
|
|
+ outputs=outputs,
|
|
+ )
|
|
+ self._event_manager.notify_layers(partial_event)
|
|
+ yield partial_event
|
|
+ else:
|
|
+ succeeded_event = GraphRunSucceededEvent(
|
|
+ outputs=outputs,
|
|
+ )
|
|
+ self._event_manager.notify_layers(succeeded_event)
|
|
+ yield succeeded_event
|
|
+
|
|
+ except Exception as e:
|
|
+ failed_event = GraphRunFailedEvent(
|
|
+ error=str(e),
|
|
+ exceptions_count=self._graph_execution.exceptions_count,
|
|
+ )
|
|
+ self._event_manager.notify_layers(failed_event)
|
|
yield failed_event
|
|
raise
|
|
+
|
|
finally:
|
|
self._stop_execution()
|
|
|
|
- def _remember_primary(self, error: BaseException) -> None:
|
|
- if self._primary_error is None:
|
|
- self._primary_error = error
|
|
- if self._local_teardown_result is not None and self._local_teardown_result.primary_error is None:
|
|
- self._local_teardown_result = replace(self._local_teardown_result, primary_error=self._primary_error)
|
|
-
|
|
- def _record_cleanup_error(self, error: BaseException) -> None:
|
|
- if self._local_teardown_result is None:
|
|
- self._abort_request_errors.append(error)
|
|
- else:
|
|
- self._local_teardown_result = replace(self._local_teardown_result,
|
|
- secondary_errors=self._local_teardown_result.secondary_errors + (error,))
|
|
- self._teardown_result["errors"].append(error)
|
|
-
|
|
- @property
|
|
- def local_teardown_result(self) -> ProducerTeardownResult | None:
|
|
- """Private same-engine local facts; no external run/proof transport."""
|
|
- return self._local_teardown_result
|
|
-
|
|
- @property
|
|
- def local_teardown_history(self) -> tuple[ProducerTeardownResult, ...]:
|
|
- return getattr(self, "_local_teardown_history", ())
|
|
-
|
|
def _initialize_layers(self) -> None:
|
|
+ """Initialize layers with context."""
|
|
self._event_manager.set_layers(self._layers)
|
|
+ # Create a read-only wrapper for the runtime state
|
|
read_only_state = ReadOnlyGraphRuntimeStateWrapper(self._graph_runtime_state)
|
|
for layer in self._layers:
|
|
- self._event_manager.invoke_lifecycle(layer, "initialize", read_only_state, self._command_channel)
|
|
- self._event_manager.invoke_lifecycle(layer, "on_graph_start")
|
|
+ try:
|
|
+ layer.initialize(read_only_state, self._command_channel)
|
|
+ except Exception as e:
|
|
+ logger.warning("Failed to initialize layer %s: %s", layer.__class__.__name__, e)
|
|
+
|
|
+ try:
|
|
+ layer.on_graph_start()
|
|
+ except Exception as e:
|
|
+ logger.warning("Layer %s failed on_graph_start: %s", layer.__class__.__name__, e)
|
|
|
|
def _start_execution(self, *, resume: bool = False) -> None:
|
|
"""Start execution subsystems."""
|
|
@@ -389,9 +322,6 @@
|
|
|
|
# Start worker pool (it calculates initial workers internally)
|
|
self._worker_pool.start()
|
|
- if self._abort_request_reason is not None:
|
|
- self._worker_pool.request_stop()
|
|
- return
|
|
|
|
# Register response nodes
|
|
for node in self._graph.nodes.values():
|
|
@@ -408,70 +338,23 @@
|
|
self._state_manager.enqueue_node(node_id)
|
|
self._state_manager.start_execution(node_id)
|
|
|
|
- # A request racing startup is signalled again before dispatcher launch.
|
|
- if self._abort_request_reason is not None:
|
|
- self._worker_pool.request_stop()
|
|
- return
|
|
# Start dispatcher
|
|
self._dispatcher.start()
|
|
|
|
- def _stop_execution(self, *, notify_layers: bool = True) -> dict:
|
|
- """One finite owner teardown, before every terminal layer notification."""
|
|
- if self._producer_owner is not threading.current_thread():
|
|
- raise RuntimeError("Producer owner mismatch")
|
|
- if self._teardown_result is None:
|
|
- errors = list(self._abort_request_errors)
|
|
- pending_workers = ()
|
|
- dispatcher_pending = True
|
|
- worker_results = ()
|
|
- results_available = False
|
|
+ def _stop_execution(self) -> None:
|
|
+ """Stop execution subsystems."""
|
|
+ self._dispatcher.stop()
|
|
+ self._worker_pool.stop()
|
|
+ # Don't mark complete here as the dispatcher already does it
|
|
+
|
|
+ # Notify layers
|
|
+ logger = logging.getLogger(__name__)
|
|
+
|
|
+ for layer in self._layers:
|
|
try:
|
|
- self._worker_pool.request_stop()
|
|
- except BaseException as error:
|
|
- errors.append(error)
|
|
- try:
|
|
- self._dispatcher.stop()
|
|
- thread = self._dispatcher._thread
|
|
- dispatcher_pending = thread is not None and thread.is_alive()
|
|
- except BaseException as error:
|
|
- errors.append(error)
|
|
- try:
|
|
- pending_workers = self._worker_pool.stop()
|
|
- if not isinstance(pending_workers, tuple):
|
|
- pending_workers = ()
|
|
- raise RuntimeError("Invalid owned pool stop result")
|
|
- errors.extend(self._worker_pool.stop_errors)
|
|
- observed = self._worker_pool.local_teardown_results
|
|
- results_available = (isinstance(observed, tuple)
|
|
- and all(isinstance(item, WorkerLocalResult) for item in observed))
|
|
- if results_available:
|
|
- worker_results = observed
|
|
- except BaseException as error:
|
|
- errors.append(error)
|
|
- if notify_layers and not self._end_layers_notified:
|
|
- self._end_layers_notified = True
|
|
- for layer in self._layers:
|
|
- try:
|
|
- selected = self._primary_error if self._primary_error is not None else self._graph_execution.error
|
|
- self._event_manager.invoke_lifecycle(layer, "on_graph_end", selected)
|
|
- except BaseException as error:
|
|
- errors.append(error)
|
|
- lifecycle_failures = ()
|
|
- try:
|
|
- lifecycle_failures = self._event_manager.lifecycle_failures
|
|
- except BaseException as error:
|
|
- errors.append(error)
|
|
- graph_primary = self._graph_execution.error
|
|
- if isinstance(graph_primary, GraphAbortError):
|
|
- graph_primary = None
|
|
- self._local_teardown_result = ProducerTeardownResult(tuple(pending_workers), dispatcher_pending,
|
|
- tuple(worker_results), results_available, tuple(errors), self._primary_error,
|
|
- graph_primary_error=graph_primary, lifecycle_failures=tuple(lifecycle_failures))
|
|
- self._teardown_result = {"pending_workers": tuple(pending_workers), "dispatcher_pending": dispatcher_pending,
|
|
- "errors": errors, "worker_results": tuple(worker_results)}
|
|
- if not self._local_teardown_result.confirmed_local:
|
|
- safe_lifecycle_log(logger, "GRAPH_LOCAL_TEARDOWN_UNCONFIRMED")
|
|
- return self._teardown_result
|
|
+ layer.on_graph_end(self._graph_execution.error)
|
|
+ except Exception as e:
|
|
+ logger.warning("Layer %s failed on_graph_end: %s", layer.__class__.__name__, e)
|
|
|
|
# Public property accessors for attributes that need external access
|
|
@property
|
|
--- a/core/workflow/graph_engine/layers/persistence.py
|
|
+++ b/core/workflow/graph_engine/layers/persistence.py
|
|
@@ -74,8 +74,6 @@
|
|
class WorkflowPersistenceLayer(GraphEngineLayer):
|
|
"""GraphEngine layer that persists workflow and node execution state."""
|
|
|
|
- critical_persistence: bool = True
|
|
-
|
|
def __init__(
|
|
self,
|
|
*,
|
|
--- a/core/workflow/graph_engine/layers/persistence_failure.py
|
|
+++ /dev/null
|
|
@@ -1,22 +0,0 @@
|
|
-"""Safe failure identity for critical workflow persistence.
|
|
-
|
|
-This is local lifecycle failure visibility, not proof of provider cancellation
|
|
-or a persisted workflow terminal status. The original exception is private.
|
|
-"""
|
|
-
|
|
-
|
|
-class CriticalWorkflowPersistenceError(RuntimeError):
|
|
- """Fixed public error with a separately retained internal original cause.
|
|
-
|
|
- Constructor text is intentionally ignored, including when the existing
|
|
- GraphExecution error deserializer reconstructs an exception by message.
|
|
- Callers must not export the private original or exception context fields.
|
|
- """
|
|
-
|
|
- __slots__ = ("_original_error",)
|
|
- code = "WORKFLOW_PERSISTENCE_FAILURE"
|
|
- message = "Workflow persistence failed"
|
|
-
|
|
- def __init__(self, _message: str | None = None) -> None:
|
|
- super().__init__(self.message)
|
|
- self._original_error: BaseException | None = None
|
|
--- a/core/workflow/graph_engine/local_teardown.py
|
|
+++ /dev/null
|
|
@@ -1,154 +0,0 @@
|
|
-"""Private immutable producer cleanup facts, never remote/provider proof."""
|
|
-from dataclasses import dataclass, field, replace
|
|
-import threading
|
|
-
|
|
-
|
|
-class GraphAbortError(RuntimeError):
|
|
- """Typed original-graph abort intent; ordinary failures remain distinct."""
|
|
-
|
|
-
|
|
-class OrdinaryPrimaryUnresolved(RuntimeError):
|
|
- code = "ORDINARY_PRIMARY_UNRESOLVED"
|
|
- def __init__(self, _message=None):
|
|
- super().__init__("Local original-exception evidence is unresolved")
|
|
-
|
|
-
|
|
-@dataclass(frozen=True)
|
|
-class LifecycleCallbackFailure:
|
|
- phase: str
|
|
- critical: bool
|
|
- original_error: BaseException = field(repr=False, compare=False)
|
|
-
|
|
-
|
|
-class LocalTeardownUnconfirmed(RuntimeError):
|
|
- code = "LOCAL_TEARDOWN_UNCONFIRMED"
|
|
- def __init__(self, _message=None):
|
|
- super().__init__("Local producer teardown is unconfirmed")
|
|
-
|
|
-
|
|
-def safe_lifecycle_log(logger, code):
|
|
- try:
|
|
- logger.warning(code)
|
|
- except BaseException:
|
|
- pass
|
|
-
|
|
-
|
|
-SNAPSHOT_BOOLS = (
|
|
- "signal_requested", "cancellation_observed", "producer_started",
|
|
- "producer_exit_observed", "worker_returned", "remote_ack_observed",
|
|
-)
|
|
-SNAPSHOT_COUNTS = (
|
|
- "next_inflight_count", "iterator_created_count", "iterator_exhausted_count",
|
|
- "iterator_close_attempted_count", "iterator_close_succeeded_count",
|
|
- "iterator_close_failed_count", "iterator_close_unavailable_count",
|
|
- "iterator_close_thread_mismatch_count", "iterator_repeat_close_ignored_count",
|
|
- "primary_exception_count", "primary_exception_identity_unknown_count",
|
|
-)
|
|
-
|
|
-
|
|
-@dataclass(frozen=True)
|
|
-class WorkerLocalResult:
|
|
- epoch: int
|
|
- worker: object = field(repr=False, compare=False)
|
|
- returned: bool
|
|
- snapshot: tuple[tuple[str, bool | int], ...]
|
|
- snapshot_valid: bool
|
|
- observation_error: BaseException | None = field(default=None, repr=False, compare=False)
|
|
-
|
|
- @property
|
|
- def primary_exception_count(self):
|
|
- value = dict(self.snapshot).get("primary_exception_count")
|
|
- return value if type(value) is int and value >= 0 else None
|
|
-
|
|
- @property
|
|
- def primary_exception_identity_unknown_count(self):
|
|
- value = dict(self.snapshot).get("primary_exception_identity_unknown_count")
|
|
- return value if type(value) is int and value >= 0 else None
|
|
-
|
|
- @property
|
|
- def confirmed_local(self):
|
|
- data = dict(self.snapshot)
|
|
- return (self.returned and self.snapshot_valid and self.observation_error is None
|
|
- and data["producer_started"] and data["producer_exit_observed"]
|
|
- and data["worker_returned"] and data["next_inflight_count"] == 0
|
|
- and data["iterator_close_failed_count"] == 0
|
|
- and data["iterator_close_unavailable_count"] == 0
|
|
- and data["iterator_close_thread_mismatch_count"] == 0
|
|
- and data["iterator_close_attempted_count"] == data["iterator_close_succeeded_count"]
|
|
- and data["iterator_created_count"] <= data["iterator_close_succeeded_count"]
|
|
- and data["remote_ack_observed"] is False)
|
|
-
|
|
-
|
|
-def observe_worker(epoch, worker):
|
|
- returned = False
|
|
- data = {}
|
|
- valid = False
|
|
- error = None
|
|
- try:
|
|
- returned = not worker.is_alive()
|
|
- raw = worker.local_cancellation_snapshot()
|
|
- if isinstance(raw, dict):
|
|
- # Invalid mutable values never enter a frozen observation.
|
|
- data = {key: raw[key] for key in SNAPSHOT_BOOLS + SNAPSHOT_COUNTS
|
|
- if key in raw and type(raw[key]) in (bool, int)}
|
|
- valid = (len(data) == len(SNAPSHOT_BOOLS) + len(SNAPSHOT_COUNTS)
|
|
- and all(type(data[key]) is bool for key in SNAPSHOT_BOOLS)
|
|
- and all(type(data[key]) is int and data[key] >= 0 for key in SNAPSHOT_COUNTS))
|
|
- except BaseException as failure:
|
|
- error = failure
|
|
- return WorkerLocalResult(epoch, worker, returned, tuple(sorted(data.items())), valid, error)
|
|
-
|
|
-
|
|
-@dataclass(frozen=True)
|
|
-class ProducerTeardownResult:
|
|
- pending_workers: tuple[object, ...] = field(repr=False)
|
|
- dispatcher_pending: bool
|
|
- worker_results: tuple[WorkerLocalResult, ...]
|
|
- worker_results_available: bool
|
|
- secondary_errors: tuple[BaseException, ...] = field(repr=False)
|
|
- primary_error: BaseException | None = field(default=None, repr=False, compare=False)
|
|
- remote_ack_observed: bool = False
|
|
- graph_primary_error: BaseException | None = field(default=None, repr=False, compare=False)
|
|
- lifecycle_failures: tuple[LifecycleCallbackFailure, ...] = field(default=(), repr=False)
|
|
-
|
|
- @property
|
|
- def primary_exception_count(self):
|
|
- return sum(item.primary_exception_count or 0 for item in self.worker_results)
|
|
-
|
|
- @property
|
|
- def primary_exception_identity_unknown_count(self):
|
|
- return sum(item.primary_exception_identity_unknown_count or 0 for item in self.worker_results)
|
|
-
|
|
- @property
|
|
- def ordinary_primary_unresolved(self):
|
|
- # A count is evidence, never an exception object/provider attribution.
|
|
- return ((self.primary_exception_count > 0 or self.primary_exception_identity_unknown_count > 0)
|
|
- and self.graph_primary_error is None)
|
|
-
|
|
- @property
|
|
- def optional_observer_failures(self):
|
|
- return tuple(item for item in self.lifecycle_failures if not item.critical)
|
|
-
|
|
- @property
|
|
- def confirmed_local(self):
|
|
- return (not self.pending_workers and not self.dispatcher_pending
|
|
- and self.worker_results_available and not self.secondary_errors
|
|
- and all(item.confirmed_local for item in self.worker_results))
|
|
-
|
|
-
|
|
-def close_owned_producer(generator, engine, owner, primary):
|
|
- """Run only on the original owner; retain first error across explicit close."""
|
|
- if threading.current_thread() is not owner:
|
|
- failure = RuntimeError("Producer owner mismatch")
|
|
- if primary is not None:
|
|
- engine._record_cleanup_error(failure)
|
|
- return
|
|
- raise failure
|
|
- if primary is not None:
|
|
- engine._remember_primary(primary)
|
|
- try:
|
|
- generator.close()
|
|
- except BaseException as failure:
|
|
- engine._record_cleanup_error(failure)
|
|
- if primary is None:
|
|
- raise
|
|
--- a/core/workflow/graph_engine/orchestration/dispatcher.py
|
|
+++ b/core/workflow/graph_engine/orchestration/dispatcher.py
|
|
@@ -33,8 +33,6 @@
|
|
with timeout and completion detection.
|
|
"""
|
|
|
|
- _COMMAND_POLL_INTERVAL = 0.1
|
|
-
|
|
_COMMAND_TRIGGER_EVENTS = (
|
|
NodeRunSucceededEvent,
|
|
NodeRunFailedEvent,
|
|
@@ -65,7 +63,6 @@
|
|
self._thread: threading.Thread | None = None
|
|
self._stop_event = threading.Event()
|
|
self._start_time: float | None = None
|
|
- self._last_command_poll: float | None = None
|
|
|
|
def start(self) -> None:
|
|
"""Start the dispatcher thread."""
|
|
@@ -102,7 +99,6 @@
|
|
self._event_queue.task_done()
|
|
self._process_commands(event)
|
|
except queue.Empty:
|
|
- self._process_commands()
|
|
time.sleep(0.1)
|
|
|
|
self._process_commands()
|
|
@@ -125,12 +121,5 @@
|
|
self._event_emitter.mark_complete()
|
|
|
|
def _process_commands(self, event: GraphNodeEventBase | None = None):
|
|
- now = time.monotonic()
|
|
- if (
|
|
- event is None
|
|
- or isinstance(event, self._COMMAND_TRIGGER_EVENTS)
|
|
- or self._last_command_poll is None
|
|
- or now - self._last_command_poll >= self._COMMAND_POLL_INTERVAL
|
|
- ):
|
|
+ if event is None or isinstance(event, self._COMMAND_TRIGGER_EVENTS):
|
|
self._execution_coordinator.process_commands()
|
|
- self._last_command_poll = time.monotonic()
|
|
--- a/core/workflow/graph_engine/worker.py
|
|
+++ b/core/workflow/graph_engine/worker.py
|
|
@@ -4,14 +4,6 @@
|
|
Workers pull node IDs from the ready_queue, execute nodes, and push events
|
|
to the event_queue for the dispatcher to process.
|
|
"""
|
|
-
|
|
-from core.workflow.iterator_cancellation import (
|
|
- CancellationState,
|
|
- IteratorCancellationRequested,
|
|
- cancellation_checkpoint,
|
|
- owned_iterator,
|
|
- producer_scope,
|
|
-)
|
|
|
|
import contextvars
|
|
import queue
|
|
@@ -74,17 +66,12 @@
|
|
self._flask_app = flask_app
|
|
self._context_vars = context_vars
|
|
self._stop_event = threading.Event()
|
|
- self._iterator_cancellation = CancellationState(self._stop_event)
|
|
self._last_task_time = time.time()
|
|
self._layers = layers if layers is not None else []
|
|
|
|
def stop(self) -> None:
|
|
"""Signal the worker to stop processing."""
|
|
self._stop_event.set()
|
|
-
|
|
- def local_cancellation_snapshot(self) -> dict[str, object]:
|
|
- """Local cleanup facts only; never a remote cancellation ACK."""
|
|
- return self._iterator_cancellation.snapshot(worker_alive=self.is_alive())
|
|
|
|
@property
|
|
def is_idle(self) -> bool:
|
|
@@ -104,42 +91,34 @@
|
|
|
|
@override
|
|
def run(self) -> None:
|
|
- """Run only in this producer thread; stop() merely signals its Event."""
|
|
- with producer_scope(self._iterator_cancellation):
|
|
- while True:
|
|
- try:
|
|
- cancellation_checkpoint()
|
|
- node_id = self._ready_queue.get(timeout=0.1)
|
|
- except queue.Empty:
|
|
- continue
|
|
- except IteratorCancellationRequested:
|
|
- break
|
|
- node = None
|
|
- try:
|
|
- cancellation_checkpoint()
|
|
- self._last_task_time = time.time()
|
|
- node = self._graph.nodes[node_id]
|
|
- self._execute_node(node)
|
|
- except IteratorCancellationRequested:
|
|
- break
|
|
- except Exception as error:
|
|
- self._iterator_cancellation.remember_primary(error)
|
|
- if node is None:
|
|
- raise
|
|
- if self._stop_event.is_set():
|
|
- self._iterator_cancellation._observe()
|
|
- break
|
|
- error_event = NodeRunFailedEvent(
|
|
- id=str(uuid4()), node_id=node.id, node_type=node.node_type,
|
|
- in_iteration_id=None, error=str(error), start_at=datetime.now(),
|
|
- )
|
|
- try:
|
|
- cancellation_checkpoint()
|
|
- self._event_queue.put(error_event)
|
|
- except IteratorCancellationRequested:
|
|
- break
|
|
- finally:
|
|
- self._ready_queue.task_done()
|
|
+ """
|
|
+ Main worker loop.
|
|
+
|
|
+ Continuously pulls node IDs from ready_queue, executes them,
|
|
+ and pushes events to event_queue until stopped.
|
|
+ """
|
|
+ while not self._stop_event.is_set():
|
|
+ # Try to get a node ID from the ready queue (with timeout)
|
|
+ try:
|
|
+ node_id = self._ready_queue.get(timeout=0.1)
|
|
+ except queue.Empty:
|
|
+ continue
|
|
+
|
|
+ self._last_task_time = time.time()
|
|
+ node = self._graph.nodes[node_id]
|
|
+ try:
|
|
+ self._execute_node(node)
|
|
+ self._ready_queue.task_done()
|
|
+ except Exception as e:
|
|
+ error_event = NodeRunFailedEvent(
|
|
+ id=str(uuid4()),
|
|
+ node_id=node.id,
|
|
+ node_type=node.node_type,
|
|
+ in_iteration_id=None,
|
|
+ error=str(e),
|
|
+ start_at=datetime.now(),
|
|
+ )
|
|
+ self._event_queue.put(error_event)
|
|
|
|
def _execute_node(self, node: Node) -> None:
|
|
"""
|
|
@@ -148,8 +127,6 @@
|
|
Args:
|
|
node: The node instance to execute
|
|
"""
|
|
- self._iterator_cancellation.begin_node()
|
|
- cancellation_checkpoint()
|
|
node.ensure_execution_id()
|
|
|
|
error: Exception | None = None
|
|
@@ -161,36 +138,22 @@
|
|
):
|
|
self._invoke_node_run_start_hooks(node)
|
|
try:
|
|
- cancellation_checkpoint()
|
|
node_events = node.run()
|
|
- with owned_iterator(node_events) as owned_events:
|
|
- for event in owned_events:
|
|
- cancellation_checkpoint()
|
|
- self._event_queue.put(event)
|
|
+ for event in node_events:
|
|
+ self._event_queue.put(event)
|
|
except Exception as exc:
|
|
error = exc
|
|
- if isinstance(exc, IteratorCancellationRequested):
|
|
- primary = self._iterator_cancellation.node_primary_exception
|
|
- if primary is not None:
|
|
- error = primary
|
|
raise
|
|
finally:
|
|
self._invoke_node_run_end_hooks(node, error)
|
|
else:
|
|
self._invoke_node_run_start_hooks(node)
|
|
try:
|
|
- cancellation_checkpoint()
|
|
node_events = node.run()
|
|
- with owned_iterator(node_events) as owned_events:
|
|
- for event in owned_events:
|
|
- cancellation_checkpoint()
|
|
- self._event_queue.put(event)
|
|
+ for event in node_events:
|
|
+ self._event_queue.put(event)
|
|
except Exception as exc:
|
|
error = exc
|
|
- if isinstance(exc, IteratorCancellationRequested):
|
|
- primary = self._iterator_cancellation.node_primary_exception
|
|
- if primary is not None:
|
|
- error = primary
|
|
raise
|
|
finally:
|
|
self._invoke_node_run_end_hooks(node, error)
|
|
--- a/core/workflow/graph_engine/worker_management/worker_pool.py
|
|
+++ b/core/workflow/graph_engine/worker_management/worker_pool.py
|
|
@@ -6,8 +6,6 @@
|
|
"""
|
|
|
|
import logging
|
|
-import math
|
|
-import time
|
|
import queue
|
|
import threading
|
|
from typing import TYPE_CHECKING, final
|
|
@@ -19,7 +17,6 @@
|
|
from ..layers.base import GraphEngineLayer
|
|
from ..ready_queue import ReadyQueue
|
|
from ..worker import Worker
|
|
-from ..local_teardown import WorkerLocalResult, observe_worker, safe_lifecycle_log
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
@@ -84,12 +81,6 @@
|
|
self._worker_counter = 0
|
|
self._lock = threading.RLock()
|
|
self._running = False
|
|
- self._stopping = False
|
|
- self._stop_deadline: float | None = None
|
|
- self._stop_errors: list[Exception] = []
|
|
- self._epoch = 0
|
|
- self._stop_in_progress = 0
|
|
- self._worker_results: list[WorkerLocalResult] = []
|
|
|
|
# No longer tracking worker states with callbacks to avoid lock contention
|
|
|
|
@@ -101,19 +92,9 @@
|
|
initial_count: Number of workers to start with (auto-calculated if None)
|
|
"""
|
|
with self._lock:
|
|
- if self._stopping or self._stop_in_progress:
|
|
- raise RuntimeError("Prior pool stop is unconfirmed or in progress")
|
|
if self._running:
|
|
return
|
|
- if any(worker.is_alive() for worker in self._workers):
|
|
- raise RuntimeError("Previous workers have not exited")
|
|
- for worker in self._workers:
|
|
- self._capture_worker_result(worker)
|
|
- self._workers = []
|
|
- self._epoch += 1
|
|
- self._stopping = False
|
|
- self._stop_deadline = None
|
|
- self._stop_errors = []
|
|
+
|
|
self._running = True
|
|
|
|
# Calculate initial worker count
|
|
@@ -138,83 +119,28 @@
|
|
for _ in range(initial_count):
|
|
self._create_worker()
|
|
|
|
- def request_stop(self) -> None:
|
|
- """Latch stopping and signal exactly this epoch under the same lock."""
|
|
+ def stop(self) -> None:
|
|
+ """Stop all workers in the pool."""
|
|
with self._lock:
|
|
self._running = False
|
|
- self._stopping = True
|
|
- for worker in tuple(self._workers):
|
|
- try:
|
|
- worker.stop()
|
|
- except Exception as error:
|
|
- self._stop_errors.append(error)
|
|
-
|
|
- @property
|
|
- def stop_errors(self) -> tuple[Exception, ...]:
|
|
- with self._lock:
|
|
- return tuple(self._stop_errors)
|
|
-
|
|
- def stop(self, timeout: float = 10.0) -> tuple[Worker, ...]:
|
|
- """Finite group join outside the lock; every result remains owned."""
|
|
- if not math.isfinite(timeout) or timeout < 0 or timeout > 10.0:
|
|
- raise ValueError("Worker stop timeout must be between zero and ten seconds")
|
|
- self.request_stop()
|
|
- with self._lock:
|
|
- self._stop_in_progress += 1
|
|
- epoch = self._epoch
|
|
- deadline = time.monotonic() + timeout
|
|
- self._stop_deadline = deadline if self._stop_deadline is None else min(self._stop_deadline, deadline)
|
|
- deadline = self._stop_deadline
|
|
- workers = tuple(self._workers)
|
|
- dead = set()
|
|
- errors = []
|
|
- try:
|
|
- for worker in workers:
|
|
- try:
|
|
- if worker.is_alive() and worker is not threading.current_thread():
|
|
- worker.join(timeout=max(0.0, deadline - time.monotonic()))
|
|
- if not worker.is_alive():
|
|
- dead.add(worker)
|
|
- except Exception as error:
|
|
- errors.append(error)
|
|
- finally:
|
|
- with self._lock:
|
|
- self._stop_errors.extend(errors)
|
|
- for worker in workers:
|
|
- self._capture_worker_result(worker)
|
|
- self._workers = [worker for worker in self._workers if worker not in dead]
|
|
- self._stop_in_progress -= 1
|
|
- pending = tuple(self._workers)
|
|
- current = self.local_teardown_results
|
|
- if (self._epoch == epoch and not pending and not self._stop_in_progress
|
|
- and not self._stop_errors and all(item.confirmed_local for item in current)):
|
|
- self._stopping = False
|
|
- if pending or errors:
|
|
- safe_lifecycle_log(logger, "WORKER_TEARDOWN_UNCONFIRMED")
|
|
- return pending
|
|
-
|
|
- def _capture_worker_result(self, worker: Worker) -> None:
|
|
- # Caller holds the pool lock. B's fixed snapshot performs local reads only.
|
|
- self._worker_results.append(observe_worker(self._epoch, worker))
|
|
-
|
|
- @property
|
|
- def local_teardown_results(self) -> tuple[WorkerLocalResult, ...]:
|
|
- with self._lock:
|
|
- latest = {}
|
|
- for item in self._worker_results:
|
|
- if item.epoch == self._epoch:
|
|
- latest[item.worker] = item
|
|
- return tuple(latest.values())
|
|
-
|
|
- @property
|
|
- def local_teardown_history(self) -> tuple[WorkerLocalResult, ...]:
|
|
- with self._lock:
|
|
- return tuple(self._worker_results)
|
|
+ worker_count = len(self._workers)
|
|
+
|
|
+ if worker_count > 0:
|
|
+ logger.debug("Stopping worker pool: %d workers", worker_count)
|
|
+
|
|
+ # Stop all workers
|
|
+ for worker in self._workers:
|
|
+ worker.stop()
|
|
+
|
|
+ # Wait for workers to finish
|
|
+ for worker in self._workers:
|
|
+ if worker.is_alive():
|
|
+ worker.join(timeout=10.0)
|
|
+
|
|
+ self._workers.clear()
|
|
|
|
def _create_worker(self) -> None:
|
|
"""Create and start a new worker."""
|
|
- if not self._running or self._stopping:
|
|
- raise RuntimeError("Cannot create a worker during teardown")
|
|
worker_id = self._worker_counter
|
|
self._worker_counter += 1
|
|
|
|
@@ -231,15 +157,18 @@
|
|
worker.start()
|
|
self._workers.append(worker)
|
|
|
|
- def _remove_worker(self, worker: Worker, worker_id: int) -> bool:
|
|
- """Request scale-down without losing a still-running worker reference."""
|
|
+ def _remove_worker(self, worker: Worker, worker_id: int) -> None:
|
|
+ """Remove a specific worker from the pool."""
|
|
+ # Stop the worker
|
|
worker.stop()
|
|
+
|
|
+ # Wait for it to finish
|
|
if worker.is_alive():
|
|
- return False
|
|
+ worker.join(timeout=2.0)
|
|
+
|
|
+ # Remove from list
|
|
if worker in self._workers:
|
|
- self._capture_worker_result(worker)
|
|
self._workers.remove(worker)
|
|
- return True
|
|
|
|
def _try_scale_up(self, queue_depth: int, current_count: int) -> bool:
|
|
"""
|
|
@@ -309,22 +238,19 @@
|
|
# Remove idle workers if any found
|
|
if workers_to_remove:
|
|
old_count = current_count
|
|
- removed = 0
|
|
for worker, worker_id in workers_to_remove:
|
|
- removed += int(self._remove_worker(worker, worker_id))
|
|
- if removed == 0:
|
|
- return False
|
|
+ self._remove_worker(worker, worker_id)
|
|
|
|
logger.debug(
|
|
"Scaled down workers: %d -> %d (removed %d idle workers after %.1fs, "
|
|
"queue_depth=%d, active=%d, idle=%d)",
|
|
old_count,
|
|
len(self._workers),
|
|
- removed,
|
|
+ len(workers_to_remove),
|
|
self._scale_down_idle_time,
|
|
queue_depth,
|
|
active_count,
|
|
- idle_count - removed,
|
|
+ idle_count - len(workers_to_remove),
|
|
)
|
|
return True
|
|
|
|
@@ -336,13 +262,6 @@
|
|
if not self._running:
|
|
return
|
|
|
|
- retained = []
|
|
- for worker in self._workers:
|
|
- if worker.is_alive():
|
|
- retained.append(worker)
|
|
- else:
|
|
- self._capture_worker_result(worker)
|
|
- self._workers = retained
|
|
current_count = len(self._workers)
|
|
queue_depth = self._ready_queue.qsize()
|
|
|
|
@@ -371,8 +290,6 @@
|
|
with self._lock:
|
|
return {
|
|
"total_workers": len(self._workers),
|
|
- "stop_pending_workers": len(self._workers) if self._stopping else 0,
|
|
- "stop_error_count": len(self._stop_errors),
|
|
"queue_depth": self._ready_queue.qsize(),
|
|
"min_workers": self._min_workers,
|
|
"max_workers": self._max_workers,
|
|
--- a/core/workflow/iterator_cancellation.py
|
|
+++ /dev/null
|
|
@@ -1,373 +0,0 @@
|
|
-"""Producer-thread-owned cooperative cancellation and iterator cleanup.
|
|
-
|
|
-Local facts only. A successful close is not a daemon/provider cancellation ACK.
|
|
-Worker.stop must only signal its Event; executing next/close is not preempted.
|
|
-"""
|
|
-
|
|
-from collections.abc import Callable, Generator, Iterator
|
|
-from contextlib import contextmanager
|
|
-import sys
|
|
-import threading
|
|
-from types import GeneratorType
|
|
-from typing import Any
|
|
-
|
|
-
|
|
-class IteratorCancellationRequested(Exception):
|
|
- """The current producer observed its own Worker's stop Event."""
|
|
-
|
|
-
|
|
-class IteratorOwnerMismatch(RuntimeError):
|
|
- """An iterator operation was attempted outside its owning thread."""
|
|
-
|
|
-
|
|
-_CURRENT = threading.local()
|
|
-_THROW_OMITTED = object()
|
|
-_COUNT_NAMES = (
|
|
- "iterator_created_count", "iterator_exhausted_count", "next_inflight_count",
|
|
- "iterator_close_attempted_count", "iterator_close_succeeded_count",
|
|
- "iterator_close_failed_count", "iterator_close_unavailable_count",
|
|
- "iterator_close_thread_mismatch_count", "iterator_repeat_close_ignored_count",
|
|
-)
|
|
-
|
|
-
|
|
-class CancellationState:
|
|
- def __init__(self, stop_event: threading.Event):
|
|
- self._event = stop_event
|
|
- self._lock = threading.Lock()
|
|
- self._owner: threading.Thread | None = None
|
|
- self._started = False
|
|
- self._exited = False
|
|
- self._observed = False
|
|
- self._counts = dict.fromkeys(_COUNT_NAMES, 0)
|
|
- self._primary_exception: BaseException | None = None
|
|
- self._node_primary_exception: BaseException | None = None
|
|
- self._primary_exception_count = 0
|
|
- self._primary_exception_identity_unknown_count = 0
|
|
-
|
|
- @property
|
|
- def primary_exception(self) -> BaseException | None:
|
|
- """First known local identity only; never an identity for unknown observations."""
|
|
- with self._lock:
|
|
- return self._primary_exception
|
|
-
|
|
- @property
|
|
- def node_primary_exception(self) -> BaseException | None:
|
|
- """This node's first original identity, separate from Cancel control."""
|
|
- with self._lock:
|
|
- return self._node_primary_exception
|
|
-
|
|
- def begin_node(self) -> None:
|
|
- with self._lock:
|
|
- self._node_primary_exception = None
|
|
-
|
|
- def remember_primary(self, error: BaseException, *, injected: bool = False) -> None:
|
|
- if not injected and isinstance(error, (IteratorCancellationRequested, GeneratorExit, StopIteration)):
|
|
- return
|
|
- with self._lock:
|
|
- if self._primary_exception is None:
|
|
- self._primary_exception = error
|
|
- if self._node_primary_exception is None:
|
|
- self._node_primary_exception = error
|
|
- self._primary_exception_count += 1
|
|
-
|
|
- def remember_primary_identity_unknown(self) -> None:
|
|
- """Record missing consumed identity without constructing or inventing one."""
|
|
- with self._lock:
|
|
- self._primary_exception_identity_unknown_count += 1
|
|
-
|
|
- def _count(self, key: str, amount: int = 1) -> None:
|
|
- with self._lock:
|
|
- self._counts[key] += amount
|
|
-
|
|
- def _start(self) -> None:
|
|
- owner = threading.current_thread()
|
|
- with self._lock:
|
|
- if self._owner is not None and self._owner is not owner:
|
|
- raise IteratorOwnerMismatch("producer_owner_mismatch")
|
|
- self._owner = owner
|
|
- self._started = True
|
|
-
|
|
- def _exit(self) -> None:
|
|
- with self._lock:
|
|
- self._observed = self._observed or self._event.is_set()
|
|
- self._exited = True
|
|
-
|
|
- def _observe(self) -> None:
|
|
- with self._lock:
|
|
- self._observed = True
|
|
-
|
|
- def snapshot(self, *, worker_alive: bool) -> dict[str, Any]:
|
|
- with self._lock:
|
|
- return {
|
|
- "signal_requested": self._event.is_set(),
|
|
- "cancellation_observed": self._observed,
|
|
- "producer_started": self._started,
|
|
- "producer_exit_observed": self._exited,
|
|
- "worker_returned": self._exited and not worker_alive,
|
|
- "primary_exception_count": self._primary_exception_count,
|
|
- "primary_exception_identity_unknown_count": self._primary_exception_identity_unknown_count,
|
|
- **self._counts,
|
|
- "remote_ack_observed": False,
|
|
- }
|
|
-
|
|
-
|
|
-@contextmanager
|
|
-def producer_scope(state: CancellationState):
|
|
- previous = getattr(_CURRENT, "state", None)
|
|
- state._start()
|
|
- _CURRENT.state = state
|
|
- try:
|
|
- yield
|
|
- finally:
|
|
- state._exit()
|
|
- _CURRENT.state = previous
|
|
-
|
|
-
|
|
-def cancellation_checkpoint() -> None:
|
|
- state = getattr(_CURRENT, "state", None)
|
|
- if state is not None and state._event.is_set():
|
|
- state._observe()
|
|
- raise IteratorCancellationRequested("producer_cancellation_requested")
|
|
-
|
|
-
|
|
-def remember_primary_exception(error: BaseException) -> None:
|
|
- """Retain original identity in this producer; export only a safe count."""
|
|
- state = getattr(_CURRENT, "state", None)
|
|
- if state is not None:
|
|
- state.remember_primary(error)
|
|
-
|
|
-
|
|
-def _closed_native_throw(*args):
|
|
- # Let the running Python's native Generator enforce all signature rules,
|
|
- # tuple construction and explicit traceback handling without any source next.
|
|
- def inert():
|
|
- if False:
|
|
- yield
|
|
- validator = inert()
|
|
- validator.close()
|
|
- return validator.throw(*args)
|
|
-
|
|
-
|
|
-def _normalized_native_throw(*args):
|
|
- # Only this private receiver is advanced. No owned source handler is entered.
|
|
- def receiver():
|
|
- try:
|
|
- yield
|
|
- except BaseException as error:
|
|
- return error
|
|
- validator = receiver()
|
|
- next(validator)
|
|
- try:
|
|
- validator.throw(*args)
|
|
- except StopIteration as ended:
|
|
- return ended.value
|
|
- finally:
|
|
- validator.close()
|
|
-
|
|
-
|
|
-class OwnedIterator(Generator):
|
|
- """Close one iterator and explicit dependents, only in the producer thread.
|
|
-
|
|
- It implements the Generator protocol so existing Generator isinstance checks
|
|
- and yield-from delegation remain valid. Dependents protect an already-created
|
|
- source even if an outer forwarding generator is closed before first next.
|
|
- """
|
|
-
|
|
- def __init__(self, source: Iterator, dependents: tuple["OwnedIterator", ...] = ()):
|
|
- self._iterator = iter(source)
|
|
- self._dependents = dependents
|
|
- self._owner = threading.current_thread()
|
|
- self._state = getattr(_CURRENT, "state", None)
|
|
- self._close_done = False
|
|
- if self._state is not None:
|
|
- self._state._count("iterator_created_count")
|
|
-
|
|
- def _assert_owner(self) -> None:
|
|
- if threading.current_thread() is not self._owner:
|
|
- if self._state is not None:
|
|
- self._state._count("iterator_close_thread_mismatch_count")
|
|
- raise IteratorOwnerMismatch("iterator_owner_mismatch")
|
|
-
|
|
- def __enter__(self):
|
|
- self._assert_owner()
|
|
- try:
|
|
- cancellation_checkpoint()
|
|
- except BaseException:
|
|
- self.__exit__(*sys.exc_info())
|
|
- raise
|
|
- return self
|
|
-
|
|
- def __exit__(self, kind, value, traceback):
|
|
- self._assert_owner()
|
|
- if self._state is not None and isinstance(value, BaseException):
|
|
- self._state.remember_primary(value)
|
|
- try:
|
|
- self.close()
|
|
- except BaseException:
|
|
- if kind is None:
|
|
- raise
|
|
- # Keep the original cancellation/GeneratorExit/ordinary exception.
|
|
- # close() has already recorded the unsuccessful cleanup.
|
|
- return False
|
|
-
|
|
- def __iter__(self):
|
|
- return self
|
|
-
|
|
- def __next__(self):
|
|
- return self.send(None)
|
|
-
|
|
- def send(self, value):
|
|
- self._assert_owner()
|
|
- if self._close_done:
|
|
- raise StopIteration
|
|
- cancellation_checkpoint()
|
|
- if self._state is not None:
|
|
- self._state._count("next_inflight_count")
|
|
- try:
|
|
- if value is None:
|
|
- result = next(self._iterator)
|
|
- else:
|
|
- result = self._iterator.send(value)
|
|
- except StopIteration:
|
|
- if self._state is not None:
|
|
- self._state._count("iterator_exhausted_count")
|
|
- raise
|
|
- except BaseException as error:
|
|
- if self._state is not None:
|
|
- self._state.remember_primary(error)
|
|
- raise
|
|
- finally:
|
|
- if self._state is not None:
|
|
- self._state._count("next_inflight_count", -1)
|
|
- # A signal arriving inside next drops that returned event. No close is
|
|
- # attempted until next has returned in this same producer thread.
|
|
- cancellation_checkpoint()
|
|
- return result
|
|
-
|
|
- def throw(self, typ, val=_THROW_OMITTED, tb=_THROW_OMITTED, /):
|
|
- self._assert_owner()
|
|
- # Argument presence is part of the delegate protocol: explicit None
|
|
- # must not collapse into an omitted second or third argument.
|
|
- args = (typ, val, tb) if tb is not _THROW_OMITTED else (typ, val) if val is not _THROW_OMITTED else (typ,)
|
|
- if self._close_done:
|
|
- return _closed_native_throw(*args)
|
|
- state = self._state or getattr(_CURRENT, "state", None)
|
|
- if state is not None and state._event.is_set():
|
|
- return self._cancel_throw(args, state)
|
|
- method = getattr(self._iterator, "throw", None)
|
|
- if method is None:
|
|
- return _closed_native_throw(*args)
|
|
- if self._state is not None:
|
|
- self._state._count("next_inflight_count")
|
|
- try:
|
|
- result = method(*args)
|
|
- except BaseException as error:
|
|
- if self._state is not None:
|
|
- self._state.remember_primary(error)
|
|
- raise
|
|
- finally:
|
|
- if self._state is not None:
|
|
- self._state._count("next_inflight_count", -1)
|
|
- if state is not None and state._event.is_set():
|
|
- return self._cancel_after_delegate(args, state)
|
|
- return result
|
|
-
|
|
- def _cancel_throw(self, args, state):
|
|
- # Signature validation errors remain outward native errors. A valid
|
|
- # injected primary is retained privately, while Cancel remains control.
|
|
- try:
|
|
- primary = _normalized_native_throw(*args)
|
|
- except BaseException as validation_error:
|
|
- state.remember_primary(validation_error)
|
|
- state._observe()
|
|
- self.__exit__(*sys.exc_info())
|
|
- raise
|
|
- state.remember_primary(primary, injected=True)
|
|
- state._observe()
|
|
- return self._close_and_raise_cancel()
|
|
-
|
|
- def _cancel_after_delegate(self, args, state):
|
|
- # The delegate already normalized/constructed its exception. Never
|
|
- # replay those parameters to try to reconstruct the consumed object.
|
|
- if isinstance(self._iterator, GeneratorType) and isinstance(args[0], BaseException) and (len(args) == 1 or args[1] is None):
|
|
- state.remember_primary(args[0], injected=True)
|
|
- else:
|
|
- # Classes/custom delegates can consume an unobservable identity.
|
|
- # Preserve that uncertainty; no gi_frame/locals inspection.
|
|
- state.remember_primary_identity_unknown()
|
|
- state._observe()
|
|
- return self._close_and_raise_cancel()
|
|
-
|
|
- def _close_and_raise_cancel(self):
|
|
- try:
|
|
- raise IteratorCancellationRequested("producer_cancellation_requested") from None
|
|
- except IteratorCancellationRequested:
|
|
- # Close only a paused iterator in this exact owner. __exit__ retains
|
|
- # the control/primary even if close fails, and counts close failure.
|
|
- self.__exit__(*sys.exc_info())
|
|
- raise
|
|
-
|
|
- def close(self):
|
|
- self._assert_owner()
|
|
- if self._close_done:
|
|
- if self._state is not None:
|
|
- self._state._count("iterator_repeat_close_ignored_count")
|
|
- return
|
|
- self._close_done = True
|
|
- primary = None
|
|
- try:
|
|
- method = getattr(self._iterator, "close", None)
|
|
- except BaseException as error:
|
|
- method = None
|
|
- primary = error
|
|
- if self._state is not None:
|
|
- self._state._count("iterator_close_attempted_count")
|
|
- self._state._count("iterator_close_failed_count")
|
|
- if method is None and primary is None:
|
|
- if self._state is not None:
|
|
- self._state._count("iterator_close_unavailable_count")
|
|
- elif method is not None:
|
|
- if self._state is not None:
|
|
- self._state._count("iterator_close_attempted_count")
|
|
- try:
|
|
- method()
|
|
- except BaseException as error:
|
|
- if self._state is not None:
|
|
- self._state._count("iterator_close_failed_count")
|
|
- primary = error
|
|
- else:
|
|
- if self._state is not None:
|
|
- self._state._count("iterator_close_succeeded_count")
|
|
- # Always try dependent cleanup, including when outer close failed.
|
|
- for child in self._dependents:
|
|
- try:
|
|
- child.close()
|
|
- except BaseException as error:
|
|
- if primary is None:
|
|
- primary = error
|
|
- if primary is not None:
|
|
- raise primary
|
|
-
|
|
-
|
|
-def owned_iterator(source: Iterator) -> OwnedIterator:
|
|
- if isinstance(source, OwnedIterator):
|
|
- source._assert_owner()
|
|
- return source
|
|
- return OwnedIterator(source)
|
|
-
|
|
-
|
|
-def owned_result(source: Any, factory: Callable[[Any], Iterator]) -> Iterator:
|
|
- """Transfer an already-created streaming result to its forwarding wrapper."""
|
|
- if not isinstance(source, Iterator):
|
|
- cancellation_checkpoint()
|
|
- return factory(source)
|
|
- child = owned_iterator(source)
|
|
- try:
|
|
- cancellation_checkpoint()
|
|
- forwarding = factory(child)
|
|
- if forwarding is child:
|
|
- return child
|
|
- return OwnedIterator(forwarding, (child,))
|
|
- except BaseException:
|
|
- child.__exit__(*sys.exc_info())
|
|
- raise
|
|
--- a/core/workflow/nodes/base/node.py
|
|
+++ b/core/workflow/nodes/base/node.py
|
|
@@ -1,10 +1,3 @@
|
|
-
|
|
-from core.workflow.iterator_cancellation import (
|
|
- remember_primary_exception,
|
|
- IteratorCancellationRequested,
|
|
- cancellation_checkpoint,
|
|
- owned_iterator,
|
|
-)
|
|
import importlib
|
|
import logging
|
|
import operator
|
|
@@ -272,7 +265,6 @@
|
|
raise NotImplementedError
|
|
|
|
def run(self) -> Generator[GraphNodeEventBase, None, None]:
|
|
- cancellation_checkpoint()
|
|
execution_id = self.ensure_execution_id()
|
|
self._start_at = naive_utc_now()
|
|
|
|
@@ -323,7 +315,6 @@
|
|
yield start_event
|
|
|
|
try:
|
|
- cancellation_checkpoint()
|
|
result = self._run()
|
|
|
|
# Handle NodeRunResult
|
|
@@ -332,22 +323,16 @@
|
|
return
|
|
|
|
# Handle event stream
|
|
- with owned_iterator(
|
|
- result
|
|
- ) as owned_node_events:
|
|
- for event in owned_node_events:
|
|
- # NOTE: this is necessary because iteration and loop nodes yield GraphNodeEventBase
|
|
- if isinstance(event, NodeEventBase): # pyright: ignore[reportUnnecessaryIsInstance]
|
|
- yield self._dispatch(event)
|
|
- elif isinstance(event, GraphNodeEventBase) and not event.in_iteration_id and not event.in_loop_id: # pyright: ignore[reportUnnecessaryIsInstance]
|
|
- event.id = self.execution_id
|
|
- yield event
|
|
- else:
|
|
- yield event
|
|
- except IteratorCancellationRequested:
|
|
- raise
|
|
+ for event in result:
|
|
+ # NOTE: this is necessary because iteration and loop nodes yield GraphNodeEventBase
|
|
+ if isinstance(event, NodeEventBase): # pyright: ignore[reportUnnecessaryIsInstance]
|
|
+ yield self._dispatch(event)
|
|
+ elif isinstance(event, GraphNodeEventBase) and not event.in_iteration_id and not event.in_loop_id: # pyright: ignore[reportUnnecessaryIsInstance]
|
|
+ event.id = self.execution_id
|
|
+ yield event
|
|
+ else:
|
|
+ yield event
|
|
except Exception as e:
|
|
- remember_primary_exception(e)
|
|
logger.exception("Node %s failed to run", self._node_id)
|
|
result = NodeRunResult(
|
|
status=WorkflowNodeExecutionStatus.FAILED,
|
|
--- a/core/workflow/nodes/llm/node.py
|
|
+++ b/core/workflow/nodes/llm/node.py
|
|
@@ -1,11 +1,3 @@
|
|
-
|
|
-from core.workflow.iterator_cancellation import (
|
|
- remember_primary_exception,
|
|
- IteratorCancellationRequested,
|
|
- cancellation_checkpoint,
|
|
- owned_iterator,
|
|
- owned_result,
|
|
-)
|
|
import base64
|
|
import io
|
|
import json
|
|
@@ -155,7 +147,6 @@
|
|
return "1"
|
|
|
|
def _run(self) -> Generator:
|
|
- cancellation_checkpoint()
|
|
node_inputs: dict[str, Any] = {}
|
|
process_data: dict[str, Any] = {}
|
|
result_text = ""
|
|
@@ -195,11 +186,10 @@
|
|
generator = self._fetch_context(node_data=self.node_data)
|
|
context = None
|
|
context_files: list[File] = []
|
|
- with owned_iterator(generator) as owned_context_events:
|
|
- for event in owned_context_events:
|
|
- context = event.context
|
|
- context_files = event.context_files or []
|
|
- yield event
|
|
+ for event in generator:
|
|
+ context = event.context
|
|
+ context_files = event.context_files or []
|
|
+ yield event
|
|
if context:
|
|
node_inputs["#context#"] = context
|
|
|
|
@@ -245,7 +235,6 @@
|
|
)
|
|
|
|
# handle invoke result
|
|
- cancellation_checkpoint()
|
|
generator = LLMNode.invoke_llm(
|
|
node_data_model=self.node_data.model,
|
|
model_instance=model_instance,
|
|
@@ -263,37 +252,36 @@
|
|
|
|
structured_output: LLMStructuredOutput | None = None
|
|
|
|
- with owned_iterator(generator) as owned_llm_events:
|
|
- for event in owned_llm_events:
|
|
- if isinstance(event, StreamChunkEvent):
|
|
- yield event
|
|
- elif isinstance(event, ModelInvokeCompletedEvent):
|
|
- # Raw text
|
|
- result_text = event.text
|
|
- usage = event.usage
|
|
- finish_reason = event.finish_reason
|
|
- reasoning_content = event.reasoning_content or ""
|
|
-
|
|
- # For downstream nodes, determine clean text based on reasoning_format
|
|
- if self.node_data.reasoning_format == "tagged":
|
|
- # Keep <think> tags for backward compatibility
|
|
- clean_text = result_text
|
|
- else:
|
|
- # Extract clean text from <think> tags
|
|
- clean_text, _ = LLMNode._split_reasoning(result_text, self.node_data.reasoning_format)
|
|
-
|
|
- # Process structured output if available from the event.
|
|
- structured_output = (
|
|
- LLMStructuredOutput(structured_output=event.structured_output)
|
|
- if event.structured_output
|
|
- else None
|
|
- )
|
|
-
|
|
- # deduct quota
|
|
- llm_utils.deduct_llm_quota(tenant_id=self.tenant_id, model_instance=model_instance, usage=usage)
|
|
- break
|
|
- elif isinstance(event, LLMStructuredOutput):
|
|
- structured_output = event
|
|
+ for event in generator:
|
|
+ if isinstance(event, StreamChunkEvent):
|
|
+ yield event
|
|
+ elif isinstance(event, ModelInvokeCompletedEvent):
|
|
+ # Raw text
|
|
+ result_text = event.text
|
|
+ usage = event.usage
|
|
+ finish_reason = event.finish_reason
|
|
+ reasoning_content = event.reasoning_content or ""
|
|
+
|
|
+ # For downstream nodes, determine clean text based on reasoning_format
|
|
+ if self.node_data.reasoning_format == "tagged":
|
|
+ # Keep <think> tags for backward compatibility
|
|
+ clean_text = result_text
|
|
+ else:
|
|
+ # Extract clean text from <think> tags
|
|
+ clean_text, _ = LLMNode._split_reasoning(result_text, self.node_data.reasoning_format)
|
|
+
|
|
+ # Process structured output if available from the event.
|
|
+ structured_output = (
|
|
+ LLMStructuredOutput(structured_output=event.structured_output)
|
|
+ if event.structured_output
|
|
+ else None
|
|
+ )
|
|
+
|
|
+ # deduct quota
|
|
+ llm_utils.deduct_llm_quota(tenant_id=self.tenant_id, model_instance=model_instance, usage=usage)
|
|
+ break
|
|
+ elif isinstance(event, LLMStructuredOutput):
|
|
+ structured_output = event
|
|
|
|
process_data = {
|
|
"model_mode": model_config.mode,
|
|
@@ -338,10 +326,7 @@
|
|
llm_usage=usage,
|
|
)
|
|
)
|
|
- except IteratorCancellationRequested:
|
|
- raise
|
|
except ValueError as e:
|
|
- remember_primary_exception(e)
|
|
yield StreamCompletedEvent(
|
|
node_run_result=NodeRunResult(
|
|
status=WorkflowNodeExecutionStatus.FAILED,
|
|
@@ -353,7 +338,6 @@
|
|
)
|
|
)
|
|
except Exception as e:
|
|
- remember_primary_exception(e)
|
|
logger.exception("error while executing llm node")
|
|
yield StreamCompletedEvent(
|
|
node_run_result=NodeRunResult(
|
|
@@ -382,7 +366,6 @@
|
|
node_type: NodeType,
|
|
reasoning_format: Literal["separated", "tagged"] = "tagged",
|
|
) -> Generator[NodeEventBase | LLMStructuredOutput, None, None]:
|
|
- cancellation_checkpoint()
|
|
model_schema = model_instance.model_type_instance.get_model_schema(
|
|
node_data_model.name, model_instance.credentials
|
|
)
|
|
@@ -395,7 +378,6 @@
|
|
)
|
|
request_start_time = time.perf_counter()
|
|
|
|
- cancellation_checkpoint()
|
|
invoke_result = invoke_llm_with_structured_output(
|
|
provider=model_instance.provider,
|
|
model_schema=model_schema,
|
|
@@ -410,7 +392,6 @@
|
|
else:
|
|
request_start_time = time.perf_counter()
|
|
|
|
- cancellation_checkpoint()
|
|
invoke_result = model_instance.invoke_llm(
|
|
prompt_messages=list(prompt_messages),
|
|
model_parameters=node_data_model.completion_params,
|
|
@@ -419,15 +400,15 @@
|
|
user=user_id,
|
|
)
|
|
|
|
- return owned_result(invoke_result, lambda owned_invoke_result: LLMNode.handle_invoke_result(
|
|
- invoke_result=owned_invoke_result,
|
|
+ return LLMNode.handle_invoke_result(
|
|
+ invoke_result=invoke_result,
|
|
file_saver=file_saver,
|
|
file_outputs=file_outputs,
|
|
node_id=node_id,
|
|
node_type=node_type,
|
|
reasoning_format=reasoning_format,
|
|
request_start_time=request_start_time,
|
|
- ))
|
|
+ )
|
|
|
|
@staticmethod
|
|
def handle_invoke_result(
|
|
@@ -441,7 +422,6 @@
|
|
request_start_time: float | None = None,
|
|
) -> Generator[NodeEventBase | LLMStructuredOutput, None, None]:
|
|
# For blocking mode
|
|
- cancellation_checkpoint()
|
|
if isinstance(invoke_result, LLMResult):
|
|
duration = None
|
|
if request_start_time is not None:
|
|
@@ -473,48 +453,42 @@
|
|
collected_structured_output = None # Collect structured_output from streaming chunks
|
|
# Consume the invoke result and handle generator exception
|
|
try:
|
|
- with owned_iterator(
|
|
- invoke_result
|
|
- ) as owned_model_chunks:
|
|
- for result in owned_model_chunks:
|
|
- if isinstance(result, LLMResultChunkWithStructuredOutput):
|
|
- # Collect structured_output from the chunk
|
|
- if result.structured_output is not None:
|
|
- collected_structured_output = dict(result.structured_output)
|
|
- yield result
|
|
- if isinstance(result, LLMResultChunk):
|
|
- contents = result.delta.message.content
|
|
- with owned_iterator(
|
|
- LLMNode._save_multimodal_output_and_convert_result_to_markdown(
|
|
- contents=contents,
|
|
- file_saver=file_saver,
|
|
- file_outputs=file_outputs,
|
|
- )
|
|
- ) as owned_text_parts:
|
|
- for text_part in owned_text_parts:
|
|
- # Detect first token for TTFT calculation
|
|
- if text_part and not has_content:
|
|
- first_token_time = time.perf_counter()
|
|
- has_content = True
|
|
-
|
|
- full_text_buffer.write(text_part)
|
|
- yield StreamChunkEvent(
|
|
- selector=[node_id, "text"],
|
|
- chunk=text_part,
|
|
- is_final=False,
|
|
- )
|
|
-
|
|
- # Update the whole metadata
|
|
- if not model and result.model:
|
|
- model = result.model
|
|
- if len(prompt_messages) == 0:
|
|
- # TODO(QuantumGhost): it seems that this update has no visable effect.
|
|
- # What's the purpose of the line below?
|
|
- prompt_messages = list(result.prompt_messages)
|
|
- if usage.prompt_tokens == 0 and result.delta.usage:
|
|
- usage = result.delta.usage
|
|
- if finish_reason is None and result.delta.finish_reason:
|
|
- finish_reason = result.delta.finish_reason
|
|
+ for result in invoke_result:
|
|
+ if isinstance(result, LLMResultChunkWithStructuredOutput):
|
|
+ # Collect structured_output from the chunk
|
|
+ if result.structured_output is not None:
|
|
+ collected_structured_output = dict(result.structured_output)
|
|
+ yield result
|
|
+ if isinstance(result, LLMResultChunk):
|
|
+ contents = result.delta.message.content
|
|
+ for text_part in LLMNode._save_multimodal_output_and_convert_result_to_markdown(
|
|
+ contents=contents,
|
|
+ file_saver=file_saver,
|
|
+ file_outputs=file_outputs,
|
|
+ ):
|
|
+ # Detect first token for TTFT calculation
|
|
+ if text_part and not has_content:
|
|
+ first_token_time = time.perf_counter()
|
|
+ has_content = True
|
|
+
|
|
+ full_text_buffer.write(text_part)
|
|
+ yield StreamChunkEvent(
|
|
+ selector=[node_id, "text"],
|
|
+ chunk=text_part,
|
|
+ is_final=False,
|
|
+ )
|
|
+
|
|
+ # Update the whole metadata
|
|
+ if not model and result.model:
|
|
+ model = result.model
|
|
+ if len(prompt_messages) == 0:
|
|
+ # TODO(QuantumGhost): it seems that this update has no visable effect.
|
|
+ # What's the purpose of the line below?
|
|
+ prompt_messages = list(result.prompt_messages)
|
|
+ if usage.prompt_tokens == 0 and result.delta.usage:
|
|
+ usage = result.delta.usage
|
|
+ if finish_reason is None and result.delta.finish_reason:
|
|
+ finish_reason = result.delta.finish_reason
|
|
except OutputParserError as e:
|
|
raise LLMNodeError(f"Failed to parse structured output: {e}")
|
|
|