chore: New SCXML fail mark with the contents of the errors

Author: fgmacedo

statemachine/engines/sync.py

@@ -107,13 +107,13 @@ def processing_loop(self):  # noqa: C901
                 while not self.external_queue.is_empty():
                     took_events = True
                     external_event = self.external_queue.pop()
-                    logger.debug("External event: %s", external_event)
                     current_time = time()
                     if external_event.execution_time > current_time:
                         self.put(external_event)
                         sleep(0.001)
                         continue
 
+                    logger.debug("External event: %s", external_event)
                     # # TODO: Handle cancel event
                     # if self.is_cancel_event(external_event):
                     #     self.running = False

statemachine/io/scxml/processor.py

@@ -3,6 +3,7 @@
 from typing import Dict
 from typing import List
 
+from ...exceptions import InvalidDefinition
 from .. import StateDefinition
 from .. import TransitionDict
 from .. import TransitionsDict
@@ -114,7 +115,7 @@ def _add(self, location: str, definition: Dict[str, Any]):
             self.scs[location] = sc_class
             return sc_class
         except Exception as e:
-            raise Exception(
+            raise InvalidDefinition(
                 f"Failed to create state machine class: {e} from definition: {definition}"
             ) from e
 

tests/conftest.py

@@ -1,19 +1,7 @@
-import sys
 from datetime import datetime
-from typing import List
 
 import pytest
 
-collect_ignore_glob: List[str] = []
-
-# We support Python 3.8+ positional only syntax
-if sys.version_info[:2] < (3, 8):  # noqa: UP036
-    collect_ignore_glob.append("*_positional_only.py")
-
-
-# TODO: Return django to collect
-collect_ignore_glob.append("django")
-
 
 @pytest.fixture()
 def current_time():

tests/scxml/conftest.py

@@ -2,8 +2,6 @@
 
 import pytest
 
-from statemachine.io.scxml.processor import SCXMLProcessor
-
 CURRENT_DIR = Path(__file__).parent
 TESTCASES_DIR = CURRENT_DIR
 SUPPORTED_EXTENSIONS = "scxml"
@@ -14,24 +12,29 @@ def processor(testcase_path: Path):
     """
     Construct a StateMachine class from the SCXML file
     """
-    processor = SCXMLProcessor()
-    processor.parse_scxml_file(testcase_path)
     return processor
 
 
+def compute_testcase_marks(testcase_path: Path) -> list[pytest.MarkDecorator]:
+    marks = [pytest.mark.scxml]
+    if testcase_path.with_name(f"{testcase_path.stem}.fail.md").exists():
+        marks.append(pytest.mark.xfail)
+    if testcase_path.with_name(f"{testcase_path.stem}.skip.md").exists():
+        marks.append(pytest.mark.skip)
+    return marks
+
+
 def pytest_generate_tests(metafunc):
     if "testcase_path" not in metafunc.fixturenames:
         return
 
-    fail_marks = [pytest.mark.xfail]
-
     metafunc.parametrize(
         "testcase_path",
         [
             pytest.param(
                 testcase_path,
                 id=str(testcase_path.relative_to(TESTCASES_DIR)),
-                marks=fail_marks if "fail" in testcase_path.name else [],
+                marks=compute_testcase_marks(testcase_path),
             )
             for testcase_path in TESTCASES_DIR.glob("**/*.scxml")
             if "sub" not in testcase_path.name

tests/scxml/test_scxml_cases.py

@@ -1,11 +1,12 @@
+import traceback
 from dataclasses import dataclass
 from dataclasses import field
-
-import pytest
+from pathlib import Path
 
 from statemachine import State
 from statemachine import StateMachine
 from statemachine.event import Event
+from statemachine.io.scxml.processor import SCXMLProcessor
 
 """
 Test cases as defined by W3C SCXML Test Suite
@@ -18,29 +19,114 @@
 """  # noqa: E501
 
 
-@dataclass(frozen=True, unsafe_hash=True)
+@dataclass(frozen=True, unsafe_hash=True, kw_only=True)
+class DebugEvent:
+    source: str
+    event: str
+    data: str
+    target: str
+
+
+@dataclass(frozen=True, unsafe_hash=True, kw_only=True)
 class DebugListener:
-    events: list = field(default_factory=list)
+    events: list[DebugEvent] = field(default_factory=list)
 
     def on_transition(self, event: Event, source: State, target: State, event_data):
         self.events.append(
-            (
-                f"{source and source.id}",
-                f"{event and event.id}",
-                f"{event_data.trigger_data.kwargs}",
-                f"{target.id}",
+            DebugEvent(
+                source=f"{source and source.id}",
+                event=f"{event and event.id}",
+                data=f"{event_data.trigger_data.kwargs}",
+                target=f"{target.id}",
             )
         )
 
 
-@pytest.mark.scxml()
-def test_scxml_usecase(testcase_path, processor):
+@dataclass(kw_only=True)
+class FailedMark:
+    reason: str
+    events: list[DebugEvent]
+    is_assertion_error: bool
+    exception: Exception
+    logs: str
+    configuration: list[str] = field(default_factory=list)
+
+    @staticmethod
+    def _get_header(report: str) -> str:
+        header_end_index = report.find("---")
+        return report[:header_end_index]
+
+    def write_fail_markdown(self, testcase_path: Path):
+        fail_file_path = testcase_path.with_suffix(".fail.md")
+        if not self.is_assertion_error:
+            exception_traceback = "".join(
+                traceback.format_exception(
+                    type(self.exception), self.exception, self.exception.__traceback__
+                )
+            )
+        else:
+            exception_traceback = "Assertion of the testcase failed."
+
+        report = f"""# Testcase: {testcase_path.stem}
+
+{self.reason}
+
+Final configuration: `{self.configuration if self.configuration else 'No configuration'}`
+
+---
+
+## Logs
+```py
+{self.logs if self.logs else 'No logs'}
+```
+
+## "On transition" events
+```py
+{'\n'.join(map(repr, self.events)) if self.events else 'No events'}
+```
+
+## Traceback
+```py
+{exception_traceback}
+```
+"""
+
+        if fail_file_path.exists():
+            last_report = fail_file_path.read_text()
+
+            if self._get_header(report) == self._get_header(last_report):
+                return
+
+        with fail_file_path.open("w") as fail_file:
+            fail_file.write(report)
+
+
+def test_scxml_usecase(testcase_path: Path, caplog):
     # from statemachine.contrib.diagram import DotGraphMachine
 
     # DotGraphMachine(sm_class).get_graph().write_png(
     #     testcase_path.parent / f"{testcase_path.stem}.png"
     # )
-    debug = DebugListener()
-    sm = processor.start(listeners=[debug])
-    assert isinstance(sm, StateMachine)
-    assert sm.current_state.id == "pass", debug
+    sm: "StateMachine | None" = None
+    try:
+        debug = DebugListener()
+        processor = SCXMLProcessor()
+        processor.parse_scxml_file(testcase_path)
+
+        sm = processor.start(listeners=[debug])
+        assert isinstance(sm, StateMachine)
+        assert "pass" in {s.id for s in sm.configuration}, debug
+    except Exception as e:
+        # Import necessary module
+        reason = f"{e.__class__.__name__}: {e.__class__.__doc__}"
+        is_assertion_error = isinstance(e, AssertionError)
+        fail_mark = FailedMark(
+            reason=reason,
+            is_assertion_error=is_assertion_error,
+            events=debug.events,
+            exception=e,
+            logs=caplog.text,
+            configuration=[s.id for s in sm.configuration] if sm else [],
+        )
+        fail_mark.write_fail_markdown(testcase_path)
+        raise

tests/scxml/w3c/mandatory/test190.fail.md

@@ -0,0 +1,53 @@
+# Testcase: test190
+
+AssertionError: Assertion failed.
+
+Final configuration: `['fail']`
+
+---
+
+## Logs
+```py
+DEBUG    statemachine.engines.base:base.py:374 States to enter: {S0}
+DEBUG    statemachine.io.scxml.actions:actions.py:443 Error executing actions
+Traceback (most recent call last):
+  File "/home/macedo/projects/python-statemachine/statemachine/io/scxml/actions.py", line 441, in datamodel
+    act(machine=machine)
+    ~~~^^^^^^^^^^^^^^^^^
+  File "/home/macedo/projects/python-statemachine/statemachine/io/scxml/actions.py", line 412, in data_initializer
+    value = _eval(action.expr, **kwargs)
+  File "/home/macedo/projects/python-statemachine/statemachine/io/scxml/actions.py", line 125, in _eval
+    return eval(expr, {}, kwargs)
+  File "<string>", line 1, in <module>
+    import sys;exec(eval(sys.stdin.readline()))
+    ^^^^^^^^^^
+NameError: name '_sessionid' is not defined
+DEBUG    statemachine.io.scxml.actions:actions.py:467 Error executing actions
+Traceback (most recent call last):
+  File "/home/macedo/projects/python-statemachine/statemachine/io/scxml/actions.py", line 463, in __call__
+    action(*args, **kwargs)
+    ~~~~~~^^^^^^^^^^^^^^^^^
+  File "/home/macedo/projects/python-statemachine/statemachine/io/scxml/actions.py", line 231, in __call__
+    value = _eval(self.action.expr, **kwargs)
+  File "/home/macedo/projects/python-statemachine/statemachine/io/scxml/actions.py", line 125, in _eval
+    return eval(expr, {}, kwargs)
+  File "<string>", line 1, in <module>
+    import sys;exec(eval(sys.stdin.readline()))
+    ^^^^^^^^^^^
+TypeError: can only concatenate str (not "NoneType") to str
+DEBUG    statemachine.engines.sync:sync.py:64 Processing loop started: s0
+DEBUG    statemachine.engines.sync:sync.py:89 Eventless/internal queue: {transition * from S0 to Fail}
+DEBUG    statemachine.engines.base:base.py:283 States to exit: {S0}
+DEBUG    statemachine.engines.base:base.py:374 States to enter: {Fail}
+
+```
+
+## "On transition" events
+```py
+DebugEvent(source='s0', event='error.execution', data='{\'event_id\': None, \'error\': NameError("name \'_sessionid\' is not defined")}', target='fail')
+```
+
+## Traceback
+```py
+Assertion of the testcase failed.
+```

tests/scxml/w3c/mandatory/test190-fail.scxml renamed to tests/scxml/w3c/mandatory/test190.scxml

@@ -0,0 +1,28 @@
+# Testcase: test191
+
+AssertionError: Assertion failed.
+
+Final configuration: `['fail']`
+
+---
+
+## Logs
+```py
+DEBUG    statemachine.engines.base:base.py:374 States to enter: {S0}
+DEBUG    statemachine.engines.sync:sync.py:64 Processing loop started: s0
+DEBUG    statemachine.engines.sync:sync.py:116 External event: TriggerData(machine=<weakproxy at 0x7f3c5f166b10; to 'statemachine.io.test191' at 0x7f3c5f17c1a0>, event=Event('timeout', delay=2000.0, internal=False), send_id='d96644f8d15f49b2bd7440c62b61dd38', _target=None, execution_time=1733943931.664144, model=Model(state=s0), args=(), kwargs={})
+DEBUG    statemachine.engines.sync:sync.py:131 Enabled transitions: {transition * from S0 to Fail}
+DEBUG    statemachine.engines.base:base.py:283 States to exit: {S0}
+DEBUG    statemachine.engines.base:base.py:374 States to enter: {Fail}
+
+```
+
+## "On transition" events
+```py
+DebugEvent(source='s0', event='timeout', data='{}', target='fail')
+```
+
+## Traceback
+```py
+Assertion of the testcase failed.
+```

tests/scxml/w3c/mandatory/test191.fail.md

@@ -0,0 +1,28 @@
+# Testcase: test192
+
+AssertionError: Assertion failed.
+
+Final configuration: `['fail']`
+
+---
+
+## Logs
+```py
+DEBUG    statemachine.engines.base:base.py:374 States to enter: {S0, S01}
+DEBUG    statemachine.engines.sync:sync.py:64 Processing loop started: ['s0', 's01']
+DEBUG    statemachine.engines.sync:sync.py:116 External event: TriggerData(machine=<weakproxy at 0x7fd4c5346ca0; to 'statemachine.io.test192' at 0x7fd4c5429400>, event=Event('timeout', delay=2000.0, internal=False), send_id='a3f00d7e247a45d4bffc85ffc3540742', _target=None, execution_time=1733943932.6457262, model=Model(state=['s0', 's01']), args=(), kwargs={})
+DEBUG    statemachine.engines.sync:sync.py:131 Enabled transitions: {transition timeout from S0 to Fail}
+DEBUG    statemachine.engines.base:base.py:283 States to exit: {S0, S01}
+DEBUG    statemachine.engines.base:base.py:374 States to enter: {Fail}
+
+```
+
+## "On transition" events
+```py
+DebugEvent(source='s0', event='timeout', data='{}', target='fail')
+```
+
+## Traceback
+```py
+Assertion of the testcase failed.
+```

tests/scxml/w3c/mandatory/test191-fail.scxml renamed to tests/scxml/w3c/mandatory/test191.scxml

@@ -0,0 +1,28 @@
+# Testcase: test207
+
+AssertionError: Assertion failed.
+
+Final configuration: `No configuration`
+
+---
+
+## Logs
+```py
+DEBUG    statemachine.engines.base:base.py:374 States to enter: {S0, S01}
+DEBUG    statemachine.engines.sync:sync.py:64 Processing loop started: ['s0', 's01']
+DEBUG    statemachine.engines.sync:sync.py:116 External event: TriggerData(machine=<weakproxy at 0x7efc22ac3d30; to 'statemachine.io.test207' at 0x7efc22b08590>, event=Event('timeout', delay=2000.0, internal=False), send_id='c56c759204124df98a1d96e2678a307a', _target=None, execution_time=1733943926.5340483, model=Model(state=['s0', 's01']), args=(), kwargs={})
+DEBUG    statemachine.engines.sync:sync.py:131 Enabled transitions: {transition timeout from S0 to }
+DEBUG    statemachine.engines.base:base.py:283 States to exit: {S0, S01}
+DEBUG    statemachine.engines.base:base.py:374 States to enter: {}
+
+```
+
+## "On transition" events
+```py
+DebugEvent(source='s0', event='timeout', data='{}', target='')
+```
+
+## Traceback
+```py
+Assertion of the testcase failed.
+```