✨ Schema validation

Author: ubmarco

docs/index.rst

@@ -208,6 +208,7 @@ Contents
    services/index
    layout_styles
    api
+   schema
    utils
 
 .. toctree::

docs/schema.rst

@@ -0,0 +1,3 @@
+Schema validation
+=================
+

pyproject.toml

@@ -49,6 +49,8 @@ test = [
   "lxml>=4.6.5,<6.0",
   "responses~=0.22.0",
   "pytest-xprocess~=1.0",
+  "tomli ~= 2.2.1",
+  "tomli-w ~= 1.2.0",
 ]
 test-parallel = ["pytest-xdist"]
 benchmark = [
@@ -73,6 +75,7 @@ dev = ["pre-commit~=3.0", "tox~=4.23", "tox-uv~=1.15"]
 [tool.pytest.ini_options]
 markers = [
   "jstest: marks tests as JavaScript test (deselect with '-m \"not jstest\"')",
+  "benchmark: marks tests as expensive benchmark test (deselect with '-m \"not benchmark\"')",
 ]
 filterwarnings = [
   "ignore:.*removed in Python 3.14.*:DeprecationWarning",

sphinx_needs/builder.py

@@ -41,7 +41,8 @@ def write(
         updated_docnames: Sequence[str],
         method: str = "update",
     ) -> None:
-        return
+        # make sure schema validation is done
+        self.events.emit("write-started", self)
 
     def finish(self) -> None:
         from sphinx_needs.filter_common import filter_needs_view
@@ -257,3 +258,39 @@ def build_needumls_pumls(app: Sphinx, _exception: Exception) -> None:
     needs_builder.outdir = os.path.join(needs_builder.outdir, config.build_needumls)  # type: ignore[assignment]
 
     needs_builder.finish()
+
+
+class SchemaBuilder(Builder):
+    """Only validate needs schema, no output is generated."""
+
+    name = "schema"
+
+    def write(
+        self,
+        build_docnames: Iterable[str] | None,
+        updated_docnames: Sequence[str],
+        method: str = "update",
+    ) -> None:
+        # make sure schema validation is done
+        self.events.emit("write-started", self)
+
+    def write_doc(self, docname: str, doctree: nodes.document) -> None:
+        pass
+
+    def finish(self) -> None:
+        pass
+
+    def get_outdated_docs(self) -> Iterable[str]:
+        return []
+
+    def prepare_writing(self, _docnames: set[str]) -> None:
+        pass
+
+    def write_doc_serialized(self, _docname: str, _doctree: nodes.document) -> None:
+        pass
+
+    def cleanup(self) -> None:
+        pass
+
+    def get_target_uri(self, _docname: str, _typ: str | None = None) -> str:
+        return ""

sphinx_needs/config.py

@@ -11,6 +11,12 @@
 from sphinx_needs.data import GraphvizStyleType, NeedsCoreFields
 from sphinx_needs.defaults import DEFAULT_DIAGRAM_TEMPLATE
 from sphinx_needs.logging import get_logger, log_warning
+from sphinx_needs.schema.config import (
+    USER_CONFIG_SCHEMA_SEVERITIES,
+    MessageRuleEnum,
+    SchemaType,
+    SeverityEnum,
+)
 
 if TYPE_CHECKING:
     from sphinx.util.logging import SphinxLoggerAdapter
@@ -23,6 +29,10 @@
 LOGGER = get_logger(__name__)
 
 
+# TODO: checking any schema against the meta model
+# TODO: constrain the schema to disallow certain keys
+
+
 @dataclass
 class ExtraOptionParams:
     """Defines a single extra option for needs"""
@@ -31,6 +41,25 @@ class ExtraOptionParams:
     """A description of the option."""
     validator: Callable[[str | None], str]
     """A function to validate the directive option value."""
+    schema: dict[str, Any] | None
+    """A JSON schema for the option."""
+
+
+class ExtraLinkSchemaItemsType(TypedDict):
+    type: Literal["string"]
+
+
+class ExtraLinkSchemaType(TypedDict):
+    """Defines a schema for a need extra link."""
+
+    type: Literal["array"]
+    """Type for extra links, can only be array."""
+    items: ExtraLinkSchemaItemsType
+    """Schema constraints for link strings."""
+    minItems: NotRequired[int]
+    """Minimum number of items in the array."""
+    maxItems: NotRequired[int]
+    """Maximum number of items in the array."""
 
 
 class FieldDefault(TypedDict):
@@ -91,6 +120,7 @@ def add_extra_option(
         name: str,
         description: str,
         *,
+        schema: dict[str, Any] | None = None,
         validator: Callable[[str | None], str] | None = None,
         override: bool = False,
     ) -> None:
@@ -110,7 +140,9 @@ def add_extra_option(
 
                 raise NeedsApiConfigWarning(f"Option {name} already registered.")
         self._extra_options[name] = ExtraOptionParams(
-            description, directives.unchanged if validator is None else validator
+            description,
+            directives.unchanged if validator is None else validator,
+            schema,
         )
 
     @property
@@ -240,6 +272,13 @@ class LinkOptionsType(TypedDict, total=False):
     """Used for needflow. Default: '->'"""
     allow_dead_links: bool
     """If True, add a 'forbidden' class to dead links"""
+    schema: ExtraLinkSchemaType
+    """
+    A JSON schema for the link option.
+    
+    If given, the schema will apply to all needs that use this link option.
+    For more granular control and graph traversal, use the `needs_schemas` configuration.
+    """
 
 
 class NeedType(TypedDict):
@@ -263,6 +302,32 @@ class NeedExtraOption(TypedDict):
     name: str
     description: NotRequired[str]
     """A description of the option."""
+    type: NotRequired[Literal["string", "integer", "number", "boolean"]]
+    """
+    The data type for schema validation. The option will still be stored as a string,
+    but during schema validation, the value will be coerced to the given type.
+
+    The type semantics are align with JSON Schema, see
+    https://json-schema.org/understanding-json-schema/reference/type.
+
+    For booleans, a predefined set of truthy/falsy strings are accepted:
+    - truthy = {"true", "yes", "y", "on", "1"}
+    - falsy = {"false", "no", "n", "off", "0"}
+
+    ``null`` is not a valid value as Sphinx options cannot actively be set to ``null``.
+    Sphinx-Needs does not distinguish between extra options being not given and given as empty string.
+    Both cases are coerced to an empty string value ``''``.
+    """
+    schema: NotRequired[dict[str, Any]]
+    """
+    A JSON schema for the option.
+    
+    If given, the schema will apply to all needs that use this option.
+    For more granular control, use the `needs_schemas` configuration.
+    """
+    # TODO check schema on config-inited, disallow certain keys such as
+    #      [if, then, else, dependentSchemas, dependentRequired, anyOf, oneOf, not]
+    #      only allow those once usecases are requested
 
 
 class NeedStatusesOption(TypedDict):
@@ -365,6 +430,120 @@ def get_default(cls, name: str) -> Any:
         default_factory=list, metadata={"rebuild": "env", "types": (list,)}
     )
     """Path to the root table in the toml file to load configuration from."""
+    schemas: list[SchemaType] = field(
+        default_factory=list,
+        metadata={
+            "rebuild": "env",
+            "types": (list,),
+            "schema": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "id": {"type": "string", "pattern": "^[a-zA-Z0-9_-]+$"},
+                        "severity": {
+                            "type": "string",
+                            "enum": [
+                                e.name
+                                for e in SeverityEnum
+                                if e in USER_CONFIG_SCHEMA_SEVERITIES
+                            ],
+                        },
+                        "message": {"type": "string"},
+                        "types": {
+                            "type": "array",
+                            "items": {"type": "string", "minLength": 1},
+                        },
+                        "local_schema": {"type": "object"},
+                        "trigger_schema": {"type": "object"},
+                        "trigger_schema_id": {
+                            "type": "string",
+                            "pattern": "^[a-zA-Z0-9_-]+$",
+                        },
+                        "link_schema": {
+                            "type": "object",
+                            "patternProperties": {
+                                "^.*$": {
+                                    "type": "object",
+                                    "properties": {
+                                        "schema_id": {"type": "string"},
+                                        "minItems": {"type": "integer"},
+                                        "maxItems": {"type": "integer"},
+                                        "unevaluatedItems": {"type": "boolean"},
+                                    },
+                                    "additionalProperties": False,
+                                }
+                            },
+                            "additionalProperties": False,
+                        },
+                        "dependency": {"type": "boolean"},
+                    },
+                    "additionalProperties": False,
+                },
+            },
+        },
+    )
+    schemas_from_json: str | None = field(
+        default=None, metadata={"rebuild": "env", "types": (str, type(None))}
+    )
+    """Path to a JSON file to load the schemas from."""
+
+    schemas_severity: str = field(
+        default=SeverityEnum.info.name,
+        metadata={
+            "rebuild": "env",
+            "types": (str,),
+            "schema": {
+                "type": "string",
+                "enum": [
+                    e.name for e in SeverityEnum if e in USER_CONFIG_SCHEMA_SEVERITIES
+                ],
+            },
+        },
+    )
+    """Severity level for the schema validation reporting."""
+
+    schemas_debug_active: bool = field(
+        default=False,
+        metadata={"rebuild": "env", "types": (bool,), "schema": {"type": "boolean"}},
+    )
+    """Activate the debug mode for schema validation to dump JSON/schema files and messages."""
+
+    schemas_debug_path: str = field(
+        default="schema_debug",
+        metadata={
+            "rebuild": "env",
+            "types": (str,),
+            "schema": {"type": "string", "minLength": 1},
+        },
+    )
+    """
+    Path to the directory where the debug files are stored.
+
+    If the path is relative, the caller needs to make sure
+    it gets converted to a use case specific absolute path, e.g.
+    with confdir for Sphinx.
+    """
+
+    schemas_debug_ignore: list[str] = field(
+        default_factory=lambda: [
+            "skipped_dependency",
+            "skipped_wrong_type",
+            "validation_success",
+        ],
+        metadata={
+            "rebuild": "env",
+            "types": (list,),
+            "schema": {
+                "type": "array",
+                "items": {
+                    "type": "string",
+                    "enum": [e.value for e in MessageRuleEnum],
+                },
+            },
+        },
+    )
+    """List of scenarios that are ignored for dumping debug information."""
 
     types: list[NeedType] = field(
         default_factory=lambda: [

sphinx_needs/environment.py

@@ -40,7 +40,7 @@ def _add_js_file(app: Sphinx, rel_path: Path) -> None:
 def install_styles_static_files(app: Sphinx, env: BuildEnvironment) -> None:
     builder = app.builder
     # Do not copy static_files for our "needs" builder
-    if builder.name == "needs":
+    if builder.name in ["needs", "schema"]:
         return
 
     logger.info("Copying static style files for sphinx-needs")
@@ -87,7 +87,7 @@ def install_lib_static_files(app: Sphinx, env: BuildEnvironment) -> None:
     """
     builder = app.builder
     # Do not copy static_files for our "needs" builder
-    if builder.name == "needs":
+    if builder.name in ["needs", "schema"]:
         return
 
     logger.info("Copying static files for sphinx-needs datatables support")
@@ -116,7 +116,7 @@ def install_permalink_file(app: Sphinx, env: BuildEnvironment) -> None:
     """
     builder = app.builder
     # Do not copy static_files for our "needs" builder
-    if builder.name == "needs":
+    if builder.name in ["needs", "schema"]:
         return
 
     # load jinja template