From 7611ffad35a2fe5793a99d308c663c0bdcb4bb33 Mon Sep 17 00:00:00 2001
From: Aaron Meurer <asmeurer@gmail.com>
Date: Wed, 31 Aug 2022 16:17:28 -0600
Subject: [PATCH 1/9] Add first draft of JSON schema
This was generated by https://json-schema-inferrer.herokuapp.com/ from an
example report.
---
report.schema.json | 267 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 267 insertions(+)
create mode 100644 report.schema.json
diff --git a/report.schema.json b/report.schema.json
new file mode 100644
index 00000000..9d833c8b
--- /dev/null
+++ b/report.schema.json
@@ -0,0 +1,267 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "properties": {
+ "duration": {
+ "type": "number"
+ },
+ "summary": {
+ "type": "object",
+ "properties": {
+ "total": {
+ "type": "integer"
+ },
+ "collected": {
+ "type": "integer"
+ },
+ "passed": {
+ "type": "integer"
+ },
+ "failed": {
+ "type": "integer"
+ }
+ }
+ },
+ "environment": {
+ "type": "object",
+ "properties": {
+ "Platform": {
+ "type": "string"
+ },
+ "Packages": {
+ "type": "object",
+ "properties": {
+ "pytest": {
+ "type": "string"
+ },
+ "pluggy": {
+ "type": "string"
+ },
+ "py": {
+ "type": "string"
+ }
+ }
+ },
+ "array_api_tests_version": {
+ "type": "string"
+ },
+ "array_api_tests_module": {
+ "type": "string"
+ },
+ "Python": {
+ "type": "string"
+ },
+ "Plugins": {
+ "type": "object",
+ "properties": {
+ "metadata": {
+ "type": "string"
+ },
+ "dependency": {
+ "type": "string"
+ },
+ "pudb": {
+ "type": "string"
+ },
+ "json-report": {
+ "type": "string"
+ },
+ "html": {
+ "type": "string"
+ },
+ "cov": {
+ "type": "string"
+ },
+ "hypothesis": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "collectors": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "result": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "lineno": {
+ "type": "integer"
+ },
+ "type": {
+ "type": "string"
+ },
+ "nodeid": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "nodeid": {
+ "type": "string"
+ },
+ "outcome": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "tests": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "call": {
+ "type": "object",
+ "properties": {
+ "duration": {
+ "type": "number"
+ },
+ "longrepr": {
+ "type": "string"
+ },
+ "traceback": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "path": {
+ "type": "string"
+ },
+ "lineno": {
+ "type": "integer"
+ },
+ "message": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "outcome": {
+ "type": "string"
+ },
+ "crash": {
+ "type": "object",
+ "properties": {
+ "path": {
+ "type": "string"
+ },
+ "lineno": {
+ "type": "integer"
+ },
+ "message": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "metadata": {
+ "type": "object",
+ "properties": {
+ "test_module": {
+ "type": "string"
+ },
+ "array_api_function_name": {
+ "type": "string"
+ },
+ "hypothesis_statistics": {
+ "type": "string"
+ },
+ "test_function": {
+ "type": "string"
+ },
+ "params": {
+ "type": "object",
+ "properties": {
+ "extension": {
+ "type": "string"
+ },
+ "stub": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "lineno": {
+ "type": "integer"
+ },
+ "keywords": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "setup": {
+ "type": "object",
+ "properties": {
+ "duration": {
+ "type": "number"
+ },
+ "outcome": {
+ "type": "string"
+ }
+ }
+ },
+ "nodeid": {
+ "type": "string"
+ },
+ "teardown": {
+ "type": "object",
+ "properties": {
+ "duration": {
+ "type": "number"
+ },
+ "outcome": {
+ "type": "string"
+ }
+ }
+ },
+ "outcome": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "created": {
+ "type": "number"
+ },
+ "root": {
+ "type": "string"
+ },
+ "warnings": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "lineno": {
+ "type": "integer"
+ },
+ "filename": {
+ "type": "string"
+ },
+ "count": {
+ "type": "integer"
+ },
+ "category": {
+ "type": "string"
+ },
+ "message": {
+ "type": "string"
+ },
+ "when": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "exitcode": {
+ "type": "integer"
+ }
+ }
+}
From e2c04b9a0378d5fac5683f2371e54343d49091e7 Mon Sep 17 00:00:00 2001
From: Aaron Meurer <asmeurer@gmail.com>
Date: Wed, 31 Aug 2022 18:14:19 -0600
Subject: [PATCH 2/9] Add additionalProperties and required to the schema
properties, and fix some things
---
report.schema.json | 191 ++++++++++++++++++++++++++++++++++++---------
1 file changed, 154 insertions(+), 37 deletions(-)
diff --git a/report.schema.json b/report.schema.json
index 9d833c8b..c442deac 100644
--- a/report.schema.json
+++ b/report.schema.json
@@ -19,8 +19,18 @@
},
"failed": {
"type": "integer"
+ },
+ "skipped": {
+ "type": "integer"
+ },
+ "deselected": {
+ "type": "integer"
}
- }
+ },
+ "additionalProperties": false,
+ "required": [
+ "total"
+ ]
},
"environment": {
"type": "object",
@@ -40,7 +50,15 @@
"py": {
"type": "string"
}
- }
+ },
+ "additionalProperties": {
+ "type": "string"
+ },
+ "required": [
+ "pytest",
+ "pluggy",
+ "py"
+ ]
},
"array_api_tests_version": {
"type": "string"
@@ -57,27 +75,31 @@
"metadata": {
"type": "string"
},
- "dependency": {
- "type": "string"
- },
- "pudb": {
- "type": "string"
- },
"json-report": {
"type": "string"
},
- "html": {
- "type": "string"
- },
- "cov": {
- "type": "string"
- },
"hypothesis": {
"type": "string"
}
- }
+ },
+ "additionalProperties": {
+ "type": "string"
+ },
+ "required": [
+ "metadata",
+ "json-report",
+ "hypothesis"
+ ]
}
- }
+ },
+ "required": [
+ "Platform",
+ "Packages",
+ "array_api_tests_version",
+ "array_api_tests_module",
+ "Python",
+ "Plugins"
+ ]
},
"collectors": {
"type": "array",
@@ -98,7 +120,12 @@
"nodeid": {
"type": "string"
}
- }
+ },
+ "additionalProperties": false,
+ "required": [
+ "type",
+ "nodeid"
+ ]
}
},
"nodeid": {
@@ -106,8 +133,17 @@
},
"outcome": {
"type": "string"
+ },
+ "longrepr": {
+ "type": "string"
}
- }
+ },
+ "additionalProperties": false,
+ "required": [
+ "result",
+ "nodeid",
+ "outcome"
+ ]
}
},
"tests": {
@@ -138,7 +174,12 @@
"message": {
"type": "string"
}
- }
+ },
+ "required": [
+ "path",
+ "lineno",
+ "message"
+ ]
}
},
"outcome": {
@@ -156,9 +197,35 @@
"message": {
"type": "string"
}
- }
+ },
+ "required": [
+ "path",
+ "lineno",
+ "message"
+ ]
+ },
+ "stdout": {
+ "type": "string"
+ },
+ "stderr": {
+ "type": "string"
+ },
+ "stdout": {
+ "type": "string"
}
- }
+ },
+ "additionalProperties": {
+ "type": [
+ "number",
+ "string",
+ "array",
+ "object"
+ ]
+ },
+ "required": [
+ "duration",
+ "outcome"
+ ]
},
"metadata": {
"type": "object",
@@ -167,26 +234,33 @@
"type": "string"
},
"array_api_function_name": {
- "type": "string"
+ "type": [
+ "string",
+ "null"
+ ]
},
"hypothesis_statistics": {
"type": "string"
},
+ "hypothesis_report_information": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
"test_function": {
"type": "string"
},
"params": {
- "type": "object",
- "properties": {
- "extension": {
- "type": "string"
- },
- "stub": {
- "type": "string"
- }
- }
+ "type": "object"
}
- }
+ },
+ "additionalProperties": false,
+ "required": [
+ "test_module",
+ "array_api_function_name",
+ "test_function"
+ ]
},
"lineno": {
"type": "integer"
@@ -205,8 +279,16 @@
},
"outcome": {
"type": "string"
+ },
+ "longrepr": {
+ "type": "string"
}
- }
+ },
+ "additionalProperties": false,
+ "required": [
+ "duration",
+ "outcome"
+ ]
},
"nodeid": {
"type": "string"
@@ -220,12 +302,26 @@
"outcome": {
"type": "string"
}
- }
+ },
+ "additionalProperties": false,
+ "required": [
+ "duration",
+ "outcome"
+ ]
},
"outcome": {
"type": "string"
}
- }
+ },
+ "additionalProperties": false,
+ "required": [
+ "lineno",
+ "keywords",
+ "setup",
+ "nodeid",
+ "teardown",
+ "outcome"
+ ]
}
},
"created": {
@@ -257,11 +353,32 @@
"when": {
"type": "string"
}
- }
+ },
+ "additionalProperties": false,
+ "required": [
+ "lineno",
+ "filename",
+ "count",
+ "category",
+ "message",
+ "when"
+ ]
}
},
"exitcode": {
"type": "integer"
}
- }
+ },
+ "additionalProperties": false,
+ "required": [
+ "duration",
+ "summary",
+ "environment",
+ "collectors",
+ "tests",
+ "created",
+ "root",
+ "warnings",
+ "exitcode"
+ ]
}
From b568f56ea2387271365329409e44c2d11c9370f0 Mon Sep 17 00:00:00 2001
From: Aaron Meurer <asmeurer@gmail.com>
Date: Thu, 1 Sep 2022 15:30:59 -0600
Subject: [PATCH 3/9] "collectors" should not be a required key
---
report.schema.json | 1 -
1 file changed, 1 deletion(-)
diff --git a/report.schema.json b/report.schema.json
index c442deac..0301f82a 100644
--- a/report.schema.json
+++ b/report.schema.json
@@ -374,7 +374,6 @@
"duration",
"summary",
"environment",
- "collectors",
"tests",
"created",
"root",
From c866954a74f8568f980cd8e6d7622b3d7281948b Mon Sep 17 00:00:00 2001
From: Aaron Meurer <asmeurer@gmail.com>
Date: Fri, 2 Sep 2022 14:05:26 -0600
Subject: [PATCH 4/9] Add several missing things to the JSON schema
---
report.schema.json | 71 +++++++++++++++-------------------------------
1 file changed, 23 insertions(+), 48 deletions(-)
diff --git a/report.schema.json b/report.schema.json
index 0301f82a..1e7d04a9 100644
--- a/report.schema.json
+++ b/report.schema.json
@@ -23,6 +23,9 @@
"skipped": {
"type": "integer"
},
+ "error": {
+ "type": "integer"
+ },
"deselected": {
"type": "integer"
}
@@ -150,8 +153,8 @@
"type": "array",
"items": {
"type": "object",
- "properties": {
- "call": {
+ "$defs": {
+ "teststage": {
"type": "object",
"properties": {
"duration": {
@@ -160,6 +163,9 @@
"longrepr": {
"type": "string"
},
+ "outcome": {
+ "type": "string"
+ },
"traceback": {
"type": "array",
"items": {
@@ -182,9 +188,6 @@
]
}
},
- "outcome": {
- "type": "string"
- },
"crash": {
"type": "object",
"properties": {
@@ -214,18 +217,25 @@
"type": "string"
}
},
- "additionalProperties": {
- "type": [
- "number",
- "string",
- "array",
- "object"
- ]
- },
+ "additionalProperties": false,
"required": [
"duration",
"outcome"
]
+ }
+ },
+ "properties": {
+ "call": {
+ "$ref": "#/properties/tests/items/$defs/teststage"
+ },
+ "crash": {
+ "$ref": "#/properties/tests/items/$defs/teststage"
+ },
+ "teardown": {
+ "$ref": "#/properties/tests/items/$defs/teststage"
+ },
+ "setup": {
+ "$ref": "#/properties/tests/items/$defs/teststage"
},
"metadata": {
"type": "object",
@@ -271,44 +281,9 @@
"type": "string"
}
},
- "setup": {
- "type": "object",
- "properties": {
- "duration": {
- "type": "number"
- },
- "outcome": {
- "type": "string"
- },
- "longrepr": {
- "type": "string"
- }
- },
- "additionalProperties": false,
- "required": [
- "duration",
- "outcome"
- ]
- },
"nodeid": {
"type": "string"
},
- "teardown": {
- "type": "object",
- "properties": {
- "duration": {
- "type": "number"
- },
- "outcome": {
- "type": "string"
- }
- },
- "additionalProperties": false,
- "required": [
- "duration",
- "outcome"
- ]
- },
"outcome": {
"type": "string"
}
From 738da29bd915756a961ede3d89be5c460488c306 Mon Sep 17 00:00:00 2001
From: Aaron Meurer <asmeurer@gmail.com>
Date: Fri, 2 Sep 2022 14:05:48 -0600
Subject: [PATCH 5/9] Add script to verify the report against the schema
---
verify_report.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
create mode 100755 verify_report.py
diff --git a/verify_report.py b/verify_report.py
new file mode 100755
index 00000000..daa516e4
--- /dev/null
+++ b/verify_report.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+import json
+import jsonschema
+
+def main():
+ schema = json.load(open('report.schema.json'))
+
+ jsonschema.Validator.check_schema(schema)
+
+ with open('.report.json') as f:
+ report = json.load(f)
+
+ jsonschema.validate(report, schema)
+
+if __name__ == '__main__':
+ main()
From 01114bfe7fa04881286ca5a618e2046fec8f2192 Mon Sep 17 00:00:00 2001
From: Aaron Meurer <asmeurer@gmail.com>
Date: Mon, 5 Sep 2022 16:01:16 -0600
Subject: [PATCH 6/9] Verify the JSON report against the schema in CI
---
.github/workflows/numpy.yml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/numpy.yml b/.github/workflows/numpy.yml
index 74eeccf4..273ee970 100644
--- a/.github/workflows/numpy.yml
+++ b/.github/workflows/numpy.yml
@@ -75,4 +75,6 @@ jobs:
EOF
- pytest -v -rxXfE --ci
+ pytest -v -rxXfE --ci --json-report
+ - name: Verify JSON report against schema
+ run: ./verify_report.py
From 516817655a5483f98b3f53c41a80917d281d4c72 Mon Sep 17 00:00:00 2001
From: Aaron Meurer <asmeurer@gmail.com>
Date: Tue, 6 Sep 2022 16:55:59 -0600
Subject: [PATCH 7/9] Allow specifying the report file in verify_report.py
---
verify_report.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/verify_report.py b/verify_report.py
index daa516e4..28f133f4 100755
--- a/verify_report.py
+++ b/verify_report.py
@@ -1,13 +1,22 @@
#!/usr/bin/env python
+import sys
import json
+
import jsonschema
def main():
+ if len(sys.argv) == 1:
+ report_file = '.report.json'
+ elif len(sys.argv) == 2:
+ report_file = sys.argv[1]
+ else:
+ sys.exit("Usage: verify_report.py [json_report_file]")
+
schema = json.load(open('report.schema.json'))
jsonschema.Validator.check_schema(schema)
- with open('.report.json') as f:
+ with open(report_file) as f:
report = json.load(f)
jsonschema.validate(report, schema)
From c6e9e530c400f69200bf7dfadd9a01443f062941 Mon Sep 17 00:00:00 2001
From: Aaron Meurer <asmeurer@gmail.com>
Date: Tue, 6 Sep 2022 16:56:12 -0600
Subject: [PATCH 8/9] Serialize float infinities as strings in the JSON report
---
reporting.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/reporting.py b/reporting.py
index f7c7d6b9..bfc1e693 100644
--- a/reporting.py
+++ b/reporting.py
@@ -7,6 +7,7 @@
import dataclasses
import json
import warnings
+import math
from hypothesis.strategies import SearchStrategy
@@ -33,6 +34,13 @@ def to_json_serializable(o):
return tuple(to_json_serializable(i) for i in o)
if isinstance(o, list):
return [to_json_serializable(i) for i in o]
+ if isinstance(o, float):
+ if math.isnan(o):
+ return "NaN"
+ if o == float('inf'):
+ return 'Infinity'
+ if o == -float('inf'):
+ return '-Infinity'
# Ensure everything is JSON serializable. If this warning is issued, it
# means the given type needs to be added above if possible.
From c79eb7c6017c8150b12678bcd447b263d8f29e73 Mon Sep 17 00:00:00 2001
From: Aaron Meurer <asmeurer@gmail.com>
Date: Tue, 6 Sep 2022 16:56:40 -0600
Subject: [PATCH 9/9] Check for float infinities in verify_report.py
---
verify_report.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/verify_report.py b/verify_report.py
index 28f133f4..01e24ee9 100755
--- a/verify_report.py
+++ b/verify_report.py
@@ -19,6 +19,9 @@ def main():
with open(report_file) as f:
report = json.load(f)
+ # Make sure there are no nans in the report
+ json.dumps(report, allow_nan=False)
+
jsonschema.validate(report, schema)
if __name__ == '__main__':