Skip to content

Add a JSON schema for the test report #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Add a JSON schema for the test report #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7611ffa
Add first draft of JSON schema
asmeurer Aug 31, 2022
e2c04b9
Add additionalProperties and required to the schema properties, and f…
asmeurer Sep 1, 2022
b568f56
"collectors" should not be a required key
asmeurer Sep 1, 2022
c866954
Add several missing things to the JSON schema
asmeurer Sep 2, 2022
738da29
Add script to verify the report against the schema
asmeurer Sep 2, 2022
01114bf
Verify the JSON report against the schema in CI
asmeurer Sep 5, 2022
5168176
Allow specifying the report file in verify_report.py
asmeurer Sep 6, 2022
c6e9e53
Serialize float infinities as strings in the JSON report
asmeurer Sep 6, 2022
c79eb7c
Check for float infinities in verify_report.py
asmeurer Sep 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/numpy.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,4 +75,6 @@ jobs:

EOF

pytest -v -rxXfE --ci
pytest -v -rxXfE --ci --json-report
- name: Verify JSON report against schema
run: ./verify_report.py
358 changes: 358 additions & 0 deletions report.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,358 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"properties": {
"duration": {
"type": "number"
},
"summary": {
"type": "object",
"properties": {
"total": {
"type": "integer"
},
"collected": {
"type": "integer"
},
"passed": {
"type": "integer"
},
"failed": {
"type": "integer"
},
"skipped": {
"type": "integer"
},
"error": {
"type": "integer"
},
"deselected": {
"type": "integer"
}
},
"additionalProperties": false,
"required": [
"total"
]
},
"environment": {
"type": "object",
"properties": {
"Platform": {
"type": "string"
},
"Packages": {
"type": "object",
"properties": {
"pytest": {
"type": "string"
},
"pluggy": {
"type": "string"
},
"py": {
"type": "string"
}
},
"additionalProperties": {
"type": "string"
},
"required": [
"pytest",
"pluggy",
"py"
]
},
"array_api_tests_version": {
"type": "string"
},
"array_api_tests_module": {
"type": "string"
},
"Python": {
"type": "string"
},
"Plugins": {
"type": "object",
"properties": {
"metadata": {
"type": "string"
},
"json-report": {
"type": "string"
},
"hypothesis": {
"type": "string"
}
},
"additionalProperties": {
"type": "string"
},
"required": [
"metadata",
"json-report",
"hypothesis"
]
}
},
"required": [
"Platform",
"Packages",
"array_api_tests_version",
"array_api_tests_module",
"Python",
"Plugins"
]
},
"collectors": {
"type": "array",
"items": {
"type": "object",
"properties": {
"result": {
"type": "array",
"items": {
"type": "object",
"properties": {
"lineno": {
"type": "integer"
},
"type": {
"type": "string"
},
"nodeid": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"type",
"nodeid"
]
}
},
"nodeid": {
"type": "string"
},
"outcome": {
"type": "string"
},
"longrepr": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"result",
"nodeid",
"outcome"
]
}
},
"tests": {
"type": "array",
"items": {
"type": "object",
"$defs": {
"teststage": {
"type": "object",
"properties": {
"duration": {
"type": "number"
},
"longrepr": {
"type": "string"
},
"outcome": {
"type": "string"
},
"traceback": {
"type": "array",
"items": {
"type": "object",
"properties": {
"path": {
"type": "string"
},
"lineno": {
"type": "integer"
},
"message": {
"type": "string"
}
},
"required": [
"path",
"lineno",
"message"
]
}
},
"crash": {
"type": "object",
"properties": {
"path": {
"type": "string"
},
"lineno": {
"type": "integer"
},
"message": {
"type": "string"
}
},
"required": [
"path",
"lineno",
"message"
]
},
"stdout": {
"type": "string"
},
"stderr": {
"type": "string"
},
"stdout": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"duration",
"outcome"
]
}
},
"properties": {
"call": {
"$ref": "#/properties/tests/items/$defs/teststage"
},
"crash": {
"$ref": "#/properties/tests/items/$defs/teststage"
},
"teardown": {
"$ref": "#/properties/tests/items/$defs/teststage"
},
"setup": {
"$ref": "#/properties/tests/items/$defs/teststage"
},
"metadata": {
"type": "object",
"properties": {
"test_module": {
"type": "string"
},
"array_api_function_name": {
"type": [
"string",
"null"
]
},
"hypothesis_statistics": {
"type": "string"
},
"hypothesis_report_information": {
"type": "array",
"items": {
"type": "string"
}
},
"test_function": {
"type": "string"
},
"params": {
"type": "object"
}
},
"additionalProperties": false,
"required": [
"test_module",
"array_api_function_name",
"test_function"
]
},
"lineno": {
"type": "integer"
},
"keywords": {
"type": "array",
"items": {
"type": "string"
}
},
"nodeid": {
"type": "string"
},
"outcome": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"lineno",
"keywords",
"setup",
"nodeid",
"teardown",
"outcome"
]
}
},
"created": {
"type": "number"
},
"root": {
"type": "string"
},
"warnings": {
"type": "array",
"items": {
"type": "object",
"properties": {
"lineno": {
"type": "integer"
},
"filename": {
"type": "string"
},
"count": {
"type": "integer"
},
"category": {
"type": "string"
},
"message": {
"type": "string"
},
"when": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"lineno",
"filename",
"count",
"category",
"message",
"when"
]
}
},
"exitcode": {
"type": "integer"
}
},
"additionalProperties": false,
"required": [
"duration",
"summary",
"environment",
"tests",
"created",
"root",
"warnings",
"exitcode"
]
}
8 changes: 8 additions & 0 deletions reporting.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import dataclasses
import json
import warnings
import math

from hypothesis.strategies import SearchStrategy

Expand All @@ -33,6 +34,13 @@ def to_json_serializable(o):
return tuple(to_json_serializable(i) for i in o)
if isinstance(o, list):
return [to_json_serializable(i) for i in o]
if isinstance(o, float):
if math.isnan(o):
return "NaN"
if o == float('inf'):
return 'Infinity'
if o == -float('inf'):
return '-Infinity'

# Ensure everything is JSON serializable. If this warning is issued, it
# means the given type needs to be added above if possible.
Expand Down
Loading