diff --git a/.travis.yml b/.travis.yml index 121fdf3..ab9021d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ script: - make mypy - make docs-lint - make docs -- make benchmark +- BENCHMARK_REPEATS=1 make benchmark - ./tests/check_tag.py after_success: diff --git a/benchmarks/run.py b/benchmarks/run.py index 08dc988..73f1f43 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -1,11 +1,13 @@ import json +import os import random import string import sys from datetime import datetime from functools import partial from pathlib import Path -from statistics import mean, stdev +from statistics import StatisticsError, mean +from statistics import stdev as stdev_ from test_pydantic import TestPydantic from test_trafaret import TestTrafaret @@ -99,6 +101,13 @@ def generate_case(): THIS_DIR = Path(__file__).parent.resolve() +def stdev(d): + try: + return stdev_(d) + except StatisticsError: + return 0 + + def main(): json_path = THIS_DIR / 'cases.json' if not json_path.exists(): @@ -109,14 +118,18 @@ def main(): else: with json_path.open() as f: cases = json.load(f) + if 'pydantic-only' in sys.argv: tests = [TestPydantic] else: tests = [TestPydantic, TestTrafaret, TestDRF] + + repeats = int(os.getenv('BENCHMARK_REPEATS', '5')) + results = [] for test_class in tests: times = [] p = test_class.package - for i in range(5): + for i in range(repeats): count, pass_count = 0, 0 start = datetime.now() test = test_class(False) @@ -130,6 +143,14 @@ def main(): print(f'{p:10} time={time:0.3f}s, success={success:0.2f}%') times.append(time) print(f'{p:10} best={min(times):0.3f}s, avg={mean(times):0.3f}s, stdev={stdev(times):0.3f}s') + model_count = repeats * 3 * len(cases) + results.append(f'{p:10} per iteration: best={min(times) / model_count * 1e6:0.3f}μs, ' + f'avg={mean(times) / model_count * 1e6:0.3f}μs, ' + f'stdev={stdev(times) / model_count * 1e6:0.3f}μs') + print() + + for r in results: + print(r) if __name__ == '__main__':