speedup benchmarks

.travis.yml

@@ -26,7 +26,7 @@ script:
 - make mypy
 - make docs-lint
 - make docs
-- make benchmark
+- BENCHMARK_REPEATS=1 make benchmark
 - ./tests/check_tag.py
 
 after_success:

benchmarks/run.py

@@ -1,11 +1,13 @@
 import json
+import os
 import random
 import string
 import sys
 from datetime import datetime
 from functools import partial
 from pathlib import Path
-from statistics import mean, stdev
+from statistics import StatisticsError, mean
+from statistics import stdev as stdev_
 
 from test_pydantic import TestPydantic
 from test_trafaret import TestTrafaret
@@ -99,6 +101,13 @@ def generate_case():
 THIS_DIR = Path(__file__).parent.resolve()
 
 
+def stdev(d):
+    try:
+        return stdev_(d)
+    except StatisticsError:
+        return 0
+
+
 def main():
     json_path = THIS_DIR / 'cases.json'
     if not json_path.exists():
@@ -109,14 +118,18 @@ def main():
     else:
         with json_path.open() as f:
             cases = json.load(f)
+
     if 'pydantic-only' in sys.argv:
         tests = [TestPydantic]
     else:
         tests = [TestPydantic, TestTrafaret, TestDRF]
+
+    repeats = int(os.getenv('BENCHMARK_REPEATS', '5'))
+    results = []
     for test_class in tests:
         times = []
         p = test_class.package
-        for i in range(5):
+        for i in range(repeats):
             count, pass_count = 0, 0
             start = datetime.now()
             test = test_class(False)
@@ -130,6 +143,14 @@ def main():
             print(f'{p:10} time={time:0.3f}s, success={success:0.2f}%')
             times.append(time)
         print(f'{p:10} best={min(times):0.3f}s, avg={mean(times):0.3f}s, stdev={stdev(times):0.3f}s')
+        model_count = repeats * 3 * len(cases)
+        results.append(f'{p:10} per iteration: best={min(times) / model_count * 1e6:0.3f}μs, '
+                       f'avg={mean(times) / model_count * 1e6:0.3f}μs, '
+                       f'stdev={stdev(times) / model_count * 1e6:0.3f}μs')
+        print()
+
+    for r in results:
+        print(r)
 
 
 if __name__ == '__main__':