Skip to content

Commit

Permalink
fix metrics flaky test?
Browse files Browse the repository at this point in the history
  • Loading branch information
@thomwolf
thomwolf committed Aug 27, 2020
1 parent 0dcfd6f commit 2b45b5f
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions tests/test_metric.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def metric_compute(arg):
On base level to be pickable.
"""
process_id, preds, refs, exp_id = arg
metric = DummyMetric(num_process=2, process_id=process_id)
metric = DummyMetric(num_process=2, process_id=process_id, experiment_id=exp_id)
return metric.compute(predictions=preds, references=refs)


Expand All @@ -63,7 +63,7 @@ def metric_add_batch_and_compute(arg):
On base level to be pickable.
"""
process_id, preds, refs, exp_id = arg
metric = DummyMetric(num_process=2, process_id=process_id)
metric = DummyMetric(num_process=2, process_id=process_id, experiment_id=exp_id)
metric.add_batch(predictions=preds, references=refs)
return metric.compute()

Expand All @@ -73,7 +73,7 @@ def metric_add_and_compute(arg):
On base level to be pickable.
"""
process_id, preds, refs, exp_id = arg
metric = DummyMetric(num_process=2, process_id=process_id)
metric = DummyMetric(num_process=2, process_id=process_id, experiment_id=exp_id)
for pred, ref in zip(preds, refs):
metric.add(prediction=pred, reference=ref)
return metric.compute()
Expand Down

1 comment on commit 2b45b5f

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Show updated benchmarks!

Benchmark: benchmark_array_xd.json

metric read_batch_formatted_as_numpy after write_array2d read_batch_formatted_as_numpy after write_flattened_sequence read_batch_formatted_as_numpy after write_nested_sequence read_batch_unformated after write_array2d read_batch_unformated after write_flattened_sequence read_batch_unformated after write_nested_sequence read_col_formatted_as_numpy after write_array2d read_col_formatted_as_numpy after write_flattened_sequence read_col_formatted_as_numpy after write_nested_sequence read_col_unformated after write_array2d read_col_unformated after write_flattened_sequence read_col_unformated after write_nested_sequence read_formatted_as_numpy after write_array2d read_formatted_as_numpy after write_flattened_sequence read_formatted_as_numpy after write_nested_sequence read_unformated after write_array2d read_unformated after write_flattened_sequence read_unformated after write_nested_sequence write_array2d write_flattened_sequence write_nested_sequence
new / old (diff) 0.016927 / 0.012199 (0.004729) 0.013663 / 0.010296 (0.003367) 0.055142 / 0.038010 (0.017132) 0.032897 / 0.024485 (0.008413) 0.357890 / 0.290521 (0.067370) 0.385939 / 0.274994 (0.110946) 0.007456 / 0.007765 (-0.000309) 0.004899 / 0.003953 (0.000946) 0.008551 / 0.004306 (0.004245) 0.042778 / 0.037601 (0.005177) 0.359151 / 0.248894 (0.110257) 0.388157 / 0.305012 (0.083145) 0.147805 / 0.144461 (0.003345) 0.108787 / 0.082270 (0.026517) 0.493993 / 0.351702 (0.142290) 0.040884 / 0.047068 (-0.006184) 0.355380 / 0.252935 (0.102445) 0.374951 / 0.279693 (0.095257) 0.119836 / 0.128483 (-0.008647) 1.875375 / 1.328119 (0.547256) 2.038411 / 1.463678 (0.574733)

Benchmark: benchmark_indices_mapping.json

metric num examples select shard shuffle sort train_test_split
new / old (diff) 500000.000000 0.043562 / 1.553135 (-1.509572) 0.012176 / 3.383377 (-3.371201) 0.183363 / 3.345788 (-3.162425) 0.686615 / 3.363875 (-2.677260) 0.352855 / 3.521816 (-3.168961)

Benchmark: benchmark_iterating.json

metric read 5000 read 50000 read_batch 50000 10 read_batch 50000 100 read_batch 50000 1000 read_formatted numpy 5000 read_formatted pandas 5000 read_formatted tensorflow 5000 read_formatted torch 5000 read_formatted_batch numpy 5000 10 read_formatted_batch numpy 5000 1000 shuffled read 5000 shuffled read 50000 shuffled read_batch 50000 10 shuffled read_batch 50000 100 shuffled read_batch 50000 1000 shuffled read_formatted numpy 5000 shuffled read_formatted_batch numpy 5000 10 shuffled read_formatted_batch numpy 5000 1000
new / old (diff) 0.250899 / 0.212904 (0.037994) 2.531008 / 2.047739 (0.483270) 1.842798 / 1.592661 (0.250137) 1.825846 / 1.485061 (0.340785) 1.891875 / 1.672371 (0.219504) 6.026170 / 4.070887 (1.955283) 4.774298 / 3.518505 (1.255793) 7.541558 / 5.317192 (2.224366) 6.432967 / 4.370158 (2.062808) 0.625040 / 0.547849 (0.077191) 0.010883 / 0.009758 (0.001125) 0.286014 / 0.205946 (0.080068) 2.989466 / 2.053364 (0.936102) 19.998854 / 1.534712 (18.464141) 3.665701 / 1.454250 (2.211451) 2.276643 / 1.570691 (0.705952) 6.274614 / 4.140602 (2.134011) 2.491456 / 0.423797 (2.067659) 0.036324 / 0.007346 (0.028977)

Benchmark: benchmark_map_filter.json

metric filter map fast-tokenizer batched map identity map identity batched map no-op batched map no-op batched numpy map no-op batched pandas map no-op batched pytorch map no-op batched tensorflow num examples
new / old (diff) 2.530261 15.600854 13.350157 1.219562 0.616817 0.760866 0.528608 0.724887 1.582676 500000.000000

Please sign in to comment.