Merge pull request #635 from bouthilx/hotfix/precision_cardinality

Compute cardinality for loguniform with precision

setup.py

@@ -59,6 +59,7 @@
             "legacy = orion.storage.legacy:Legacy",
         ],
         "Executor": [
+            "singleexecutor = orion.executor.single_backend:SingleExecutor",
             "joblib = orion.executor.joblib_backend:Joblib",
             "dask = orion.executor.dask_backend:Dask",
         ],

src/orion/algo/space.py

@@ -35,6 +35,7 @@
 import numpy
 from scipy.stats import distributions
 
+from orion.core.utils import float_to_digits_list
 from orion.core.utils.points import flatten_dims, regroup_dims
 
 logger = logging.getLogger(__name__)
@@ -319,7 +320,7 @@ def shape(self):
         _, _, _, size = self.prior._parse_args_rvs(
             *self._args,  # pylint:disable=protected-access
             size=self._shape,
-            **self._kwargs
+            **self._kwargs,
         )
         return size
 
@@ -470,14 +471,61 @@ def cast(self, point):
         return casted_point
 
     @staticmethod
-    def get_cardinality(shape, interval):
+    def get_cardinality(shape, interval, precision, prior_name):
         """Return the number of all the possible points based and shape and interval"""
-        return numpy.inf
+        if precision is None or prior_name not in ["loguniform", "reciprocal"]:
+            return numpy.inf
+
+        # If loguniform, compute every possible combinations based on precision
+        # for each orders of magnitude.
+
+        def format_number(number):
+            """Turn number into an array of digits, the size of the precision"""
+
+            formated_number = numpy.zeros(precision)
+            digits_list = float_to_digits_list(number)
+            lenght = min(len(digits_list), precision)
+            formated_number[:lenght] = digits_list[:lenght]
+
+            return formated_number
+
+        min_number = format_number(interval[0])
+        max_number = format_number(interval[1])
+
+        # Compute the number of orders of magnitude spanned by lower and upper bounds
+        # (if lower and upper bounds on same order of magnitude, span is equal to 1)
+        lower_order = numpy.floor(numpy.log10(numpy.abs(interval[0])))
+        upper_order = numpy.floor(numpy.log10(numpy.abs(interval[1])))
+        order_span = upper_order - lower_order + 1
+
+        # Total number of possibilities for an order of magnitude
+        full_cardinality = 9 * 10 ** (precision - 1)
+
+        def num_below(number):
+
+            return (
+                numpy.clip(number, a_min=0, a_max=9)
+                * 10 ** numpy.arange(precision - 1, -1, -1)
+            ).sum()
+
+        # Number of values out of lower bound on lowest order of magnitude
+        cardinality_below = num_below(min_number)
+        # Number of values out of upper bound on highest order of magnitude.
+        # Remove 1 to be inclusive.
+        cardinality_above = full_cardinality - num_below(max_number) - 1
+
+        # Full cardinality on all orders of magnitude, minus those out of bounds.
+        cardinality = (
+            full_cardinality * order_span - cardinality_below - cardinality_above
+        )
+        return int(cardinality) ** int(numpy.prod(shape) if shape else 1)
 
     @property
     def cardinality(self):
         """Return the number of all the possible points from Integer `Dimension`"""
-        return Real.get_cardinality(self.shape, self.interval())
+        return Real.get_cardinality(
+            self.shape, self.interval(), self.precision, self._prior_name
+        )
 
 
 class _Discrete(Dimension):

src/orion/client/__init__.py

@@ -335,7 +335,8 @@ def workon(
         producer = Producer(experiment)
 
         experiment_client = ExperimentClient(experiment, producer)
-        experiment_client.workon(function, n_workers=1, max_trials=max_trials)
+        with experiment_client.tmp_executor("singleexecutor", n_workers=1):
+            experiment_client.workon(function, n_workers=1, max_trials=max_trials)
 
     finally:
         # Restore singletons

src/orion/core/utils/__init__.py

@@ -25,6 +25,25 @@ def nesteddict():
     return defaultdict(nesteddict)
 
 
+def float_to_digits_list(number):
+    """Convert a float into a list of digits, without conserving exponant"""
+    # Get rid of scientific-format exponant
+    str_number = str(number)
+    str_number = str_number.split("e")[0]
+
+    res = [int(ele) for ele in str_number if ele.isdigit()]
+
+    # Remove trailing 0s in front
+    while len(res) > 1 and res[0] == 0:
+        res.pop(0)
+
+    # Remove training 0s at end
+    while len(res) > 1 and res[-1] == 0:
+        res.pop(-1)
+
+    return res
+
+
 def get_all_subclasses(parent):
     """Get set of subclasses recursively"""
     subclasses = set()

src/orion/core/worker/transformer.py

@@ -690,16 +690,12 @@ def shape(self):
     @property
     def cardinality(self):
         """Wrap original :class:`orion.algo.space.Dimension` capacity"""
-        if self.type == "real":
-            return Real.get_cardinality(self.shape, self.interval())
-        elif self.type == "integer":
+        # May be a discretized real, must reduce cardinality
+        if self.type == "integer":
             return Integer.get_cardinality(self.shape, self.interval())
-        elif self.type == "categorical":
-            return Categorical.get_cardinality(self.shape, self.interval())
-        elif self.type == "fidelity":
-            return Fidelity.get_cardinality(self.shape, self.interval())
-        else:
-            raise RuntimeError(f"No cardinality can be computed for type `{self.type}`")
+
+        # Else we don't care what transformation is.
+        return self.original_dimension.cardinality
 
 
 class ReshapedDimension(TransformedDimension):

src/orion/executor/single_backend.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+Executor without parallelism for debugging
+==========================================
+
+"""
+import functools
+
+from orion.executor.base import BaseExecutor
+
+
+class SingleExecutor(BaseExecutor):
+    """Single thread executor
+
+    Simple executor for debugging. No parameters.
+
+    The submitted functions are wrapped with ``functools.partial``
+    which are then executed in ``wait()``.
+
+    """
+
+    def __init__(self, n_workers=1, **config):
+        super(SingleExecutor, self).__init__(n_workers=1)
+
+    def wait(self, futures):
+        return [future() for future in futures]
+
+    def submit(self, function, *args, **kwargs):
+        return functools.partial(function, *args, **kwargs)

tests/functional/algos/test_algos.py

@@ -132,6 +132,14 @@ def test_cardinality_stop(algorithm):
     assert len(trials) == 16
     assert trials[-1].status == "completed"
 
+    discrete_space["x"] = "loguniform(0.1, 1, precision=1)"
+    exp = workon(rosenbrock, discrete_space, algorithms=algorithm, max_trials=30)
+    print(exp.space.cardinality)
+
+    trials = exp.fetch_trials()
+    assert len(trials) == 10
+    assert trials[-1].status == "completed"
+
 
 @pytest.mark.parametrize(
     "algorithm", algorithm_configs.values(), ids=list(algorithm_configs.keys())

tests/unittests/algo/test_space.py

@@ -317,6 +317,50 @@ def test_cast_array(self):
         dim = Real("yolo", "uniform", -3, 4)
         assert np.all(dim.cast(np.array(["1", "2"])) == np.array([1.0, 2.0]))
 
+    def test_basic_cardinality(self):
+        """Brute force test for a simple cardinality use case"""
+        dim = Real("yolo", "reciprocal", 0.043, 2.3, precision=2)
+        order_0012 = np.arange(43, 99 + 1)
+        order_010 = np.arange(10, 99 + 1)
+        order_23 = np.arange(10, 23 + 1)
+        assert dim.cardinality == sum(map(len, [order_0012, order_010, order_23]))
+
+    @pytest.mark.parametrize(
+        "prior_name,min_bound,max_bound,precision,cardinality",
+        [
+            ("uniform", 0, 10, 2, np.inf),
+            ("reciprocal", 1e-10, 1e-2, None, np.inf),
+            ("reciprocal", 0.1, 1, 2, 90 + 1),
+            ("reciprocal", 0.1, 1.2, 2, 90 + 2 + 1),
+            ("reciprocal", 0.1, 1.25, 2, 90 + 2 + 1),
+            ("reciprocal", 1e-4, 1e-2, 2, 90 * 2 + 1),
+            ("reciprocal", 1e-5, 1e-2, 2, 90 + 90 * 2 + 1),
+            ("reciprocal", 5.234e-3, 1.5908e-2, 2, (90 - 52) + 15 + 1),
+            ("reciprocal", 5.234e-3, 1.5908e-2, 4, (9 * 10 ** 3 - 5234) + 1590 + 1),
+            (
+                "reciprocal",
+                5.234e-5,
+                1.5908e-2,
+                4,
+                (9 * 10 ** 3 * 3 - 5234) + 1590 + 1,
+            ),
+            ("uniform", 1e-5, 1e-2, 2, np.inf),
+            ("uniform", -3, 4, 3, np.inf),
+        ],
+    )
+    def test_cardinality(
+        self, prior_name, min_bound, max_bound, precision, cardinality
+    ):
+        """Check whether cardinality is correct"""
+        dim = Real(
+            "yolo", prior_name, min_bound, max_bound, precision=precision, shape=None
+        )
+        assert dim.cardinality == cardinality
+        dim = Real(
+            "yolo", prior_name, min_bound, max_bound, precision=precision, shape=(2, 3)
+        )
+        assert dim.cardinality == cardinality ** (2 * 3)
+
 
 class TestInteger(object):
     """Test methods of a `Integer` object."""

tests/unittests/core/test_transformer.py

@@ -1095,11 +1095,15 @@ def test_reshape(self, rspace):
     def test_cardinality(self, dim2):
         """Check cardinality of reshaped space"""
         space = Space()
-        space.register(Real("yolo0", "uniform", 0, 2, shape=(2, 2)))
+        space.register(Real("yolo0", "reciprocal", 0.1, 1, precision=1, shape=(2, 2)))
         space.register(dim2)
 
         rspace = build_required_space(space, shape_requirement="flattened")
-        assert rspace.cardinality == numpy.inf
+        assert rspace.cardinality == (10 ** (2 * 2)) * 4
+
+        space = Space()
+        space.register(Real("yolo0", "uniform", 0, 2, shape=(2, 2)))
+        space.register(dim2)
 
         rspace = build_required_space(
             space, type_requirement="integer", shape_requirement="flattened"

tests/unittests/core/test_utils.py → tests/unittests/core/utils/test_utils.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from orion.core.utils import Factory
+from orion.core.utils import Factory, float_to_digits_list
 
 
 def test_factory_subclasses_detection():
@@ -55,3 +55,21 @@ class Random(Base):
         pass
 
     assert type(MyFactory(of_type="random")) is Random
+
+
+@pytest.mark.parametrize(
+    "number,digits_list",
+    [
+        (float("inf"), []),
+        (0.0, [0]),
+        (0.00001, [1]),
+        (12.0, [1, 2]),
+        (123000.0, [1, 2, 3]),
+        (10.0001, [1, 0, 0, 0, 0, 1]),
+        (1e-50, [1]),
+        (5.32156e-3, [5, 3, 2, 1, 5, 6]),
+    ],
+)
+def test_float_to_digits_list(number, digits_list):
+    """Test that floats are correctly converted to list of digits"""
+    assert float_to_digits_list(number) == digits_list