Merge branch 'master' into enhance/speed-up-wfg

docs/source/reference/samplers/index.rst

@@ -93,8 +93,6 @@ The :mod:`~optuna.samplers` module defines a base class for parameter sampling a
     optuna.samplers.NSGAIIISampler
     optuna.samplers.QMCSampler
     optuna.samplers.BruteForceSampler
-    optuna.samplers.IntersectionSearchSpace
-    optuna.samplers.intersection_search_space
 
 .. note::
     The following :mod:`optuna.samplers.nsgaii` module defines crossover operations used by :class:`~optuna.samplers.NSGAIISampler`.

optuna/_gp/optim_mixed.py

@@ -262,7 +262,7 @@ def local_search_mixed(
             # Parameters not changed from the beginning.
             return (best_normalized_params, best_fval)
 
-    _logger.warn("local_search_mixed: Local search did not converge.")
+    _logger.warning("local_search_mixed: Local search did not converge.")
     return (best_normalized_params, best_fval)
 
 
@@ -298,10 +298,13 @@ def optimize_acqf_mixed(
     probs = np.exp(f_vals - f_vals[max_i])
     probs[max_i] = 0.0  # We already picked the best param, so remove it from roulette.
     probs /= probs.sum()
+    n_non_zero_probs_improvement = np.count_nonzero(probs > 0.0)
     # n_additional_warmstart becomes smaller when study starts to converge.
     n_additional_warmstart = min(
-        n_local_search - len(warmstart_normalized_params_array) - 1, np.count_nonzero(probs > 0.0)
+        n_local_search - len(warmstart_normalized_params_array) - 1, n_non_zero_probs_improvement
     )
+    if n_additional_warmstart == n_non_zero_probs_improvement:
+        _logger.warning("Study already converged, so the number of local search is reduced.")
     chosen_idxs = np.array([max_i])
     if n_additional_warmstart > 0:
         additional_idxs = rng.choice(

optuna/_transform.py

@@ -5,7 +5,7 @@
 from typing import Tuple
 from typing import Union
 
-import numpy
+import numpy as np
 
 from optuna.distributions import BaseDistribution
 from optuna.distributions import CategoricalDistribution
@@ -78,21 +78,21 @@ def __init__(
         self._transform_0_1 = transform_0_1
 
     @property
-    def bounds(self) -> numpy.ndarray:
+    def bounds(self) -> np.ndarray:
         if self._transform_0_1:
-            return numpy.array([[0.0, 1.0]] * self._raw_bounds.shape[0])
+            return np.array([[0.0, 1.0]] * self._raw_bounds.shape[0])
         else:
             return self._raw_bounds
 
     @property
-    def column_to_encoded_columns(self) -> List[numpy.ndarray]:
+    def column_to_encoded_columns(self) -> List[np.ndarray]:
         return self._column_to_encoded_columns
 
     @property
-    def encoded_column_to_column(self) -> numpy.ndarray:
+    def encoded_column_to_column(self) -> np.ndarray:
         return self._encoded_column_to_column
 
-    def transform(self, params: Dict[str, Any]) -> numpy.ndarray:
+    def transform(self, params: Dict[str, Any]) -> np.ndarray:
         """Transform a parameter configuration from actual values to continuous space.
 
         Args:
@@ -104,7 +104,7 @@ def transform(self, params: Dict[str, Any]) -> numpy.ndarray:
             configuration.
 
         """
-        trans_params = numpy.zeros(self._raw_bounds.shape[0], dtype=numpy.float64)
+        trans_params = np.zeros(self._raw_bounds.shape[0], dtype=np.float64)
 
         bound_idx = 0
         for name, distribution in self._search_space.items():
@@ -130,7 +130,7 @@ def transform(self, params: Dict[str, Any]) -> numpy.ndarray:
 
         return trans_params
 
-    def untransform(self, trans_params: numpy.ndarray) -> Dict[str, Any]:
+    def untransform(self, trans_params: np.ndarray) -> Dict[str, Any]:
         """Untransform a parameter configuration from continuous space to actual values.
 
         Args:
@@ -172,25 +172,25 @@ def untransform(self, trans_params: numpy.ndarray) -> Dict[str, Any]:
 
 def _transform_search_space(
     search_space: Dict[str, BaseDistribution], transform_log: bool, transform_step: bool
-) -> Tuple[numpy.ndarray, List[numpy.ndarray], numpy.ndarray]:
+) -> Tuple[np.ndarray, List[np.ndarray], np.ndarray]:
     assert len(search_space) > 0, "Cannot transform if no distributions are given."
 
     n_bounds = sum(
         len(d.choices) if isinstance(d, CategoricalDistribution) else 1
         for d in search_space.values()
     )
 
-    bounds = numpy.empty((n_bounds, 2), dtype=numpy.float64)
-    column_to_encoded_columns: List[numpy.ndarray] = []
-    encoded_column_to_column = numpy.empty(n_bounds, dtype=numpy.int64)
+    bounds = np.empty((n_bounds, 2), dtype=np.float64)
+    column_to_encoded_columns: List[np.ndarray] = []
+    encoded_column_to_column = np.empty(n_bounds, dtype=np.int64)
 
     bound_idx = 0
     for distribution in search_space.values():
         d = distribution
         if isinstance(d, CategoricalDistribution):
             n_choices = len(d.choices)
             bounds[bound_idx : bound_idx + n_choices] = (0, 1)  # Broadcast across all choices.
-            encoded_columns = numpy.arange(bound_idx, bound_idx + n_choices)
+            encoded_columns = np.arange(bound_idx, bound_idx + n_choices)
             encoded_column_to_column[encoded_columns] = len(column_to_encoded_columns)
             column_to_encoded_columns.append(encoded_columns)
             bound_idx += n_choices
@@ -229,7 +229,7 @@ def _transform_search_space(
                 assert False, "Should not reach. Unexpected distribution."
 
             bounds[bound_idx] = bds
-            encoded_column = numpy.atleast_1d(bound_idx)
+            encoded_column = np.atleast_1d(bound_idx)
             encoded_column_to_column[encoded_column] = len(column_to_encoded_columns)
             column_to_encoded_columns.append(encoded_column)
             bound_idx += 1
@@ -277,29 +277,25 @@ def _untransform_numerical_param(
             if d.single():
                 pass
             else:
-                param = min(param, numpy.nextafter(d.high, d.high - 1))
+                param = min(param, np.nextafter(d.high, d.high - 1))
         elif d.step is not None:
             param = float(
-                numpy.clip(
-                    numpy.round((trans_param - d.low) / d.step) * d.step + d.low, d.low, d.high
-                )
+                np.clip(np.round((trans_param - d.low) / d.step) * d.step + d.low, d.low, d.high)
             )
         else:
             if d.single():
                 param = trans_param
             else:
-                param = min(trans_param, numpy.nextafter(d.high, d.high - 1))
+                param = min(trans_param, np.nextafter(d.high, d.high - 1))
     elif isinstance(d, IntDistribution):
         if d.log:
             if transform_log:
-                param = int(numpy.clip(numpy.round(math.exp(trans_param)), d.low, d.high))
+                param = int(np.clip(np.round(math.exp(trans_param)), d.low, d.high))
             else:
                 param = int(trans_param)
         else:
             param = int(
-                numpy.clip(
-                    numpy.round((trans_param - d.low) / d.step) * d.step + d.low, d.low, d.high
-                )
+                np.clip(np.round((trans_param - d.low) / d.step) * d.step + d.low, d.low, d.high)
             )
     else:
         assert False, "Should not reach. Unexpected distribution."

optuna/cli.py

@@ -6,13 +6,11 @@
 from argparse import Namespace
 import datetime
 from enum import Enum
-from importlib.machinery import SourceFileLoader
 import inspect
 import json
 import logging
 import os
 import sys
-import types
 from typing import Any
 from typing import Dict
 from typing import List
@@ -632,93 +630,6 @@ def take_action(self, parsed_args: Namespace) -> int:
         return 0
 
 
-class _StudyOptimize(_BaseCommand):
-    """Start optimization of a study. Deprecated since version 2.0.0."""
-
-    def add_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument(
-            "--n-trials",
-            type=int,
-            help="The number of trials. If this argument is not given, as many "
-            "trials run as possible.",
-        )
-        parser.add_argument(
-            "--timeout",
-            type=float,
-            help="Stop study after the given number of second(s). If this argument"
-            " is not given, as many trials run as possible.",
-        )
-        parser.add_argument(
-            "--n-jobs",
-            type=int,
-            default=1,
-            help="The number of parallel jobs. If this argument is set to -1, the "
-            "number is set to CPU counts.",
-        )
-        parser.add_argument(
-            "--study", default=None, help="This argument is deprecated. Use --study-name instead."
-        )
-        parser.add_argument(
-            "--study-name", default=None, help="The name of the study to start optimization on."
-        )
-        parser.add_argument(
-            "file",
-            help="Python script file where the objective function resides.",
-        )
-        parser.add_argument(
-            "method",
-            help="The method name of the objective function.",
-        )
-
-    def take_action(self, parsed_args: Namespace) -> int:
-        message = (
-            "The use of the `study optimize` command is deprecated. Please execute your Python "
-            "script directly instead."
-        )
-        warnings.warn(message, FutureWarning)
-
-        storage = _get_storage(parsed_args.storage, parsed_args.storage_class)
-
-        if parsed_args.study and parsed_args.study_name:
-            raise ValueError(
-                "Both `--study-name` and the deprecated `--study` was specified. "
-                "Please remove the `--study` flag."
-            )
-        elif parsed_args.study:
-            message = "The use of `--study` is deprecated. Please use `--study-name` instead."
-            warnings.warn(message, FutureWarning)
-            study = optuna.load_study(storage=storage, study_name=parsed_args.study)
-        elif parsed_args.study_name:
-            study = optuna.load_study(storage=storage, study_name=parsed_args.study_name)
-        else:
-            raise ValueError("Missing study name. Please use `--study-name`.")
-
-        # We force enabling the debug flag. As we are going to execute user codes, we want to show
-        # exception stack traces by default.
-        parsed_args.debug = True
-
-        module_name = "optuna_target_module"
-        target_module = types.ModuleType(module_name)
-        loader = SourceFileLoader(module_name, parsed_args.file)
-        loader.exec_module(target_module)
-
-        try:
-            target_method = getattr(target_module, parsed_args.method)
-        except AttributeError:
-            self.logger.error(
-                "Method {} not found in file {}.".format(parsed_args.method, parsed_args.file)
-            )
-            return 1
-
-        study.optimize(
-            target_method,
-            n_trials=parsed_args.n_trials,
-            timeout=parsed_args.timeout,
-            n_jobs=parsed_args.n_jobs,
-        )
-        return 0
-
-
 class _StorageUpgrade(_BaseCommand):
     """Upgrade the schema of an RDB storage."""
 
@@ -914,7 +825,6 @@ def take_action(self, parsed_args: Namespace) -> int:
     "trials": _Trials,
     "best-trial": _BestTrial,
     "best-trials": _BestTrials,
-    "study optimize": _StudyOptimize,
     "storage upgrade": _StorageUpgrade,
     "ask": _Ask,
     "tell": _Tell,
@@ -1012,7 +922,7 @@ def _get_parser(description: str = "") -> Tuple[ArgumentParser, Dict[str, Argume
 
 def _preprocess_argv(argv: List[str]) -> List[str]:
     # Some preprocess is necessary for argv because some subcommand includes space
-    # (e.g. optuna study optimize, optuna storage upgrade, ...).
+    # (e.g. optuna storage upgrade).
     argv = argv[1:] if len(argv) > 1 else ["help"]
 
     for i in range(len(argv)):

optuna/importance/_base.py

@@ -7,7 +7,7 @@
 from typing import Optional
 from typing import Union
 
-import numpy
+import numpy as np
 
 from optuna._transform import _SearchSpaceTransform
 from optuna.distributions import BaseDistribution
@@ -147,27 +147,27 @@ def _get_filtered_trials(
         trial
         for trial in trials
         if set(params) <= set(trial.params)
-        and numpy.isfinite(target(trial) if target is not None else cast(float, trial.value))
+        and np.isfinite(target(trial) if target is not None else cast(float, trial.value))
     ]
 
 
 def _param_importances_to_dict(
-    params: Collection[str], param_importances: Union[numpy.ndarray, float]
+    params: Collection[str], param_importances: Union[np.ndarray, float]
 ) -> Dict[str, float]:
     return {
         name: value
-        for name, value in zip(params, numpy.broadcast_to(param_importances, (len(params),)))
+        for name, value in zip(params, np.broadcast_to(param_importances, (len(params),)))
     }
 
 
-def _get_trans_params(trials: List[FrozenTrial], trans: _SearchSpaceTransform) -> numpy.ndarray:
-    return numpy.array([trans.transform(trial.params) for trial in trials])
+def _get_trans_params(trials: List[FrozenTrial], trans: _SearchSpaceTransform) -> np.ndarray:
+    return np.array([trans.transform(trial.params) for trial in trials])
 
 
 def _get_target_values(
     trials: List[FrozenTrial], target: Optional[Callable[[FrozenTrial], float]]
-) -> numpy.ndarray:
-    return numpy.array([target(trial) if target is not None else trial.value for trial in trials])
+) -> np.ndarray:
+    return np.array([target(trial) if target is not None else trial.value for trial in trials])
 
 
 def _sort_dict_by_importance(param_importances: Dict[str, float]) -> Dict[str, float]:

optuna/importance/_fanova/_evaluator.py

@@ -3,7 +3,7 @@
 from typing import List
 from typing import Optional
 
-import numpy
+import numpy as np
 
 from optuna._transform import _SearchSpaceTransform
 from optuna.importance._base import _get_distributions
@@ -113,8 +113,8 @@ def evaluate(
             non_single_distributions, transform_log=False, transform_step=False
         )
 
-        trans_params: numpy.ndarray = _get_trans_params(trials, trans)
-        target_values: numpy.ndarray = _get_target_values(trials, target)
+        trans_params: np.ndarray = _get_trans_params(trials, trans)
+        target_values: np.ndarray = _get_target_values(trials, target)
 
         evaluator = self._evaluator
         evaluator.fit(
@@ -123,11 +123,9 @@ def evaluate(
             search_spaces=trans.bounds,
             column_to_encoded_columns=trans.column_to_encoded_columns,
         )
-        param_importances = numpy.array(
+        param_importances = np.array(
             [evaluator.get_importance(i)[0] for i in range(len(non_single_distributions))]
         )
-        # We normalize here to keep the backward compatibility.
-        param_importances /= numpy.sum(param_importances)
 
         return _sort_dict_by_importance(
             {