| Method | Seeds | Median regret at half budget | Median regret | Geometric mean regret | Worst | Median proposal s/run |
|---|
| egobox-q1 | 10 | 0.0124 | 6.57e-08 | 5.64e-08 | 7.94e-07 | 0.78 |
| egobox-q4 | 10 | 0.0239 | 2.15e-07 | 4.73e-07 | 0.000255 | 0.76 |
| fcmaes-q4 | 10 | 0.0268 | 3.75e-07 | 5.8e-07 | 4.83e-06 | 0.03 |
| fcmaes-q1 | 10 | 0.00221 | 1.51e-06 | 1.1e-06 | 8.49e-06 | 0.04 |
| fcmaes-pass1-q4 | 10 | 0.0276 | 1.01e-05 | 8.42e-06 | 0.000116 | 0.03 |
| fcmaes-pass1-q1 | 10 | 0.0384 | 1.9e-05 | 1.52e-05 | 0.000123 | 0.04 |
| botorch-noiseless-q1 | 10 | 0.0208 | 5.99e-05 | 6.87e-05 | 0.00099 | 33.44 |
| optuna-gp-q1 | 10 | 0.0345 | 7.69e-05 | 8.44e-05 | 0.000276 | 6.13 |
| botorch-q1 | 10 | 0.04 | 0.000273 | 0.000217 | 0.00251 | 32.91 |
| skopt-q1 | 10 | 0.0111 | 0.000514 | 0.000452 | 0.00965 | 14.39 |
| botorch-q4 | 10 | 0.207 | 0.00491 | 0.00329 | 0.041 | 38.42 |
| biteopt | 10 | 0.739 | 0.132 | 0.137 | 0.581 | – |
| optuna-tpe-q1 | 10 | 0.368 | 0.147 | 0.133 | 0.383 | 0.08 |
| cmaes | 10 | 0.876 | 0.317 | 0.223 | 0.9 | – |
| random | 10 | 1.96 | 0.938 | 0.546 | 2.19 | – |
| Method | Seeds | Median regret at half budget | Median regret | Geometric mean regret | Worst | Median proposal s/run |
|---|
| fcmaes-q1 | 10 | 0.000565 | 1.48e-07 | 8.09e-08 | 2.43e-07 | 0.07 |
| fcmaes-q4 | 10 | 0.00717 | 1.64e-07 | 1.54e-07 | 7.5e-07 | 0.03 |
| fcmaes-pass1-q1 | 10 | 0.000309 | 1.16e-06 | 1.15e-06 | 5.19e-06 | 0.05 |
| fcmaes-pass1-q4 | 10 | 0.00925 | 2.88e-06 | 2.24e-06 | 7.41e-06 | 0.03 |
| optuna-gp-q1 | 10 | 0.000262 | 2.8e-05 | 2.38e-05 | 0.000155 | 5.28 |
| egobox-q1 | 10 | 0.00331 | 4.19e-05 | 4.38e-05 | 0.00023 | 0.91 |
| egobox-q4 | 10 | 0.00819 | 6.08e-05 | 7.84e-05 | 0.000926 | 1.11 |
| skopt-q1 | 10 | 0.00252 | 0.000146 | 0.000243 | 0.0082 | 13.31 |
| botorch-q1 | 10 | 0.00985 | 0.00472 | 0.00232 | 0.00959 | 70.96 |
| botorch-noiseless-q1 | 10 | 0.00802 | 0.00788 | 0.00121 | 0.00821 | 67.57 |
| botorch-q4 | 10 | 0.0322 | 0.00921 | 0.00696 | 0.0119 | 39.38 |
| optuna-tpe-q1 | 10 | 0.371 | 0.09 | 0.0811 | 0.362 | 0.07 |
| biteopt | 10 | 0.425 | 0.101 | 0.114 | 0.608 | – |
| cmaes | 10 | 0.27 | 0.128 | 0.111 | 0.485 | – |
| random | 10 | 0.638 | 0.453 | 0.393 | 0.885 | – |
| Method | Seeds | Median regret at half budget | Median regret | Geometric mean regret | Worst | Median proposal s/run |
|---|
| fcmaes-q1 | 10 | 0.000659 | 4.99e-07 | 4.3e-06 | 0.119 | 0.33 |
| fcmaes-q4 | 10 | 0.0433 | 9.09e-07 | 7.51e-06 | 0.119 | 0.20 |
| fcmaes-pass1-q1 | 10 | 0.00165 | 5.88e-05 | 0.000187 | 0.119 | 0.35 |
| fcmaes-pass1-q4 | 10 | 0.0604 | 7.71e-05 | 0.000289 | 0.119 | 0.19 |
| skopt-q1 | 10 | 0.0272 | 0.000466 | 0.000771 | 0.119 | 47.84 |
| optuna-gp-q1 | 10 | 0.0259 | 0.00122 | 0.00462 | 2.07 | 13.52 |
| botorch-noiseless-q1 | 10 | 0.0844 | 0.00288 | 0.00695 | 0.185 | 90.08 |
| botorch-q1 | 10 | 0.0884 | 0.0203 | 0.0272 | 0.188 | 114.79 |
| egobox-q1 | 10 | 0.0685 | 0.0598 | 0.00323 | 0.12 | 6.31 |
| egobox-q4 | 10 | 0.139 | 0.119 | 0.00818 | 0.126 | 11.07 |
| optuna-tpe-q1 | 10 | 0.477 | 0.157 | 0.159 | 0.348 | 0.25 |
| botorch-q4 | 10 | 0.224 | 0.193 | 0.133 | 0.23 | 172.98 |
| cmaes | 10 | 1.02 | 0.415 | 0.347 | 0.965 | – |
| biteopt | 10 | 1.29 | 0.654 | 0.56 | 0.999 | – |
| random | 10 | 1.3 | 1.1 | 1.09 | 1.9 | – |
| Method | Seeds | Median regret at half budget | Median regret | Geometric mean regret | Worst | Median proposal s/run |
|---|
| biteopt | 10 | 110 | 16.7 | 4.95 | 81.8 | – |
| fcmaes-q1 | 10 | 92.2 | 56.7 | 51.3 | 85.1 | 0.89 |
| fcmaes-q4 | 10 | 130 | 76.7 | 66.5 | 111 | 0.68 |
| optuna-tpe-q1 | 10 | 119 | 80.6 | 76 | 106 | 0.50 |
| egobox-q1 | 10 | 143 | 81.9 | 68.2 | 152 | 88.33 |
| fcmaes-pass1-q4 | 10 | 122 | 84 | 81 | 98.7 | 0.62 |
| botorch-q1 | 10 | 122 | 87.8 | 78.3 | 132 | 200.06 |
| skopt-q1 | 10 | 122 | 92 | 90.1 | 165 | 129.53 |
| fcmaes-pass1-q1 | 10 | 116 | 101 | 102 | 162 | 1.06 |
| optuna-gp-q1 | 10 | 123 | 117 | 112 | 146 | 39.19 |
| egobox-q4 | 10 | 183 | 129 | 112 | 182 | 104.78 |
| cmaes | 10 | 166 | 134 | 129 | 155 | – |
| botorch-q4 | 10 | 163 | 145 | 141 | 182 | 252.16 |
| random | 10 | 179 | 153 | 149 | 184 | – |
| Method | Seeds | Median regret at half budget | Median regret | Geometric mean regret | Worst | Median proposal s/run |
|---|
| biteopt | 10 | 7 | 6.92 | 5.3 | 7 | – |
| cmaes | 10 | 7 | 7 | 7 | 7 | – |
| fcmaes-pass1-q1 | 10 | 7 | 7 | 7 | 7 | 0.72 |
| fcmaes-pass1-q4 | 10 | 7 | 7 | 7 | 7 | 0.52 |
| fcmaes-q1 | 10 | 7 | 7 | 6.99 | 7 | 0.71 |
| fcmaes-q4 | 10 | 7 | 7 | 7 | 7 | 0.54 |
| random | 10 | 7 | 7 | 7 | 7 | – |
| egobox-q1 | 10 | 85.7 | 21.9 | 24.3 | 83.3 | 45.96 |
| skopt-q1 | 10 | 76 | 24.2 | 20 | 36.2 | 102.66 |
| egobox-q4 | 10 | 109 | 27.1 | 31.6 | 119 | 27.38 |
| optuna-tpe-q1 | 10 | 126 | 32.4 | 34.4 | 122 | 0.43 |
| Method | Seeds | Median regret at half budget | Median regret | Geometric mean regret | Worst | Median proposal s/run |
|---|
| fcmaes-q1 | 10 | 0.000397 | 9.69e-09 | 7.36e-09 | 7.7e-08 | 0.02 |
| fcmaes-pass1-q1 | 10 | 0.000404 | 3.93e-08 | 2.1e-08 | 3.87e-07 | 0.02 |
| egobox-q4 | 10 | 0.00237 | 3.03e-07 | 3.72e-07 | 4.24e-05 | 0.24 |
| fcmaes-q4 | 10 | 0.00098 | 5.61e-07 | 4.54e-07 | 0.000143 | 0.01 |
| egobox-q1 | 10 | 0.000349 | 6.64e-07 | 4.77e-07 | 6.11e-05 | 0.26 |
| optuna-gp-q1 | 10 | 0.000252 | 3.35e-06 | 2.08e-06 | 7.57e-06 | 3.13 |
| fcmaes-pass1-q4 | 10 | 0.00161 | 4.77e-06 | 2.14e-06 | 6.62e-05 | 0.01 |
| botorch-noiseless-q1 | 10 | 0.000121 | 5.07e-06 | 3.91e-06 | 2.51e-05 | 19.53 |
| skopt-q1 | 10 | 0.00347 | 1.57e-05 | 9.33e-06 | 6.6e-05 | 5.22 |
| botorch-q1 | 10 | 0.00251 | 2.45e-05 | 3.09e-05 | 0.000476 | 11.12 |
| botorch-q4 | 10 | 0.00258 | 0.000215 | 9.64e-05 | 0.00134 | 14.69 |
| optuna-tpe-q1 | 10 | 0.00433 | 0.0014 | 0.000681 | 0.00398 | 0.05 |
| cmaes | 10 | 0.00706 | 0.00181 | 0.00234 | 0.0378 | – |
| biteopt | 10 | 0.00784 | 0.00183 | 0.00166 | 0.0134 | – |
| random | 10 | 0.00862 | 0.00266 | 0.00284 | 0.0251 | – |