GradientDescent { public static <T> VectorModel minimize(LossDefinition<T> lossDefinition, Collection<Observation<T>> observations, int dimensionality, double learningRate, int numRepetitions, double minStepSize) { final Random random = new Random(); VectorModel bestModel = null; double bestLoss = Double.NaN; for (int repetition = 0; repetition < numRepetitions; repetition++) { logger.info("Performing repetition {}/{}.", repetition + 1, numRepetitions); final double[] parameters = new double[dimensionality]; for (int dimension = 0; dimension < parameters.length; dimension++) { parameters[dimension] = random.nextGaussian() * 10; } final VectorModel model = new VectorModel(parameters); double stepSize; double lastLoss = Double.NaN; long nextLogMillis = System.currentTimeMillis() + 30_000L; long round = 1L; do { final double[] gradient = lossDefinition.calculateGradient(model, observations); for (int i = 0; i < gradient.length; i++) { gradient[i] *= -learningRate; } model.add(gradient); stepSize = calculateEuclidianDistance(gradient); double loss = lossDefinition.calculateLoss(model, observations); if (!Double.isNaN(lastLoss)) { if (lastLoss >= loss) learningRate *= 1.05; else learningRate *= .5; } lastLoss = loss; if (System.currentTimeMillis() >= nextLogMillis) { logger.info("Current loss after {} rounds: {}", round, loss); nextLogMillis += 30_000L; } round++; } while (stepSize > minStepSize); double loss = lossDefinition.calculateLoss(model, observations); logger.info("Final loss after {} rounds: {}", round, loss); if (bestModel == null || bestLoss < loss) { bestModel = model; bestLoss = loss; } } return bestModel; }  static VectorModel minimize(LossDefinition<T> lossDefinition,
                                           Collection<Observation<T>> observations,
                                           int dimensionality,
                                           double learningRate,
                                           int numRepetitions,
                                           double minStepSize);  }
@Test public void testFindSingleLocalMinimum() { final int numDimension = 10; GradientDescent.LossDefinition<Void> lossDefinition = new GradientDescent.LossDefinition<Void>() { @Override public double calculateLoss(VectorModel model, Collection<Observation<Void>> observations) { double loss = 0d; double[] parameters = model.getParameters(); for (int i = 0; i < parameters.length; i++) { double v = parameters[i]; loss += v * v; } return loss; } @Override public double[] calculateGradient(VectorModel model, Collection<Observation<Void>> observations) { double[] gradient = new double[numDimension]; double[] parameters = model.getParameters(); for (int i = 0; i < parameters.length; i++) { double v = parameters[i]; gradient[i] = 2 * v; } return gradient; } }; Collection<Observation<Void>> observations = Collections.emptyList(); VectorModel model = GradientDescent.minimize(lossDefinition, observations, numDimension, 0.1, 1, 0.0001); double loss = lossDefinition.calculateLoss(model, observations); double[] parameters = model.getParameters(); Assert.assertEquals(numDimension, parameters.length); Assert.assertEquals(0, loss, 0.01); for (int i = 0; i < parameters.length; i++) { double v = parameters[i]; Assert.assertEquals(0d, v, 0.01); } }
@Test public void testFindSingleLocalMinimum2() { final int numDimension = 10; GradientDescent.LossDefinition<Void> lossDefinition = new GradientDescent.LossDefinition<Void>() { @Override public double calculateLoss(VectorModel model, Collection<Observation<Void>> observations) { double loss = 0d; double[] parameters = model.getParameters(); for (int i = 0; i < parameters.length; i++) { double v = parameters[i] - i; loss += v * v; } return loss; } @Override public double[] calculateGradient(VectorModel model, Collection<Observation<Void>> observations) { double[] gradient = new double[numDimension]; double[] parameters = model.getParameters(); for (int i = 0; i < parameters.length; i++) { double v = parameters[i]; gradient[i] = 2 * (v - i); } return gradient; } }; Collection<Observation<Void>> observations = Collections.emptyList(); VectorModel model = GradientDescent.minimize(lossDefinition, observations, numDimension, 0.1, 1, 0.0001); double loss = lossDefinition.calculateLoss(model, observations); double[] parameters = model.getParameters(); Assert.assertEquals(numDimension, parameters.length); Assert.assertEquals(0, loss, 0.01); for (int i = 0; i < parameters.length; i++) { double v = parameters[i]; Assert.assertEquals(i, v, 0.01); } }