# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Masterthesis
#     language: python
#     name: myenv
# ---

# +
import numpy as np
np.random.seed(1)

import scipy

import time

# grAdapt
import grAdapt
from grAdapt.space.datatype import Float
from grAdapt.models import Sequential
from grAdapt.surrogate import NoGradient, NoModel, GPRSlidingWindow
from grAdapt.optimizer import AMSGradBisection, AMSGrad

# scikit learn k Means for comparison
import sklearn as sk

# plot
import matplotlib.pyplot as plt
# -

# ## 1. Introduction
#
# In clustering we want to find centroids which minimize the variance of each cluster. This can be formulated in an optimization problem by minimizing the sum of the distances of each sample to its centroid. Similar to $k$-means, we set a fixed $k$ which is the predefined number of centroids. When treating this problem as a black-box, the search for a $kd$ vector which represents the $k$ centroids in the $d$ dimensional space. Of course, this formulation projects the initial problem to a much higher dimension.
#
# ### 1.1 Creating Artificial Data Clusters

import os
plot_path = 'plots/'
if not os.path.exists(plot_path):
    os.makedirs(plot_path)

cluster1 = np.random.multivariate_normal(np.array([3, 3]), np.eye(2), size=500)
cluster2 = np.random.multivariate_normal(np.array([-3, -3]), np.eye(2), size=500)
cluster3 = np.random.multivariate_normal(np.array([-3, 3]), np.eye(2), size=500)
cluster4 = np.random.multivariate_normal(np.array([3, -3]), np.eye(2), size=500)
dataset = np.vstack([cluster1, cluster2, cluster3, cluster4])

plt.scatter(dataset[:,0], dataset[:,1], s=2)
plt.show()


# ## 2. Define Black-Box

def variance_minimization(centroids):
    # reshape (k*d, ) to (k, d)
    centroids = centroids.reshape(-1, 2)
    
    # closest point
    dists = grAdapt.utils.math.spatial.pairwise_distances(dataset, centroids)
    # dists = scipy.spatial.distance.cdist(dataset, centroids)
    
    # assign
    j_opt = np.sum(np.amin(dists, axis=1))
    
    return j_opt


# ## 3. Creating grAdapt instance

# +
# assuming we have 4 clusters
# each of the four clusters has x and y-values
# they are contrained in the following:

upper1, upper2 = np.amax(dataset, axis=0)
lower1, lower2 = np.amin(dataset, axis=0)

firstCentroid1 = Float(lower1, upper1)
firstCentroid2 = Float(lower2, upper2)

secondCentroid1 = Float(lower1, upper1)
secondCentroid2 = Float(lower2, upper2)

thirdCentroid1 = Float(lower1, upper1)
thirdCentroid2 = Float(lower2, upper2)

forthCentroid1 = Float(lower1, upper1)
forthCentroid2 = Float(lower2, upper2)

bounds = [firstCentroid1, firstCentroid2, secondCentroid1, secondCentroid2,
         thirdCentroid1, thirdCentroid2, forthCentroid1, forthCentroid2]
# -

# Try different surrogates

surrogate = GPRSlidingWindow()
model = Sequential(surrogate=surrogate, random_state=1)
n_evals = 1000
res_gpr = model.minimize(variance_minimization, bounds, n_evals)

surrogate = NoModel()
model = Sequential(surrogate=surrogate, random_state=1)
n_evals = 1000
res_nomodel = model.minimize(variance_minimization, bounds, n_evals)

surrogate = NoModel()
optimizer = AMSGrad(surrogate=surrogate)
model = Sequential(surrogate=surrogate, optimizer=optimizer, random_state=1)
n_evals = 1000
res_nomodel_amsgrad = model.minimize(variance_minimization, bounds, n_evals)

surrogate = NoGradient()
model = Sequential(surrogate=surrogate, random_state=1)
n_evals = 1000
res_nogradient = model.minimize(variance_minimization, bounds, n_evals)


# ### 3.1 Plot solution with grAdapt

def plot_grAdapt_solution(centroids, filename='grAdapt_solution.pdf'):
    #plt.figure(figsize=(7,7))
    plt.scatter(dataset[:,0], dataset[:,1], s=1, c='blue', label='Data points')
    plt.scatter(centroids[:,0], centroids[:,1], c='red', label='Centroids')
    #plt.title('grAdapt solution')
    plt.legend(loc='upper right', fontsize='medium')
    
    #plt.xticks(fontsize=20)
    #plt.yticks(fontsize=20)
    
    plot_filepath = plot_path+filename
    plt.savefig(plot_filepath, bbox_inches='tight')
    plt.show()


centroids_grAdapt = res_gpr['x_sol'].reshape(-1, 2)
plot_grAdapt_solution(centroids_grAdapt, 'grAdapt_solution_gpr.pdf')

centroids_grAdapt = res_nomodel['x_sol'].reshape(-1, 2)
plot_grAdapt_solution(centroids_grAdapt, 'grAdapt_solution_nomodel.pdf')

centroids_grAdapt = res_nomodel['x_sol'].reshape(-1, 2)
plot_grAdapt_solution(centroids_grAdapt, 'grAdapt_solution_nomodel_amsgrad.pdf')

centroids_grAdapt = res_nogradient['x_sol'].reshape(-1, 2)
plot_grAdapt_solution(centroids_grAdapt, 'grAdapt_solution_nogradient.pdf')


# ### 3.2 Plot Training Loss

def plot_grAdapt_loss(res):
    plt.figure(figsize=(4,3))
    plt.plot(res['y'], linewidth=1)
    plt.yscale('log')
    plt.xlabel('Iteration')
    plt.ylabel('Loss (log scale)')


plot_grAdapt_loss(res_gpr)

plot_grAdapt_loss(res_nomodel)

plot_grAdapt_loss(res_nomodel_amsgrad)

plot_grAdapt_loss(res_nogradient)

print(res_gpr['y_sol'])
print(res_nomodel['y_sol'])
print(res_nomodel_amsgrad['y_sol'])
print(res_nogradient['y_sol'])

# ## 4. BFGS (scipy)

import scipy

x0 = np.random.choice(len(dataset), 4)
x0 = dataset[x0].flatten()
res_scipy = scipy.optimize.minimize(variance_minimization, x0)

centroids_scipy = res_scipy.x.reshape(-1, 2)

# ### 4.1 Plot solution with scipy

plt.figure(figsize=(4,3))
plt.scatter(dataset[:,0], dataset[:,1], s=1, c='blue', label='Datapoints')
plt.scatter(centroids_scipy[:,0], centroids_scipy[:,1], c='red', label='Centroids')
plt.title('BFGS solution')
plt.legend(loc='upper right')
plt.show()

res_scipy

# ## 5. k-Means

from sklearn.cluster import KMeans

kmeans = KMeans(n_clusters=4, random_state=1).fit(dataset)

centroids_kmeans = kmeans.cluster_centers_

loss_kmeans = variance_minimization(centroids_kmeans.flatten())

# ### 5.1 Plot solution with k-Means

plot_grAdapt_solution(centroids_kmeans, 'kmeans_solution.pdf')


# ### 5.2 Plot Loss with k-Means solution

# +
def plot_grAdapt_kMeans():
    #plt.figure(figsize=(4,3))
    #plt.figure(figsize=())
    #plt.figure(figsize=(7.5, 4.5))
    #plt.title('Loss of variance minimization')
    plt.plot(res_gpr['y'], label='grAdapt (GPR)')
    plt.plot(res_nomodel['y'], label='grAdapt (No Model)')
    #plt.plot(res_nomodel_amsgrad['y'], label='grAdapt (No Model, AMSGrad)')
    plt.plot(res_nogradient['y'], label='grAdapt (No Gradient)')
    plt.plot(np.arange(len(res_gpr['y'])), loss_kmeans * np.ones((len(res_gpr['y']),)), label='k-means base solution')
    plt.legend()
    plt.xlabel('Iteration')
    plt.ylabel('Loss (log scale)')
    plt.yscale('log')

    plot_filepath = plot_path+'loss_variance_minimization.pdf'
    plt.savefig(plot_filepath, bbox_inches='tight')
    plt.show()

def plot_grAdapt_kMeans2():
    #plt.figure(figsize=(4,3))
    #plt.figure(figsize=())
    #plt.figure(figsize=(7.5, 4.5))
    #plt.title('Loss of variance minimization')
    plt.plot(res_gpr['y'], label='grAdapt (GPR)')
    #plt.plot(res_nomodel['y'], label='grAdapt (No Model)')
    plt.plot(res_nomodel_amsgrad['y'], label='grAdapt (No Model, AMSGrad)')
    plt.plot(res_nogradient['y'], label='grAdapt (No Gradient)')
    plt.plot(np.arange(len(res_gpr['y'])), loss_kmeans * np.ones((len(res_gpr['y']),)), label='k-means base solution')
    plt.legend()
    plt.xlabel('Iteration')
    plt.ylabel('Loss (log scale)')
    plt.yscale('log')

    plot_filepath = plot_path+'loss_variance_minimization_amsgrad.pdf'
    plt.savefig(plot_filepath, bbox_inches='tight')
    plt.show()


# -

plot_grAdapt_kMeans()

plot_grAdapt_kMeans2()

# +
#plt.figure(figsize=(4,3))
#plt.figure(figsize=())
plt.figure(figsize=(7.5, 4.5))
plt.title('Loss of variance minimization')
plt.plot(res_gpr['y'], label='grAdapt (GPR): Training loss')
plt.scatter(np.arange(len(res_gpr['y'])), res_nomodel['y'], s=0.05, label='grAdapt (No Model): Training loss')
plt.plot(res_nogradient['y'], label='grAdapt (No Gradient): Training loss')
plt.plot(np.arange(len(res_gpr['y'])), loss_kmeans * np.ones((len(res_gpr['y']),)), label='k-means base solution')
plt.legend()
plt.xlabel('Iteration')
plt.ylabel('Loss (log scale)')
#plt.yscale('log')
plt.ylim(2400, 2700)

# plot_filepath = plot_path+'loss_variance_minimization.pdf'
plt.savefig(plot_filepath, bbox_inches='tight')
plt.show()
# -

# grAdapt
print(res_gpr['y_sol'])
print(res_nomodel['y_sol'])
print(res_nomodel_amsgrad['y_sol'])
print(res_nogradient['y_sol'])

# scipy
res_scipy.fun

# kmeans
loss_kmeans

res_gpr['x_sol']

res_nomodel['x_sol']

res_nomodel_amsgrad['x_sol']

res_nogradient['x_sol']

centroids_kmeans[:,0]

centroids_kmeans[:,1]



StarcoderdataJupyterScriptsDedupFiltered