#### 74 lines 2.6 KiB Python Raw Permalink Blame History

 ```import numpy as np ``` ``` ``` ```def sigmoid(x): ``` ``` return 1 / (1 + np.exp(-x)) ``` ``` ``` ```class EvolutionStrategy(): ``` ``` # fn: function that plays the game and returns the rewards, this function must take as argument another function that ``` ``` # determines whether the bird should jump or not ``` ``` # e.g. def fn shouldJump = if shouldJump(some_input): ... return reward ``` ``` # noisep: noise population, how many different noises should be tried at each step ``` ``` # sigma: standard deviation of generated noise ``` ``` # alpha: learning rate ``` ``` # layer_sizes: sizes of neural network layers, e.g. [[4, 500], [500, 1]] ``` ``` # input_size: number of inputs ``` ``` def __init__(self, fn, noisep, sigma, alpha, layer_sizes, input_size): ``` ``` self.fn = fn ``` ``` ``` ``` self.sigma = sigma ``` ``` self.noisep = noisep ``` ``` self.alpha = alpha ``` ``` self.layer_sizes = layer_sizes ``` ``` self.input_size = input_size ``` ``` ``` ``` # initialize layers randomly ``` ``` self.layers = [] ``` ``` for i, layer in enumerate(layer_sizes): ``` ``` self.layers.append(np.random.uniform(-0.1, 0.1, layer)) ``` ``` ``` ``` # forward propagation: sigmoid(xW) for every layer ``` ``` def forward(self, input): ``` ``` output = input ``` ``` for i, layer in enumerate(self.layers): ``` ``` output = sigmoid(np.dot(output, layer)) ``` ``` ``` ``` return output ``` ``` ``` ``` # train the model ``` ``` def train(self): ``` ``` N = [[] for i in range(len(self.layers))] ``` ``` R = np.zeros(self.noisep) ``` ``` ``` ``` for i in range(self.noisep): ``` ``` noisy_layers = [] ``` ``` ``` ``` for j, (layer_size, layer) in enumerate(zip(self.layer_sizes, self.layers)): ``` ``` # for each layer, generate a noise ``` ``` n = np.random.randn(*layer_size) ``` ``` N[j].append(n) ``` ``` ``` ``` # add noise to layer ``` ``` noisy_w = layer + self.sigma * n ``` ``` ``` ``` noisy_layers.append(noisy_w) ``` ``` ``` ``` # generate another network with the same parameters, but with noisy layers ``` ``` es = EvolutionStrategy(fn=self.fn, noisep=self.noisep, sigma=self.sigma, alpha=self.alpha, layer_sizes=self.layer_sizes, input_size=self.input_size) ``` ``` ``` ``` es.layers = noisy_layers ``` ``` # run a forward propagation using the noisy layer and save the reward ``` ``` R[i] = self.fn(es.forward) ``` ``` ``` ``` # normalize the rewards ``` ``` A = (R - np.mean(R)) / np.std(R) ``` ``` ``` ``` # update layers ``` ``` for n, i in zip(N, range(len(self.layers))): ``` ``` n = np.array(n) ``` ``` ``` ``` # np.dot(n.T, A) scales each noise's contribution to the update by how much reward it had received ``` ``` update = self.alpha / (self.noisep * self.sigma) * np.dot(n.T, A).T ``` ``` self.layers[i] = self.layers[i] + update ``` ``` ``` ``` return R ```
{"ft_lang_label":"__label__en","ft_lang_prob":0.5117505,"math_prob":0.9062117,"size":2400,"snap":"2023-40-2023-50","text_gpt3_token_len":673,"char_repetition_ratio":0.17988314,"word_repetition_ratio":0.010256411,"special_character_ratio":0.32458332,"punctuation_ratio":0.2309322,"nsfw_num_words":0,"has_unicode_error":false,"math_prob_llama3":0.9996511,"pos_list":[0],"im_url_duplicate_count":[null],"WARC_HEADER":"{\"WARC-Type\":\"response\",\"WARC-Date\":\"2023-10-02T04:48:11Z\",\"WARC-Record-ID\":\"<urn:uuid:3c738935-8dfc-4de7-b107-7b24e072c30b>\",\"Content-Length\":\"55804\",\"Content-Type\":\"application/http; msgtype=response\",\"WARC-Warcinfo-ID\":\"<urn:uuid:de160d6d-9bf6-4686-af05-3e7f06d56f3b>\",\"WARC-Concurrent-To\":\"<urn:uuid:b8a0701e-c742-40b6-9674-c48ce9ee1e01>\",\"WARC-IP-Address\":\"65.109.2.125\",\"WARC-Target-URI\":\"https://git.mahdi.blog/mahdi/flappy-es/src/branch/master/es.py\",\"WARC-Payload-Digest\":\"sha1:OIMVOP7YHQTM4IKBZN7DUJISKKDG5DHP\",\"WARC-Block-Digest\":\"sha1:GV7G25EQ3ZX2NAO5TA2VQVFSWYTZNOIM\",\"WARC-Identified-Payload-Type\":\"text/html\",\"warc_filename\":\"/cc_download/warc_2023/CC-MAIN-2023-40/CC-MAIN-2023-40_segments_1695233510967.73_warc_CC-MAIN-20231002033129-20231002063129-00746.warc.gz\"}"}