diff --git a/.cache/v/cache/lastfailed b/.cache/v/cache/lastfailed new file mode 100644 index 0000000..03723a1 --- /dev/null +++ b/.cache/v/cache/lastfailed @@ -0,0 +1,9 @@ +{ + "tests/test_adaptive_linear.py": true, + "tests/test_cppn.py": true, + "tests/test_maze.py": true, + "tests/test_recurrent.py": true, + "tests/test_strict_t_maze.py": true, + "tests/test_t_maze.py": true, + "tests/test_turning_t_maze.py": true +} \ No newline at end of file diff --git a/examples/es-hyperneat/main.py b/examples/es-hyperneat/main.py new file mode 100644 index 0000000..655103c --- /dev/null +++ b/examples/es-hyperneat/main.py @@ -0,0 +1,93 @@ +import multiprocessing +import os + +import click +import neat +import gym +# import torch +import numpy as np +import tensorflow as tf + +from tf_neat import t_maze +from tf_neat.activations import tanh_activation +from tf_neat.adaptive_linear_net import AdaptiveLinearNet +from tf_neat.multi_env_eval import MultiEnvEvaluator +from tf_neat.neat_reporter import LogReporter +from tf_neat.es_hyperneat import ESNetwork +from tf_neat.substrate import Substrate +from tf_neat.cppn import create_cppn + + +max_env_steps = 200 + + +def make_env(): + return gym.make("CartPole-v0") + +def make_net(genome, config, bs): + #start by setting up a substrate for this bad cartpole boi + params = {"initial_depth": 2, + "max_depth": 4, + "variance_threshold": 0.00013, + "band_threshold": 0.00013, + "iteration_level": 3, + "division_threshold": 0.00013, + "max_weight": 3.0, + "activation": "tanh"} + input_cords = [] + output_cords = [(0.0, -1.0, 0.0)] + sign = 1 + # we will use a 3 dimensional substrate, coords laid out here + for i in range(3): + input_cords.append((0.0 - i/10*sign, 1.0, 0.0)) + sign *= -1 + leaf_names = [] + for i in range(3): + leaf_names.append('leaf_one_'+str(i)) + leaf_names.append('leaf_two_'+str(i)) + + [cppn] = create_cppn(genome, config, leaf_names, ['cppn_out']) + net_builder = ESNetwork(Substrate(input_cords, output_cords), cppn, params) + net = net_builder.create_phenotype_network_nd('./genome_vis') + return net + +def activate_net(net, states): + outputs = net.activate(states).numpy() + return outputs[0] > 0.5 + + +@click.command() +@click.option("--n_generations", type=int, default=100) +def run(n_generations): + # Load the config file, which is assumed to live in + # the same directory as this script. + config_path = os.path.join(os.path.dirname(__file__), "neat.cfg") + config = neat.Config( + neat.DefaultGenome, + neat.DefaultReproduction, + neat.DefaultSpeciesSet, + neat.DefaultStagnation, + config_path, + ) + + evaluator = MultiEnvEvaluator( + make_net, activate_net, make_env=make_env, max_env_steps=max_env_steps + ) + + def eval_genomes(genomes, config): + for _, genome in genomes: + genome.fitness = evaluator.eval_genome(genome, config) + + pop = neat.Population(config) + stats = neat.StatisticsReporter() + pop.add_reporter(stats) + reporter = neat.StdOutReporter(True) + pop.add_reporter(reporter) + #logger = LogReporter("neat.log", evaluator.eval_genome) + #pop.add_reporter(logger) + + pop.run(eval_genomes, n_generations) + + +if __name__ == "__main__": + run() # pylint: disable=no-value-for-parameter diff --git a/examples/es-hyperneat/neat.cfg b/examples/es-hyperneat/neat.cfg new file mode 100644 index 0000000..45d8ea1 --- /dev/null +++ b/examples/es-hyperneat/neat.cfg @@ -0,0 +1,61 @@ +# The `NEAT` section specifies parameters particular to the NEAT algorithm +# or the experiment itself. This is the only required section. +[NEAT] +fitness_criterion = max +fitness_threshold = 200 +pop_size = 250 +reset_on_extinction = 0 + +[DefaultGenome] +num_inputs = 6 +num_hidden = 1 +num_outputs = 1 +initial_connection = partial_direct 0.5 +feed_forward = True +compatibility_disjoint_coefficient = 1.0 +compatibility_weight_coefficient = 0.6 +conn_add_prob = 0.2 +conn_delete_prob = 0.2 +node_add_prob = 0.2 +node_delete_prob = 0.2 +activation_default = sigmoid +activation_options = sigmoid abs gauss sin identity +activation_mutate_rate = 0.0 +aggregation_default = sum +aggregation_options = sum +aggregation_mutate_rate = 0.0 +bias_init_mean = 0.0 +bias_init_stdev = 1.0 +bias_replace_rate = 0.1 +bias_mutate_rate = 0.7 +bias_mutate_power = 0.5 +bias_max_value = 30.0 +bias_min_value = -30.0 +response_init_mean = 1.0 +response_init_stdev = 0.0 +response_replace_rate = 0.0 +response_mutate_rate = 0.0 +response_mutate_power = 0.0 +response_max_value = 30.0 +response_min_value = -30.0 + +weight_max_value = 30 +weight_min_value = -30 +weight_init_mean = 0.0 +weight_init_stdev = 1.0 +weight_mutate_rate = 0.8 +weight_replace_rate = 0.1 +weight_mutate_power = 0.5 +enabled_default = True +enabled_mutate_rate = 0.01 + +[DefaultSpeciesSet] +compatibility_threshold = 3.0 + +[DefaultStagnation] +species_fitness_func = max +max_stagnation = 20 + +[DefaultReproduction] +elitism = 2 +survival_threshold = 0.2 diff --git a/tests/test_nd_get_coords.py b/tests/test_nd_get_coords.py new file mode 100644 index 0000000..c7e214d --- /dev/null +++ b/tests/test_nd_get_coords.py @@ -0,0 +1,26 @@ +import torch +from tf_neat.cppn import get_nd_coord_inputs, get_coord_inputs, create_cppn + +def test_coords(): + input_coords = [[-1.0, 0.0, 1.0], [0.0, 0.0, 1.0], [1.0, 0.0, 1.0], [-1.0, 0.0, 1.0]] + output_coords = [[-2.0, 0.0, 2.0], [0.0, 0.0, -2.0], [2.0, 0.0, 1.0]] + + input_coords_2d = [[-1.0, 0.0], [0.0, 0.0], [1.0, 0.0], [0.0, -1.0]] + output_coords_2d = [[-1.0, 0.0], [0.0, 0.0], [1.0, 0.0]] + + inputs = torch.tensor( + input_coords, dtype=torch.float32 + ) + outputs = torch.tensor( + output_coords, dtype=torch.float32 + ) + inputs_2 = torch.tensor( + input_coords_2d, dtype=torch.float32 + ) + outputs_2 = torch.tensor( + output_coords_2d, dtype=torch.float32 + ) + print(get_coord_inputs(inputs_2, outputs_2)) + print(get_nd_coord_inputs(inputs, outputs)) + +test_coords() \ No newline at end of file diff --git a/tf_neat/adaptive_example.py b/tf_neat/adaptive_example.py new file mode 100644 index 0000000..8f03e4b --- /dev/null +++ b/tf_neat/adaptive_example.py @@ -0,0 +1,130 @@ +# Copyright (c) 2018 Uber Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing +import os + +import click +import neat + +# import torch +import numpy as np +import tensorflow as tf + +import t_maze +from activations import tanh_activation +from adaptive_linear_net import AdaptiveLinearNet +from multi_env_eval import MultiEnvEvaluator +from neat_reporter import LogReporter + +# Activate eager TensorFlow execution +tf.enable_eager_execution() +print("Executing eagerly: ", tf.executing_eagerly()) + +batch_size = 4 +DEBUG = False + + +def make_net(genome, config, _batch_size): + input_coords = [[-1.0, 0.0], [0.0, 0.0], [1.0, 0.0], [0.0, -1.0]] + output_coords = [[-1.0, 0.0], [0.0, 0.0], [1.0, 0.0]] + return AdaptiveLinearNet.create( + genome, + config, + input_coords=input_coords, + output_coords=output_coords, + weight_threshold=0.4, + batch_size=batch_size, + activation=tanh_activation, + output_activation=tanh_activation, + device="cpu", + ) + + +def activate_net(net, states, debug=False, step_num=0): + if debug and step_num == 1: + print("\n" + "=" * 20 + " DEBUG " + "=" * 20) + print(net.delta_w_node) + print("W init: ", net.input_to_output[0]) + outputs = net.activate(states).numpy() + if debug and (step_num - 1) % 100 == 0: + print("\nStep {}".format(step_num - 1)) + print("Outputs: ", outputs[0]) + print("Delta W: ", net.delta_w[0]) + print("W: ", net.input_to_output[0]) + return np.argmax(outputs, axis=1) + + +@click.command() +@click.option("--n_generations", type=int, default=10000) +@click.option("--n_processes", type=int, default=1) +def run(n_generations, n_processes): + # Load the config file, which is assumed to live in + # the same directory as this script. + config_path = os.path.join(os.path.dirname(__file__), "neat.cfg") + config = neat.Config( + neat.DefaultGenome, + neat.DefaultReproduction, + neat.DefaultSpeciesSet, + neat.DefaultStagnation, + config_path, + ) + + envs = [t_maze.TMazeEnv(init_reward_side=i, n_trials=100) for i in [1, 0, 1, 0]] + + evaluator = MultiEnvEvaluator( + make_net, activate_net, envs=envs, batch_size=batch_size, max_env_steps=1000 + ) + + if n_processes > 1: + pool = multiprocessing.Pool(processes=n_processes) + + def eval_genomes(genomes, config): + fitnesses = pool.starmap( + evaluator.eval_genome, ((genome, config) for _, genome in genomes) + ) + for (_, genome), fitness in zip(genomes, fitnesses): + genome.fitness = fitness + + else: + + def eval_genomes(genomes, config): + for i, (_, genome) in enumerate(genomes): + try: + genome.fitness = evaluator.eval_genome( + genome, config, debug=DEBUG and i % 100 == 0 + ) + except Exception as e: + print(genome) + raise e + + pop = neat.Population(config) + stats = neat.StatisticsReporter() + pop.add_reporter(stats) + reporter = neat.StdOutReporter(True) + pop.add_reporter(reporter) + logger = LogReporter("./logs/adaptive.json", evaluator.eval_genome) + pop.add_reporter(logger) + + winner = pop.run(eval_genomes, n_generations) + + print(winner) + final_performance = evaluator.eval_genome(winner, config) + print("Final performance: {}".format(final_performance)) + generations = reporter.generation + 1 + return generations + + +if __name__ == "__main__": + run() # pylint: disable=no-value-for-parameter diff --git a/tf_neat/adaptive_linear_net.py b/tf_neat/adaptive_linear_net.py index 9c7b185..559fb12 100755 --- a/tf_neat/adaptive_linear_net.py +++ b/tf_neat/adaptive_linear_net.py @@ -17,9 +17,9 @@ import tensorflow as tf -from .activations import identity_activation, tanh_activation -from .cppn import clamp_weights_, create_cppn, get_coord_inputs -from .helpers import expand +from activations import identity_activation, tanh_activation +from cppn import clamp_weights_, create_cppn, get_coord_inputs +from helpers import expand class AdaptiveLinearNet: diff --git a/tf_neat/cppn.py b/tf_neat/cppn.py index 00b8120..e63f777 100755 --- a/tf_neat/cppn.py +++ b/tf_neat/cppn.py @@ -97,14 +97,13 @@ def get_activs(self, shape): self.activs = self.activate(xs, shape) return self.activs - def __call__(self, **inputs): + def __call__(self, inputs={}): assert self.leaves is not None assert inputs shape = list(inputs.values())[0].shape self.reset() for name in self.leaves.keys(): - assert (inputs[name].shape == shape), \ - "Wrong activs shape for leaf {}, {} != {}".format(name, inputs[name].shape, shape) + assert inputs[name].shape == shape,"Wrong activs shape for leaf {}, {} != {}".format(name, inputs[name].shape, shape) self.leaves[name].set_activs(inputs[name]) return self.get_activs(shape) @@ -267,3 +266,16 @@ def get_coord_inputs(in_coords, out_coords, batch_size=None): y_in = expand(tf.expand_dims(in_coords[:, 1], 0), (n_out, n_in)) return (x_out, y_out), (x_in, y_in) + +def get_nd_coord_inputs(in_coords, out_coords): + n_in = in_coords.shape[0] + n_out = out_coords.shape[0] + + dims = in_coords.shape[1] + + arrays = {} + for x in range(dims): + arrays[str(x) + "_out"] = expand(tf.expand_dims(out_coords[:, x], 1), (n_out, n_in)) + arrays[str(x) + "_in"] = expand(tf.expand_dims(in_coords[:, x], 0), (n_out, n_in)) + + return arrays \ No newline at end of file diff --git a/tf_neat/es_hyperneat.py b/tf_neat/es_hyperneat.py new file mode 100644 index 0000000..7c92670 --- /dev/null +++ b/tf_neat/es_hyperneat.py @@ -0,0 +1,427 @@ +import neat +import copy +import numpy as np +import itertools +from math import factorial +import tensorflow as tf +from recurrent_net import RecurrentNet + +#encodes a substrate of input and output coords with a cppn, adding +#hidden coords along the + +class ESNetwork: + + def __init__(self, substrate, cppn, params): + self.substrate = substrate + self.cppn = cppn + self.initial_depth = params["initial_depth"] + self.max_depth = params["max_depth"] + self.variance_threshold = params["variance_threshold"] + self.band_threshold = params["band_threshold"] + self.iteration_level = params["iteration_level"] + self.division_threshold = params["division_threshold"] + self.max_weight = params["max_weight"] + self.connections = set() + self.activations = 2 ** params["max_depth"] + 1 # Number of layers in the network. + activation_functions = neat.activations.ActivationFunctionSet() + self.activation = activation_functions.get(params["activation"]) + self.width = len(substrate.output_coordinates) + self.root_x = self.width/2 + self.root_y = (len(substrate.input_coordinates)/self.width)/2 + self.root_tree = nDimensionTree((0.0, 0.0, 0.0), 1.0, 1) + + # creates phenotype with n dimensions + def create_phenotype_network_nd(self, filename=None): + input_coordinates = self.substrate.input_coordinates + output_coordinates = self.substrate.output_coordinates + + input_nodes = range(len(input_coordinates)) + output_nodes = range(len(input_nodes), len(input_nodes)+len(output_coordinates)) + hidden_idx = len(input_coordinates)+len(output_coordinates) + + coordinates, indices, draw_connections, node_evals = [], [], [], [] + nodes = {} + + coordinates.extend(input_coordinates) + coordinates.extend(output_coordinates) + indices.extend(input_nodes) + indices.extend(output_nodes) + + # Map input and output coordinates to their IDs. + coords_to_id = dict(zip(coordinates, indices)) + + # Where the magic happens. + hidden_nodes, connections = self.es_hyperneat_nd() + + for cs in hidden_nodes: + coords_to_id[cs] = hidden_idx + hidden_idx += 1 + for cs, idx in coords_to_id.items(): + for c in connections: + if c.coord2 == cs: + draw_connections.append(c) + if idx in nodes: + initial = nodes[idx] + initial.append((coords_to_id[c.coord1], c.weight)) + nodes[idx] = initial + else: + nodes[idx] = [(coords_to_id[c.coord1], c.weight)] + + for idx, links in nodes.items(): + node_evals.append((idx, self.activation, sum, 0.0, 1.0, links)) + + # Visualize the network? + if filename is not None: + print(filename) + #draw_es_nd(coords_to_id, draw_connections, filename) + + return RecurrentNet.create_from_es(input_nodes, output_nodes, node_evals) + + # Create a RecurrentNetwork using the ES-HyperNEAT approach. + + # Recursively collect all weights for a given QuadPoint. + @staticmethod + def get_weights(p): + temp = [] + + def loop(pp): + if pp is not None and len(pp.cs) > 0: + if len(pp.cs) > 0: + for i in range(0, pp.num_children): + loop(pp.cs[i]) + else: + if pp is not None: + temp.append(pp.w) + loop(p) + return temp + + # Find the variance of a given QuadPoint. + def variance(self, p): + if not p: + return 0.0 + return np.var(self.get_weights(p)) + + def initialize_at_depth(depth=3): + root_coord = [] + for s in range(depth): + root_coord.append(0.0) + + root = nDimensionTree(root_coord, 1.0, 1) + return root + # my personal extension to the algo, we use our n-dimensional subdivision tree instead of a quadtree + # you will notice some extra looping to account for n-dimensional functionality but one could choose a dimension + # + def division_initialization_nd(self, coord, outgoing): + root = self.root_tree + q = [root] + while q: + p = q.pop(0) + # here we will subdivide to 2^coordlength as described above + # this allows us to search from +- midpoints on each axis of the input coord + p.divide_childrens() + for c in p.cs: + c.w = query_torch_cppn(coord, c.coord, outgoing, self.cppn, self.max_weight) + print(p.lvl) + if (p.lvl < self.initial_depth) or (p.lvl < self.max_depth and self.variance(p) > self.division_threshold): + for child in p.cs: + q.append(child) + + return root + + # this code is from the pureples repo, i have included it so that you can utilize the + # more explicit and 2d specific methods, they should run a bit faster + # Initialize the quadtree by dividing it in appropriate quads. + def division_initialization(self, coord, outgoing): + root = QuadPoint(0.0, 0.0, 1.0, 1.0) + q = [root] + while q: + p = q.pop(0) + + p.cs[0] = QuadPoint(p.x - p.width/2.0, p.y - p.width/2.0, p.width/2.0, p.lvl + 1) + p.cs[1] = QuadPoint(p.x - p.width/2.0, p.y + p.width/2.0, p.width/2.0, p.lvl + 1) + p.cs[2] = QuadPoint(p.x + p.width/2.0, p.y + p.width/2.0, p.width/2.0, p.lvl + 1) + p.cs[3] = QuadPoint(p.x + p.width/2.0, p.y - p.width/2.0, p.width/2.0, p.lvl + 1) + + for c in p.cs: + c.w = query_cppn(coord, (c.x, c.y), outgoing, self.cppn, self.max_weight) + if (p.lvl < self.initial_depth) or (p.lvl < self.max_depth and self.variance(p) > self.division_threshold): + for child in p.cs: + q.append(child) + + return root + # n-dimensional pruning and extradition + # the extension to use the nd tree here again + def prune_all_the_dimensions(self, coord, p, outgoing): + for c in p.cs: + child_array = [] + # this is a bit that is inconsistent in the original implementation + # its rather pointless to honor the initial depth in division initialization + # and ignor it here, hence the or statement and extra condition for that + if self.variance(c) > self.variance_threshold or p.lvl < self.initial_depth: + self.prune_all_the_dimensions(coord, c, outgoing) + else: + c_len = len(child_array) + sign = 1 + for i in range(len(c.coord)): + query_coord = [] + query_coord2 = [] + dimen = c.coord[i] - p.width + dimen2 = c.coord[i] + p.width + for x in range(len(coord)): + if x != i: + query_coord.append(c.coord[x]) + query_coord2.append(c.coord[x]) + else: + query_coord.append(dimen2) + query_coord2.append(dimen) + child_array.append(abs(c.w - query_torch_cppn(coord, query_coord, outgoing, self.cppn, self.max_weight))) + child_array.append(abs(c.w - query_torch_cppn(coord, query_coord2, outgoing, self.cppn, self.max_weight))) + con = None + max_val = 0.0 + cntrl = len(child_array)-1 + for new_ix in range(cntrl): + if(min(child_array[new_ix], child_array[new_ix+1]) > max_val): + max_val = min(child_array[new_ix], child_array[new_ix+1]) + if max_val > self.band_threshold: + if outgoing: + con = nd_Connection(coord, c.coord, c.w) + else: + con = nd_Connection(c.coord, coord, c.w) + if con is not None: + if not c.w == 0.0: + self.connections.add(con) + + # Determines which connections to express - high variance = more connetions. + def pruning_extraction(self, coord, p, outgoing): + for c in p.cs: + + d_left, d_right, d_top, d_bottom = None, None, None, None + + if self.variance(c) > self.variance_threshold: + self.pruning_extraction(coord, c, outgoing) + else: + d_left = abs(c.w - query_cppn(coord, (c.x - p.width, c.y), outgoing, self.cppn, self.max_weight)) + d_right = abs(c.w - query_cppn(coord, (c.x + p.width, c.y), outgoing, self.cppn, self.max_weight)) + d_top = abs(c.w - query_cppn(coord, (c.x, c.y - p.width), outgoing, self.cppn, self.max_weight)) + d_bottom = abs(c.w - query_cppn(coord, (c.x, c.y + p.width), outgoing, self.cppn, self.max_weight)) + + con = None + if max(min(d_top, d_bottom), min(d_left, d_right)) > self.band_threshold: + if outgoing: + con = Connection(coord[0], coord[1], c.x, c.y, c.w) + else: + con = Connection(c.x, c.y, coord[0], coord[1], c.w) + if con is not None: + if not c.w == 0.0 and con.y1 <= con.y2 and not (con.x1 == con.x2 and con.y1 == con.y2): + self.connections.add(con) + + # Explores the hidden nodes and their connections. + def es_hyperneat_nd(self): + inputs = self.substrate.input_coordinates + outputs = self.substrate.output_coordinates + hidden_nodes, unexplored_hidden_nodes = set(), set() + connections1, connections2, connections3 = set(), set(), set() + + for i in inputs: + root = self.division_initialization_nd(i, True) + self.prune_all_the_dimensions(i, root, True) + connections1 = connections1.union(self.connections) + for c in connections1: + hidden_nodes.add(tuple(c.coord2)) + self.connections = set() + + unexplored_hidden_nodes = copy.deepcopy(hidden_nodes) + + for i in range(self.iteration_level): + for index_coord in unexplored_hidden_nodes: + root = self.division_initialization_nd(index_coord, True) + self.prune_all_the_dimensions(index_coord, root, True) + connections2 = connections2.union(self.connections) + for c in connections2: + hidden_nodes.add(tuple(c.coord2)) + self.connections = set() + + unexplored_hidden_nodes -= hidden_nodes + + for c_index in range(len(outputs)): + root = self.division_initialization_nd(outputs[c_index], False) + self.prune_all_the_dimensions(outputs[c_index], root, False) + connections3 = connections3.union(self.connections) + self.connections = set() + connections = connections1.union(connections2.union(connections3)) + return self.clean_n_dimensional(connections) + + + def es_hyperneat(self): + inputs = self.substrate.input_coordinates + outputs = self.substrate.output_coordinates + hidden_nodes, unexplored_hidden_nodes = set(), set() + connections1, connections2, connections3 = set(), set(), set() + + for x, y in inputs: # Explore from inputs. + root = self.division_initialization((x, y), True) + self.pruning_extraction((x, y), root, True) + connections1 = connections1.union(self.connections) + for c in connections1: + hidden_nodes.add((c.x2, c.y2)) + self.connections = set() + + unexplored_hidden_nodes = copy.deepcopy(hidden_nodes) + + for i in range(self.iteration_level): # Explore from hidden. + for x, y in unexplored_hidden_nodes: + root = self.division_initialization((x, y), True) + self.pruning_extraction((x, y), root, True) + connections2 = connections2.union(self.connections) + for c in connections2: + hidden_nodes.add((c.x2, c.y2)) + self.connections = set() + + unexplored_hidden_nodes -= hidden_nodes + + for x, y in outputs: # Explore to outputs. + root = self.division_initialization((x, y), False) + self.pruning_extraction((x, y), root, False) + connections3 = connections3.union(self.connections) + self.connections = set() + + connections = connections1.union(connections2.union(connections3)) + + return self.clean_net(connections) + + # clean n dimensional net + def clean_n_dimensional(self, connections): + connect_to_inputs = set(tuple(i) for i in self.substrate.input_coordinates) + connect_to_outputs = set(tuple(i) for i in self.substrate.output_coordinates) + true_connections = set() + initial_input_connections = copy.deepcopy(connections) + initial_output_connections = copy.deepcopy(connections) + + add_happened = True + while add_happened: + add_happened = False + temp_input_connections = copy.deepcopy(initial_input_connections) + for c in temp_input_connections: + if c.coord1 in connect_to_inputs: + connect_to_inputs.add(c.coord2) + initial_input_connections.remove(c) + add_happened = True + add_happened = True + while add_happened: + add_happened = False + temp_output_connections = copy.deepcopy(initial_output_connections) + for c in temp_output_connections: + if c.coord2 in connect_to_outputs: + connect_to_outputs.add(c.coord1) + initial_output_connections.remove(c) + add_happened = True + true_nodes = connect_to_inputs.intersection(connect_to_outputs) + for c in connections: + if (c.coord1 in true_nodes) and (c.coord2 in true_nodes): + true_connections.add(c) + true_nodes -= (set(self.substrate.input_coordinates).union(set(self.substrate.output_coordinates))) + return true_nodes, true_connections + + + # Clean a net for dangling connections by intersecting paths from input nodes with paths to output. + + +# Class representing an area in the quadtree defined by a center coordinate and the distance to the edges of the area. +class QuadPoint: + + def __init__(self, x, y, width, lvl): + self.x = x + self.y = y + self.w = 0.0 + self.width = width + self.cs = [None] * 4 + self.lvl = lvl + +# +class nDimensionTree: + + def __init__(self, in_coord, width, level): + self.w = 0.0 + self.coord = in_coord + self.width = width + self.lvl = level + self.num_children = 2**len(self.coord) + self.cs = [] + self.signs = self.set_signs() + #print(self.signs) + def set_signs(self): + return list(itertools.product([1,-1], repeat=len(self.coord))) + + def divide_childrens(self): + for x in range(self.num_children): + new_coord = [] + for y in range(len(self.coord)): + new_coord.append(self.coord[y] + (self.width/(2*self.signs[x][y]))) + newby = nDimensionTree(new_coord, self.width/2, self.lvl+1) + self.cs.append(newby) + +# new tree's corresponding connection structure +class nd_Connection: + def __init__(self, coord1, coord2, weight): + if(type(coord1) == list): + coord1 = tuple(coord1) + if(type(coord2) == list): + coord2 = tuple(coord2) + self.coord1 = coord1 + self.coords = coord1 + coord2 + self.weight = weight + self.coord2 = coord2 + def __eq__(self, other): + return self.coords == other.coords + def __hash__(self): + return hash(self.coords + (self.weight,)) +# Class representing a connection from one point to another with a certain weight. +class Connection: + + def __init__(self, x1, y1, x2, y2, weight): + self.x1 = x1 + self.y1 = y1 + self.x2 = x2 + self.y2 = y2 + self.weight = weight + + # Below is needed for use in set. + def __eq__(self,other): + return self.x1, self.y1, self.x2, self.y2 == other.x1, other.y1, other.x2, other.y2 + + def __hash__(self): + return hash((self.x1, self.y1, self.x2, self.y2, self.weight)) + + +# From a given point, query the cppn for weights to all other points. This can be visualized as a connectivity pattern. +def find_pattern(cppn, coord, res=60, max_weight=5.0): + im = np.zeros((res, res)) + + for x2 in range(res): + for y2 in range(res): + + x2_scaled = -1.0 + (x2/float(res))*2.0 + y2_scaled = -1.0 + (y2/float(res))*2.0 + + i = [coord[0], coord[1], x2_scaled, y2_scaled, 1.0] + n = cppn.activate(i)[0] + + im[x2][y2] = n * max_weight + + return im + +def query_torch_cppn(coord1, coord2, outgoing, cppn, max_weight=5.0): + result = 0.0 + num_dimen = len(coord1) + master = {} + for x in range(num_dimen): + if(outgoing): + master["leaf_one_"+str(x)] = np.array(coord1[x]) + master["leaf_two_"+str(x)] = np.array(coord2[x]) + else: + master["leaf_one_"+str(x)] = np.array(coord2[x]) + master["leaf_two_"+str(x)] = np.array(coord1[x]) + activs = tf.Session().run(cppn(master)) + #print(activs) + w = activs + return w diff --git a/tf_neat/es_hyperneat_example.py b/tf_neat/es_hyperneat_example.py new file mode 100644 index 0000000..7a1354d --- /dev/null +++ b/tf_neat/es_hyperneat_example.py @@ -0,0 +1,94 @@ +import multiprocessing +import os + +import click +import neat +import gym +# import torch +import numpy as np +import tensorflow as tf + +import t_maze +from activations import tanh_activation +from adaptive_linear_net import AdaptiveLinearNet +from multi_env_eval import MultiEnvEvaluator +from neat_reporter import LogReporter +from es_hyperneat import ESNetwork +from substrate import Substrate +from cppn import create_cppn + + +max_env_steps = 200 + + +def make_env(): + return gym.make("CartPole-v0") + +def make_net(genome, config, bs): + #start by setting up a substrate for this bad cartpole boi + params = {"initial_depth": 2, + "max_depth": 2, + "variance_threshold": 0.00013, + "band_threshold": 0.00013, + "iteration_level": 3, + "division_threshold": 0.00013, + "max_weight": 3.0, + "activation": "tanh"} + input_cords = [] + output_cords = [(0.0, -1.0, 0.0)] + sign = 1 + # we will use a 3 dimensional substrate, coords laid out here + for i in range(3): + input_cords.append((0.0 - i/10*sign, 1.0, 0.0)) + sign *= -1 + leaf_names = [] + for i in range(3): + leaf_names.append('leaf_one_'+str(i)) + leaf_names.append('leaf_two_'+str(i)) + + [cppn] = create_cppn(genome, config, leaf_names, ['cppn_out']) + net_builder = ESNetwork(Substrate(input_cords, output_cords), cppn, params) + net = net_builder.create_phenotype_network_nd('./genome_vis') + return net + +def activate_net(net, states): + outputs = net.activate(states).numpy() + return outputs[0] > 0.5 + + +@click.command() +@click.option("--n_generations", type=int, default=100) +def run(n_generations): + # Load the config file, which is assumed to live in + # the same directory as this script. + config_path = os.path.join(os.path.dirname(__file__), "neat.cfg") + config = neat.Config( + neat.DefaultGenome, + neat.DefaultReproduction, + neat.DefaultSpeciesSet, + neat.DefaultStagnation, + config_path, + ) + + evaluator = MultiEnvEvaluator( + make_net, activate_net, make_env=make_env, max_env_steps=max_env_steps + ) + + def eval_genomes(genomes, config): + for _, genome in genomes: + genome.fitness = evaluator.eval_genome(genome, config) + print(genome.fitness) + + pop = neat.Population(config) + stats = neat.StatisticsReporter() + pop.add_reporter(stats) + reporter = neat.StdOutReporter(True) + pop.add_reporter(reporter) + #logger = LogReporter("neat.log", evaluator.eval_genome) + #pop.add_reporter(logger) + + pop.run(eval_genomes, n_generations) + + +if __name__ == "__main__": + run() # pylint: disable=no-value-for-parameter diff --git a/tf_neat/logs/adaptive.json b/tf_neat/logs/adaptive.json new file mode 100644 index 0000000..4911dda --- /dev/null +++ b/tf_neat/logs/adaptive.json @@ -0,0 +1,12 @@ +{"generation": 0, "fitness_avg": -39.99999999999993, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 15.942710638046265, "time_elapsed_avg": 15.942710638046265, "n_extinctions": 0} +{"generation": 1, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.110224485397339, "time_elapsed_avg": 9.526467561721802, "n_extinctions": 0} +{"generation": 2, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.1094629764556885, "time_elapsed_avg": 7.387466033299764, "n_extinctions": 0} +{"generation": 3, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.119898557662964, "time_elapsed_avg": 6.320574164390564, "n_extinctions": 0} +{"generation": 4, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.119873285293579, "time_elapsed_avg": 5.680433988571167, "n_extinctions": 0} +{"generation": 5, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.1549160480499268, "time_elapsed_avg": 5.259514331817627, "n_extinctions": 0} +{"generation": 6, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.110283851623535, "time_elapsed_avg": 4.952481406075614, "n_extinctions": 0} +{"generation": 7, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.1896393299102783, "time_elapsed_avg": 4.732126146554947, "n_extinctions": 0} +{"generation": 8, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.3800108432769775, "time_elapsed_avg": 4.581891112857395, "n_extinctions": 0} +{"generation": 9, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.2130324840545654, "time_elapsed_avg": 4.445005249977112, "n_extinctions": 0} +{"generation": 10, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.2103259563446045, "time_elapsed_avg": 3.1717667818069457, "n_extinctions": 0} +{"generation": 11, "fitness_avg": -39.999999999999915, "fitness_std": 7.105427357601002e-15, "fitness_best": -39.99999999999992, "fitness_best_val": -39.99999999999992, "n_neurons_best": 6, "n_conns_best": 21, "pop_size": 20, "n_species": 10, "time_elapsed": 3.4501054286956787, "time_elapsed_avg": 3.20575487613678, "n_extinctions": 0} diff --git a/tf_neat/neat.cfg b/tf_neat/neat.cfg new file mode 100644 index 0000000..45d8ea1 --- /dev/null +++ b/tf_neat/neat.cfg @@ -0,0 +1,61 @@ +# The `NEAT` section specifies parameters particular to the NEAT algorithm +# or the experiment itself. This is the only required section. +[NEAT] +fitness_criterion = max +fitness_threshold = 200 +pop_size = 250 +reset_on_extinction = 0 + +[DefaultGenome] +num_inputs = 6 +num_hidden = 1 +num_outputs = 1 +initial_connection = partial_direct 0.5 +feed_forward = True +compatibility_disjoint_coefficient = 1.0 +compatibility_weight_coefficient = 0.6 +conn_add_prob = 0.2 +conn_delete_prob = 0.2 +node_add_prob = 0.2 +node_delete_prob = 0.2 +activation_default = sigmoid +activation_options = sigmoid abs gauss sin identity +activation_mutate_rate = 0.0 +aggregation_default = sum +aggregation_options = sum +aggregation_mutate_rate = 0.0 +bias_init_mean = 0.0 +bias_init_stdev = 1.0 +bias_replace_rate = 0.1 +bias_mutate_rate = 0.7 +bias_mutate_power = 0.5 +bias_max_value = 30.0 +bias_min_value = -30.0 +response_init_mean = 1.0 +response_init_stdev = 0.0 +response_replace_rate = 0.0 +response_mutate_rate = 0.0 +response_mutate_power = 0.0 +response_max_value = 30.0 +response_min_value = -30.0 + +weight_max_value = 30 +weight_min_value = -30 +weight_init_mean = 0.0 +weight_init_stdev = 1.0 +weight_mutate_rate = 0.8 +weight_replace_rate = 0.1 +weight_mutate_power = 0.5 +enabled_default = True +enabled_mutate_rate = 0.01 + +[DefaultSpeciesSet] +compatibility_threshold = 3.0 + +[DefaultStagnation] +species_fitness_func = max +max_stagnation = 20 + +[DefaultReproduction] +elitism = 2 +survival_threshold = 0.2 diff --git a/tf_neat/recurrent_net.py b/tf_neat/recurrent_net.py index 5a7451d..59ee051 100755 --- a/tf_neat/recurrent_net.py +++ b/tf_neat/recurrent_net.py @@ -16,7 +16,7 @@ import numpy as np import tensorflow as tf -from .activations import sigmoid_activation +from activations import sigmoid_activation def tran(tensor): @@ -205,7 +205,67 @@ def key_to_idx(key): idxs.append((o_idx, i_idx)) # to, from vals.append(conn.weight) + @staticmethod + def create_from_es(in_nodes, out_nodes, node_evals, batch_size=1, activation=sigmoid_activation, + prune_empty=False, use_current_activs=False, n_internal_steps=1): + hidden_responses = [1.0 for k in range(len(node_evals)-(len(in_nodes)+len(out_nodes)))] + output_responses = [1.0 for k in range(len(out_nodes))] + + hidden_biases = [1.0 for k in range(len(node_evals)-(len(in_nodes)+len(out_nodes)))] + output_biases = [1.0 for k in range(len(out_nodes))] + + input_key_to_idx = {k: i for i, k in enumerate(in_nodes)} + output_key_to_idx = {k: i for i, k in enumerate(out_nodes)} + hidden_key_to_idx = {} + + hidden_idx = -1 + + def key_to_idx(key, hid_idx): + if key in in_nodes: + return input_key_to_idx[key] + elif key in out_nodes: + return output_key_to_idx[key] + elif key in hidden_key_to_idx.keys(): + return hidden_key_to_idx[key] + else: + hid_idx += 1 + hidden_key_to_idx[key] = hid_idx + return hid_idx + + input_to_hidden = ([], []) + hidden_to_hidden = ([], []) + output_to_hidden = ([], []) + input_to_output = ([], []) + hidden_to_output = ([], []) + output_to_output = ([], []) + # this could be optimized by first checking in out or hidden + # of ikey but for now this is how im doing it to keep it looking familiar + for conn in node_evals: + #pruning is done in the eshyperneat class + i_key = conn[0] + for x in conn[5]: + o_key = x[0] + i_idx = key_to_idx(i_key, hidden_idx) + o_idx = key_to_idx(o_key, hidden_idx) + add_conn = True + if i_key in in_nodes and o_key not in out_nodes: + idxs, vals = input_to_hidden + elif i_key not in in_nodes and i_key not in out_nodes and o_key not in in_nodes and o_key not in out_nodes: + idxs, vals = hidden_to_hidden + elif i_key in out_nodes and o_key not in out_nodes and o_key not in in_nodes: + idxs, vals = output_to_hidden + elif i_key in in_nodes and o_key in out_nodes: + idxs, vals = input_to_output + elif i_key not in in_nodes and i_key not in out_nodes and o_key in out_nodes: + idxs, vals = hidden_to_output + elif i_key in out_nodes and o_key in out_nodes: + idxs, vals = output_to_output + else: + add_conn = False + if add_conn == True: + idxs.append((o_idx, i_idx)) # to, from + vals.append(float(x[1])) return RecurrentNet(n_inputs, n_hidden, n_outputs, input_to_hidden, hidden_to_hidden, output_to_hidden, input_to_output, hidden_to_output, output_to_output, diff --git a/tf_neat/substrate.py b/tf_neat/substrate.py new file mode 100644 index 0000000..807ebef --- /dev/null +++ b/tf_neat/substrate.py @@ -0,0 +1,7 @@ +class Substrate(object): + + def __init__(self, input_coordinates, output_coordinates, hidden_coordinates=(), res=10.0): + self.input_coordinates = input_coordinates + self.hidden_coordinates = hidden_coordinates + self.output_coordinates = output_coordinates + self.res = res