feat(ml): add deep learning implementations dir

- from my deep-learning-map repo

Author: jessicayung

algorithms/README.md

@@ -1,5 +1,7 @@
 # Algorithms
 
+Algorithms implemented in Python. Implementations of machine learning algorithms can be found in the directory `machine-learning`.
+
 Algorithms implemented:
 - Breadth-first search `bfs.py`
 - Depth-first search `dfs.py`

algorithms/machine-learning/deep-learning/README.md

@@ -0,0 +1,12 @@
+# Implementations
+
+A collection of implementations of networks, layers and evaluation metrics in Python.
+
+TODO:
+- Multi-layer Perceptron
+- CNN
+- RNN
+- LSTM
+- Backpropagation
+- RL agent
+- (GAN)

algorithms/machine-learning/deep-learning/neural-networks/__pycache__/cnn_class.cpython-36.pyc

@@ -0,0 +1,139 @@
+"""
+Adapted from Udacity's MiniFlow.
+"""
+import numpy as np
+
+
+class Node:
+    def __init__(self, inbound_nodes=[]):
+        self.inbound_nodes = inbound_nodes
+        self.value = None
+        self.outbound_nodes = []
+        self.gradients = {}
+        for node in inbound_nodes:
+            node.outbound_nodes.append(self)
+
+    def forward(self):
+        """
+        Every node that uses this class as a base class will
+        need to define its own 'forward' method.
+        :return:
+        """
+        raise NotImplementedError
+
+    def backward(self):
+        """
+        Every node that uses this class as a base class will
+        need to define its own 'forward' method.
+        :return:
+        """
+        raise NotImplementedError
+
+
+class Input(Node):
+    """An input to the network."""
+    def __init__(self):
+        Node.__init__(self)
+
+    def forward(self):
+        # Do nothing because nothing is calculated
+        pass
+
+    def backward(self):
+        # TODO: (don't get this) Input node has no inputs, so gradient of node is zero
+        # I get it if it's init at zero. but if it's got inputs, still init at zero?
+        self.gradients = {self: 0}
+        # Sum gradient from output nodes
+        for n in self.outbound_nodes:
+            self.gradients[self] += n.gradients[self]
+
+
+class Linear(Node):
+    """Node that performs a linear transform."""
+    def __init__(self, X, W, b):
+        Node.__init__(self, [X, W, b])
+
+    def forward(self):
+        X = self.inbound_nodes[0].value
+        W = self.inbound_nodes[1].value
+        b = self.inbound_nodes[2].value
+
+        # output: each column is a unit, each row is an example
+        self.value = np.dot(X, W) + b
+
+    def backward(self):
+        # init partial derivative for each inbound node
+        self.gradients = {n: np.zeros_like(n.value) for n in self.inbound_nodes}
+        # sum gradients over all outputs
+        for n in self.outbound_nodes:
+            # df/dthis
+            grad_cost = n.gradients[self]
+
+            # X:
+            # take coeff (W) and mult with grad cost in a way that preserves correct dims
+            self.gradients[self.inbound_nodes[0]] = np.dot(grad_cost, self.inbound_nodes[1].value.T)
+            # W
+            self.gradients[self.inbound_nodes[0]] = np.dot(self.inbound_nodes[0].value.T, grad_cost)
+            # b
+            self.gradients[self.inbound_nodes[2]] += np.sum(grad_cost, axis=0, keepdims=False) # sum each column, i.e. sum across examples
+
+
+class Sigmoid(Node):
+    """Node that performs sigmoid activation function."""
+    def __init__(self, node):
+        Node.__init__(self, [node])
+
+    def _sigmoid(self, x):
+        """
+        TODO: tbh can combine with forward method
+        :param x: numpy array-like object
+        :return: elementwise (TODO: check?) sigmoid of x
+        """
+        return 1. / (1. + np.exp(-x))
+
+    def forward(self):
+        input_value = self.inbound_nodes[0].value
+        self.value = self._sigmoid(input_value)
+
+    def backward(self):
+        self.gradients = {n: np.zeros_like(n.value) for n in self.inbound_nodes}
+        # sum gradients over all outputs
+        for n in self.outbound_nodes:
+            grad_cost = n.gradients[self]
+            sigmoid = self.value
+            self.gradients[self.inbound_nodes[0]] += grad_cost * sigmoid * (1-sigmoid)
+
+class Layer:
+    def __init__(self, units=0, W=None, activation="relu", n_inputs=0):
+        if W is not None:
+            self.units = len(W)
+            self.n_inputs = len(W[0])
+            self.W = W
+        else:
+            # TODO: make this more robust
+            self.n_inputs = n_inputs
+            self.units = units
+            self.W = np.random.randn(self.units,self.n_inputs+1)
+        self.W = W
+        self.activation = activation
+
+    def forward_pass(self,x):
+        prod = np.matmul(self.W,x)
+        return relu(prod)
+
+    def backward_pass(self, grad):
+        self.dW = grad.dot(x.T)
+        self.dx = W.T.dot(grad)
+
+def relu(array):
+    return [max(0, x) for x in array]
+
+
+W = [[1,2,3],[4,5,6],[7,8,9]]
+x = [1,1,-1]
+
+h1 = Layer(W)
+print(h1.forward_pass(x))
+
+
+

algorithms/machine-learning/deep-learning/neural-networks/__pycache__/data.cpython-36.pyc

@@ -0,0 +1,85 @@
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+def relu(X):
+    return [[max(0, x) for x in example] for example in X]
+
+
+def softmax(X):
+    # TODO: optimise
+    probs = []
+    for example in X:
+        exps = [np.exp(x) for x in example]
+        den = sum(exps)
+        probs.append(exps/den)
+    return probs
+
+
+def cross_entropy_loss(pred, true):
+    if len(pred) != len(true):
+        # TODO: rewrite exception
+        raise Exception("Number of predictions different from number of labelled examples.")
+    n_examples = len(pred)
+    mean_loss = 0
+    for i in range(n_examples):
+        yh = pred[i]
+        yt = true[i]
+        loss = sum(-yt*np.log(yh))
+        mean_loss += loss/n_examples
+    return mean_loss
+
+
+# Generate data
+np.random.seed(42)
+num_examples = 20
+input_length = 3
+int_range = 10
+X = np.random.randint(int_range, size=[num_examples, input_length])
+y = [x[0]+x[2]**2 for x in X]
+y = [max(np.ceil(el/25), 4) - 1 for el in y]
+print(X[:5])
+print(y[:5])
+# X, y = sklearn.datasets.make_moons(200, noise=0.20)
+# plt.scatter(X[:,0], X[:,1], s=40, c=y)
+
+# Set up an MLP with two hidden layers, 5 neurons in each layer
+h1_units = 5
+h2_units = 5
+output_classes = 4
+
+# Initialise weights
+stdev = 0.001
+W1 = np.random.randn(input_length, h1_units) * stdev
+b1 = np.random.randn(h1_units) * stdev
+# W2 = np.random.randn(h1_units, h2_units) * stdev
+# b2 = np.random.randn(h2_units) * stdev
+Wo = np.random.randn(h1_units, output_classes) * stdev
+bo = np.random.randn(output_classes) * stdev
+
+# Forward prop
+h1_mul = np.dot(X, W1) + b1
+print("h1 shape: ", h1.shape)
+# print("h1: ", h1)
+h1 = relu(h1_mul)
+# h2 = np.dot(h1, W2) + b2
+# print("h2: ", h2)
+# h2 = relu(h2)
+output = np.dot(h1, Wo) + bo
+pred = softmax(output)
+# print("output: ", output)
+print(cross_entropy_loss(pred, y))
+
+
+# Backward prop
+dpred = sum([-yt * 1.0 / yh for yt, yh in zip(y.ravel(), pred.ravel())])
+# sum_exps = sum([np.exp(sk) for sk in ...])
+# doutput expr: (sum_exps - np.exp(sj))*np.exp(sj)/(sum_exps**2)
+doutput = None # TODO: calculate, need to diff softmax
+dWo = doutput * h1
+dbo = doutput
+dh1 = doutput * Wo
+dh1_mul = dh1 * np.int64(h1 > 0)
+db1 = dh1_mul
+dW1 = X * dh1_mul
+dX = W1 * dh1_mul