Automate backward propagation

3 files changed, 109 insertions, 44 deletions

diff --git a/example.py b/example.py
index 4c00845..502db98 100755
--- a/example.py
+++ b/example.py
@@ -1,50 +1,72 @@
 #!/usr/bin/env python
 
 from src.scalar import Scalar
+from src.graph import Graph
 
-h = 0.0001
+# Manual Backpropagation
 
-# def one():
-#     a = Scalar(2, label='a')
-#     b = Scalar(-3, label='b')
-#     c = Scalar(10, label='c')
-#     f = Scalar(-2, label='f')
+# a = Scalar(2, label='a')
+# b = Scalar(-3, label='b')
+# c = Scalar(10, label='c')
+# f = Scalar(-2, label='f')
+#
+# d = a * b; d.label = 'd'
+# e = d + c; e.label = 'e'
+# L = e * f; L.label = 'L'
 #
-#     d = a * b; d.label = 'd'
-#     e = d + c; e.label = 'e'
-#     L = e * f; L.label = 'L'
+# print(f'L before gradient descent: {L.data}')
 #
-#     return L.data
+# L.grad = 1.0
+# e.grad = -2.0
+# f.grad = 4.0
+# d.grad = -2.0
+# c.grad = -2.0
+# a.grad = 6.0
+# b.grad = -4.0
 #
-# def two():
-#     a = Scalar(2, label='a')
-#     b = Scalar(-3, label='b')
-#     c = Scalar(10, label='c')
-#     f = Scalar(-2, label='f')
+# g = Graph(L)
 #
-#     d = a * b; d.label = 'd'
-#     d.data += h
-#     e = d + c; e.label = 'e'
-#     L = e * f; L.label = 'L'
+# for x in [a, b, c, f]:
+#     x.data += 0.01 * x.grad
 #
-#     return L.data
+# d = a * b
+# e = d + c
+# L = e * f
 #
-# print((two() - one()) / h)
+# print(f'L after gradient descent: {L.data}')
+# g.show()
 
-a = Scalar(2, label='a')
-b = Scalar(-3, label='b')
-c = Scalar(10, label='c')
-f = Scalar(-2, label='f')
+# Neuron
 
-d = a * b; d.label = 'd'
-e = d + c; e.label = 'e'
-L = e * f; L.label = 'L'
+x1 = Scalar(2, label='x1')
+x2 = Scalar(0, label='x2')
 
-L.grad = 1.0
-e.grad = -2.0
-f.grad = 4.0
-d.grad = -2.0
-c.grad = -2.0
+w1 = Scalar(-3, label='w1')
+w2 = Scalar(1, label='w2')
 
-from src.graph import Graph
-Graph(L).show()
+b = Scalar(6.7, label='b')
+
+x1w1 = x1 * w1; x1w1.label = 'x1w1'
+x2w2 = x2 * w2; x2w2.label = 'x2w2'
+
+x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1w1 + x2w2'
+
+L = x1w1x2w2 + b; L.label = 'L'
+o = L.tanh(); o.label = 'o'
+
+# o.grad = 1.0
+# L.grad = 1 - (o.data ** 2)
+# b.grad = L.grad
+# x1w1x2w2.grad = L.grad
+# x1w1.grad = x1w1x2w2.grad
+# x2w2.grad = x1w1x2w2.grad
+#
+# x1.grad = w1.data * x1w1.grad
+# w1.grad = x1.data * x1w1.grad
+#
+# x2.grad = w2.data * x2w2.grad
+# w2.grad = x2.data * x2w2.grad
+
+o.backward()
+
+Graph(o).show()
diff --git a/src/graph.py b/src/graph.py
index 6c8ab53..6bebb06 100644
--- a/src/graph.py
+++ b/src/graph.py
@@ -1,8 +1,5 @@
 import tkinter as tk
-
 from graphviz import Digraph
-import matplotlib.pyplot as plt
-import matplotlib.image as mpimg
 
 from .scalar import Scalar
 
@@ -33,7 +30,7 @@ class Graph:
             # Create a node in the graph
             self.dot.node(
                 name=uid,
-                label=f"{node.label} | {node.data} | grad: {node.grad}",
+                label=f"{node.label} | {node.data:.4f} | grad: {node.grad:.4f}",
                 shape='record'
             )
 
diff --git a/src/scalar.py b/src/scalar.py
index a67b7ae..c8c7601 100644
--- a/src/scalar.py
+++ b/src/scalar.py
@@ -1,3 +1,5 @@
+import math
+
 class Scalar:
     def __init__(self, data, _children=(), _op='', label='') -> None:
         self.label = label
@@ -7,14 +9,58 @@ class Scalar:
 
         self._prev = set(_children)
         self._op = _op
+        
+        self._backward = lambda: None
     
     def __repr__(self) -> str:
-        return f'Scalar({self.data})'
+        return f'Scalar({self.label}: {self.data})'
 
     def __add__(self, y):
-        result = self.data + y.data
-        return Scalar(result, (self, y), _op='+')
+        result = Scalar(self.data + y.data, (self, y), _op='+')
+
+        def _backward():
+            self.grad = result.grad
+            y.grad = result.grad
+
+        self._backward = _backward
+
+        return result
 
     def __mul__(self, y):
-        result = self.data * y.data
-        return Scalar(result, (self, y), _op='*')
+        result = Scalar(self.data * y.data, (self, y), _op='*')
+
+        def _backward():
+            self.grad = y.data * result.grad
+            y.grad = self.data * result.grad
+
+        self._backward = _backward
+
+        return result
+
+    def tanh(self):
+        x = self.data
+        t = (math.exp(2 * x) - 1) / (math.exp(2 * x) + 1)
+        result = Scalar(t, (self, ), 'tanh')
+
+        def _backward():
+            self.grad = (1 - (t ** 2)) * result.grad
+
+        self._backward = _backward
+
+        return result
+
+    def build_children(self):
+        result = []
+
+        result.append(self)
+        for child in self._prev:
+            result += child.build_children()
+
+        return result
+
+    def backward(self):
+        self.grad = 1.0
+        children = self.build_children()
+
+        for child in children:
+            child._backward()