MadalineNetwork.py

# -*- coding: utf-8 -*-
"""HW1_Q2_MadaLine.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1FL6A_8CteSMYhlrYAvSzOb8on7qxsydS

# 2.2. MadaLine

## 2.2.B. Define and Plot Data
"""

import random
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

!pip install --upgrade --no-cache-dir gdown
!gdown 1NRwDhqrhXF8tRy1HKe9QofJZjPBnHV1r

df = pd.read_csv('/content/MadaLine.csv',names=['x', 'y','label'],header=None)
df

df = df.sample(frac = 1)
df

data = df[['x','y']]
inputs = data.to_numpy()
inputs.shape[0]

# Convert to Bipolar
label = df[['label']]
target = label.to_numpy()
target[np.isclose(target, 0)] = -1

df0 = df.loc[df['label']  == 0 ]
df1 = df.loc[df['label']  == 1]

# scatter plot
plt.scatter(df0['x'], df0['y'], c="red", linewidths=2)
plt.scatter(df1['x'], df1['y'], c="blue", linewidths=2)

# add axis labels and legend
plt.xlabel("x")
plt.ylabel("y")
plt.legend(["Class 1", "Class 2"])

# add grid
plt.grid(True)

# add title
plt.title("Scatter Plot")

# adjust subplot spacing
plt.tight_layout()

# save the figure as PDF
plt.savefig("scatter_plot.pdf")

# show the plot
plt.show()

df = df.sample(frac=1, random_state=5).reset_index()
df

"""## 2.2.C. MRI Implementation"""

def find_decision_boundary(start_x, end_x, weights, biases):
    """
    Calculates the decision boundary given the start and end x values, weights, and biases.
    
    Args:
    start_x (float): The start x value.
    end_x (float): The end x value.
    weights (numpy.ndarray): An array of weights.
    biases (float): The bias value.
    
    Returns:
    inputs (numpy.ndarray): An array of x values.
    output (numpy.ndarray): An array of y values representing the decision boundary.
    """
    
    # Create an array of x values
    inputs = np.linspace(start_x, end_x)
    
    # Calculate the corresponding y values using the given weights and biases
    output = -(weights[0] * inputs + biases)
    output = output / weights[1]
    
    return inputs, output


def initialize_weights(sm, num_neurons_layer1, num_neurons_layer2):
    # Set a random seed for reproducibility
    np.random.seed(10)
    
    # Generate random weights and biases for the first layer
    weights = np.random.rand(num_neurons_layer1, num_neurons_layer2) * sm
    biases = np.zeros((num_neurons_layer1, 1))
    
    # Initialize weights and biases for the second layer
    # The weights for the second layer are initialized to 1
    weights_layer2 = np.array([[1]*num_neurons_layer1])
    biases_layer2 = num_neurons_layer1 - 1
    
    # Return all the initialized weights and biases
    return weights, biases, weights_layer2, biases_layer2

def apply_activation_function(net):
    # np.where(condition, x, y) returns an array with the same shape as condition, 
    # where the elements are taken from x where condition is True, 
    # and from y elsewhere. In this case, the condition is whether each element of 
    # the input net is greater than or equal to 0. If it is, the corresponding element 
    # in the output h is set to 1; otherwise, it is set to -1.
    h = np.where(net >= 0, 1, -1)
    return h

def forward_propagation(weights, inputs, biases, should_reshape):
    # Check if inputs should be reshaped
    if should_reshape:
        inputs = inputs.reshape((2, 1))
    # Calculate the net input
    net_input = np.dot(weights, inputs) + biases
    # Apply activation function to net input to obtain outputs
    outputs = apply_activation_function(net_input)
    # Return both net input and outputs
    return net_input, outputs

def update_weights(weights, biases, inputs, target, net_input, output, learning_rate, num_neurons_layer1):
    # Reshape inputs and net_input
    inputs = inputs.reshape((1, 2))
    net_input = net_input.reshape((num_neurons_layer1, 1))

    # If target is equal to output, no weight or bias update is necessary
    if target == output:
        return weights, biases
    # If target is 1 and output is -1, update weights and biases for the neuron with the highest net input
    elif target == 1 and target != output:    #output=-1 but target=1
        argmax_neuron = np.argmax(net_input)
        diff_bias = learning_rate * (1 - net_input[argmax_neuron])
        diff_weight = learning_rate * np.dot((1 - net_input[argmax_neuron]), inputs)
        biases[argmax_neuron] = biases[argmax_neuron] + diff_bias
        weights[argmax_neuron] = weights[argmax_neuron] + diff_weight
    # If target is -1 and output is 1, update weights and biases for all neurons with positive net input
    elif target == -1 and target != output:   # output=1 but target=-1
        positive_indices = np.argwhere(net_input > 0)
        diff_bias = learning_rate * (-1 - net_input)
        diff_weight = learning_rate * np.dot((-1 - net_input), inputs)
        new_biases = biases + diff_bias
        new_weights = weights + diff_weight
        for i in positive_indices[:, 0]:
            weights[i] = new_weights[i]
            biases[i] = new_biases[i]
    # Return updated weights and biases
    return weights, biases

# Define a function named calculate_error
def calculate_error(target, output):
    # Calculate the error using the mean squared error formula
    error = 0.5 * np.power((target - output), 2)
    # Return the calculated error
    return error

def predict(inputs, target, weights, biases, num_neurons_layer1):
    # initialize an empty list to store predicted outputs
    predicted_output = []
    
    # initialize biases_layer2 as a numpy array of zeros
    biases_layer2 = np.zeros((num_neurons_layer1, 1))
    
    # initialize weights_layer2 as a numpy array of shape (1, num_neurons_layer1) with all elements as 1
    weights_layer2 = np.array([[1]*num_neurons_layer1])
    
    # update biases_layer2 to have a value of num_neurons_layer1 - 1
    biases_layer2 = num_neurons_layer1 - 1
    
    # loop through each input and calculate the predicted output using forward propagation
    for i in range(inputs.shape[0]):
        # call the forward_propagation function with inputs, weights, biases, and should_reshape=True
        net_input, outputs = forward_propagation(weights, inputs[i], biases, should_reshape=True)
        
        # call the forward_propagation function with weights_layer2, outputs, biases_layer2, and should_reshape=False
        net_input2, output = forward_propagation(weights_layer2, outputs, biases_layer2, should_reshape=False)
        
        # append the predicted output to the predicted_output list
        predicted_output.append(output[0])
    
    # return the list of predicted outputs
    return predicted_output

def MRI(df0, df1, inputs, target, num_neurons_layer1=3, num_neurons_layer2=2, learning_rate=0.0001, max_iter=200, samples=None, plot=True):
    # If samples is not provided, set it to the number of rows in the input data.
    if samples is None:
        samples = inputs.shape[0]
    # Print the number of samples.
    print('sample:', samples)
    # Initialize variables
    sm = 0.001
    error_list = []
    errors = []
    mean_error = 10**3
    weights, biases, weights_layer2, biases_layer2 = initialize_weights(sm, num_neurons_layer1, num_neurons_layer2)  # Step 0
    # Iterate through the training process
    for i in range(max_iter):
        # Perform forward propagation
        net_input, outputs = forward_propagation(weights, inputs[i % samples], biases, should_reshape=True)  # Step 4 and 5
        net_input2, output = forward_propagation(weights_layer2, outputs, biases_layer2, should_reshape=False)  # Step 6
        # Calculate the error of the output
        error = calculate_error(target[i % samples], output)
        errors.append(error)
        # If an epoch has ended, calculate the mean error of the epoch and append it to the error list
        if i % samples == 0 and i != 0:
            mean_error = np.mean(errors)
            error_list.append(mean_error)
            errors = []
            # Print the epoch number and the mean error of the epoch
            print('Epoch %d / %d' % (len(error_list), int(max_iter / samples)))
            print('loss:', mean_error)
            for j in range(len(weights)):
                # Print the weights and biases for each layer
                print('W%d:'%(j+1),weights[j])
                print('b%d:'%(j+1),biases[j])
        # If the mean error is 0 or the difference between the mean error of the last two epochs is 0, an early stop occurred
        if mean_error == 0 or (i > 50 and len(error_list) >= 2 and error_list[-1] - error_list[-2] == 0):
            print('An early stop occurred!')
            # If plot is True, plot the error, decision boundary, confusion matrix, and classification report
            if plot:
                plt.plot(error_list)
                plt.xlabel('Epochs')
                plt.ylabel('Mean Squared Error')
                plt.title('Error Plot')
                plt.grid(True)
                plt.savefig('error_plot.pdf')
                plt.show()
                df0 = df0
                df1 = df1
                plt.scatter(df0['x'], df0['y'], c ="red", linewidths = 0.1)
                plt.scatter(df1['x'], df1['y'], c ="blue", linewidths = .1)
                # Plot the decision boundary
                for i in range(num_neurons_layer1):
                    px1, px2 = find_decision_boundary(-2, 2, weights[i], biases[i])
                    plt.plot(px1, px2)

                plt.xlabel("x")
                plt.ylabel("y")
                plt.legend(["Class 1" , "Class 2"])
                plt.xlim([-2, 2])
                plt.ylim([-2, 2])
                plt.savefig('error_plot1.pdf')
                plt.show()
                # Generate predictions
                predicted_output = predict(inputs, target, weights, biases, num_neurons_layer1)
                # Create confusion matrix
                cm = confusion_matrix(target, predicted_output)
                # Plot heatmap of confusion matrix
                plt.figure(figsize=(6, 4))
                sns.heatmap(cm, annot=True, cmap="Blues", fmt="g")
                plt.xlabel("Predicted labels")
                plt.ylabel("True labels")
                plt.title("Confusion Matrix")
                plt.tight_layout()
                # Save plot as PDF
                plt.savefig("confusion_matrix.pdf")
                plt.show()
                predicted_output = predict(inputs, target,weights,biases,num_neurons_layer1)
                print(classification_report(target, predicted_output))    
            return weights, biases, error_list
        weights, b = update_weights(weights, biases, inputs[i % samples], target[i % samples], net_input, output, learning_rate, num_neurons_layer1)  # Step 7
    
    if plot:
        plt.plot(error_list)
        plt.xlabel('Epochs')
        plt.ylabel('Mean Squared Error')
        plt.title('Error Plot')
        plt.grid(True)
        plt.savefig('error_plot.pdf')
        plt.show()
        df0 = df0
        df1 = df1
        plt.scatter(df0['x'], df0['y'], c ="red", linewidths = 0.1)
        plt.scatter(df1['x'], df1['y'], c ="blue", linewidths = .1)

        for i in range(num_neurons_layer1):
            px1, px2 = find_decision_boundary(-2, 2, weights[i], biases[i])
            plt.plot(px1, px2)

        plt.xlabel("x")
        plt.ylabel("y")
        plt.legend(["Class 1" , "Class 2"])
        plt.xlim([-2, 2])
        plt.ylim([-2, 2])
        plt.savefig('error_plot1.pdf')
        plt.show()
        # Generate predictions
        predicted_output = predict(inputs, target, weights, biases, num_neurons_layer1)
        # Create confusion matrix
        cm = confusion_matrix(target, predicted_output)
        # Plot heatmap of confusion matrix
        plt.figure(figsize=(6, 4))
        sns.heatmap(cm, annot=True, cmap="Blues", fmt="g")
        plt.xlabel("Predicted labels")
        plt.ylabel("True labels")
        plt.title("Confusion Matrix")
        plt.tight_layout()
        # Save plot as PDF
        plt.savefig("confusion_matrix.pdf")
        plt.show()
        predicted_output = predict(inputs, target,weights,biases,num_neurons_layer1)
        print(classification_report(target, predicted_output))  

    return weights, biases, error_list

import random
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

def find_decision_boundary(start_x, end_x, weights, biases):
    inputs = np.linspace(start_x, end_x)
    output = -(weights[0] * inputs + biases)
    output = output / weights[1]
    return inputs, output

def initialize_weights(sm, num_neurons_layer1, num_neurons_layer2):
    np.random.seed(10)
    weights = np.random.rand(num_neurons_layer1, num_neurons_layer2) * sm
    biases = np.zeros((num_neurons_layer1, 1))
    weights_layer2 = np.array([[1]*num_neurons_layer1])
    biases_layer2 = num_neurons_layer1 - 1
    return weights, biases, weights_layer2, biases_layer2

def apply_activation_function(net):
    h = np.where(net >= 0, 1, -1)
    return h

def forward_propagation(weights, inputs, biases, should_reshape):
    if should_reshape:
        inputs = inputs.reshape((2, 1))
    net_input = np.dot(weights, inputs) + biases
    outputs = apply_activation_function(net_input)
    return net_input, outputs

def update_weights(weights, biases, inputs, target, net_input, output, learning_rate, num_neurons_layer1):
    inputs = inputs.reshape((1, 2))
    net_input = net_input.reshape((num_neurons_layer1, 1))
    if target == output:
        return weights, biases
    elif target == 1 and target != output:    #output=-1 but target=1
        argmax_neuron = np.argmax(net_input)
        diff_bias = learning_rate * (1 - net_input[argmax_neuron])
        diff_weight = learning_rate * np.dot((1 - net_input[argmax_neuron]), inputs)
        biases[argmax_neuron] = biases[argmax_neuron] + diff_bias
        weights[argmax_neuron] = weights[argmax_neuron] + diff_weight
    elif target == -1 and target != output:   # output=1 but target=-1
        positive_indices = np.argwhere(net_input > 0)
        diff_bias = learning_rate * (-1 - net_input)
        diff_weight = learning_rate * np.dot((-1 - net_input), inputs)
        new_biases = biases + diff_bias
        new_weights = weights + diff_weight
        for i in positive_indices[:, 0]:
            weights[i] = new_weights[i]
            biases[i] = new_biases[i]
    return weights, biases

def calculate_error(target, output):
    error = 0.5 * np.power((target - output), 2)
    return error

def predict(inputs, target, weights, biases, num_neurons_layer1):
    predicted_output = []
    biases_layer2 = np.zeros((num_neurons_layer1, 1))
    weights_layer2 = np.array([[1]*num_neurons_layer1])
    biases_layer2 = num_neurons_layer1 - 1
    for i in range(inputs.shape[0]):
        net_input, outputs = forward_propagation(weights, inputs[i], biases, should_reshape=True)
        net_input2, output = forward_propagation(weights_layer2, outputs, biases_layer2, should_reshape=False)
        predicted_output.append(output[0])
    return predicted_output
    

def MRI(df0, df1, inputs, target, num_neurons_layer1=3, num_neurons_layer2=2, learning_rate=0.0001, max_iter=200, samples=None, plot=True):
    if samples is None:
        samples = inputs.shape[0]
    print('sample:', samples)
    sm = 0.001
    error_list = []
    errors = []
    mean_error = 10**3
    weights, biases, weights_layer2, biases_layer2 = initialize_weights(sm, num_neurons_layer1, num_neurons_layer2)  # Step 0
    for i in range(max_iter):
        net_input, outputs = forward_propagation(weights, inputs[i % samples], biases, should_reshape=True)  # Step 4 and 5
        net_input2, output = forward_propagation(weights_layer2, outputs, biases_layer2, should_reshape=False)  # Step 6
        error = calculate_error(target[i % samples], output)
        errors.append(error)
        if i % samples == 0 and i != 0:
            mean_error = np.mean(errors)
            error_list.append(mean_error)
            errors = []
            print('Epoch %d / %d' % (len(error_list), int(max_iter / samples)))
            print('loss:', mean_error)
            for j in range(len(weights)):
                print('W%d:'%(j+1),weights[j])
                print('b%d:'%(j+1),biases[j])
        if mean_error == 0 or (i > 50 and len(error_list) >= 2 and error_list[-1] - error_list[-2] == 0):
            print('An early stop occurred!')
            if plot:
                plt.plot(error_list)
                plt.xlabel('Epochs')
                plt.ylabel('Mean Squared Error')
                plt.title('Error Plot')
                plt.grid(True)
                plt.savefig('error_plot.pdf')
                plt.show()
                df0 = df0
                df1 = df1
                plt.scatter(df0['x'], df0['y'], c ="red", linewidths = 0.1)
                plt.scatter(df1['x'], df1['y'], c ="blue", linewidths = .1)

                for i in range(num_neurons_layer1):
                    px1, px2 = find_decision_boundary(-2, 2, weights[i], biases[i])
                    plt.plot(px1, px2)

                plt.xlabel("x")
                plt.ylabel("y")
                plt.legend(["Class 1" , "Class 2"])
                plt.xlim([-2, 2])
                plt.ylim([-2, 2])
                plt.savefig('error_plot1.pdf')
                plt.show()
                # Generate predictions
                predicted_output = predict(inputs, target, weights, biases, num_neurons_layer1)
                # Create confusion matrix
                cm = confusion_matrix(target, predicted_output)
                # Plot heatmap of confusion matrix
                plt.figure(figsize=(6, 4))
                sns.heatmap(cm, annot=True, cmap="Blues", fmt="g")
                plt.xlabel("Predicted labels")
                plt.ylabel("True labels")
                plt.title("Confusion Matrix")
                plt.tight_layout()
                # Save plot as PDF
                plt.savefig("confusion_matrix.pdf")
                plt.show()
                predicted_output = predict(inputs, target,weights,biases,num_neurons_layer1)
                print(classification_report(target, predicted_output))    
            return weights, biases, error_list
        weights, b = update_weights(weights, biases, inputs[i % samples], target[i % samples], net_input, output, learning_rate, num_neurons_layer1)  # Step 7
    
    if plot:
        plt.plot(error_list)
        plt.xlabel('Epochs')
        plt.ylabel('Mean Squared Error')
        plt.title('Error Plot')
        plt.grid(True)
        plt.savefig('error_plot.pdf')
        plt.show()
        df0 = df0
        df1 = df1
        plt.scatter(df0['x'], df0['y'], c ="red", linewidths = 0.1)
        plt.scatter(df1['x'], df1['y'], c ="blue", linewidths = .1)

        for i in range(num_neurons_layer1):
            px1, px2 = find_decision_boundary(-2, 2, weights[i], biases[i])
            plt.plot(px1, px2)

        plt.xlabel("x")
        plt.ylabel("y")
        plt.legend(["Class 1" , "Class 2"])
        plt.xlim([-2, 2])
        plt.ylim([-2, 2])
        plt.savefig('error_plot1.pdf')
        plt.show()
        # Generate predictions
        predicted_output = predict(inputs, target, weights, biases, num_neurons_layer1)
        # Create confusion matrix
        cm = confusion_matrix(target, predicted_output)
        # Plot heatmap of confusion matrix
        plt.figure(figsize=(6, 4))
        sns.heatmap(cm, annot=True, cmap="Blues", fmt="g")
        plt.xlabel("Predicted labels")
        plt.ylabel("True labels")
        plt.title("Confusion Matrix")
        plt.tight_layout()
        # Save plot as PDF
        plt.savefig("confusion_matrix.pdf")
        plt.show()
        predicted_output = predict(inputs, target,weights,biases,num_neurons_layer1)
        print(classification_report(target, predicted_output))  

    return weights, biases, error_list

"""### Madaline Results (3 neurons)"""

weights, biases, error_list = MRI(df0, df1, inputs,target,num_neurons_layer1 = 3 ,num_neurons_layer2 = 2,max_iter = 200,learning_rate = 0.0001,samples = inputs.shape[0])

weights, biases, error_list = MRI(df0, df1, inputs,target,num_neurons_layer1 = 3 ,num_neurons_layer2 = 2,max_iter = 200,learning_rate = 0.01,samples = inputs.shape[0])

"""### Madaline Results (4 neurons)"""

weights, biases, error_list = MRI(df0, df1, inputs,target,num_neurons_layer1 = 4 ,num_neurons_layer2 = 2,max_iter = 200,learning_rate = 0.0001,samples = inputs.shape[0])

weights, biases, error_list = MRI(df0, df1, inputs,target,num_neurons_layer1 = 4 ,num_neurons_layer2 = 2,max_iter = 200,learning_rate = 0.0005,samples = inputs.shape[0])

"""### Madaline Results (10 neurons)"""

weights, biases, error_list = MRI(df0, df1, inputs,target,num_neurons_layer1 = 10 ,num_neurons_layer2 = 2,max_iter = 200,learning_rate = 0.01,samples = inputs.shape[0])

weights, biases, error_list = MRI(df0, df1, inputs,target,num_neurons_layer1 = 10 ,num_neurons_layer2 = 2,max_iter = 200,learning_rate = 0.0001,samples = inputs.shape[0])