% Neural Network 1 hidden layer, 2 neurons
% Fred Park
% Whittier College, November 2017

clc 

% %lin sepa'ble movie rec data
% x1 = [1 1 1.5 2.5 2.5 2.5 2.5 3.5 3.5 4.5 4.5]';
% x2 = [5 4 4 5 1 1.5 3 5 4 5 4]';
% y = [0 0 0 1 0 0 0 1 1 1 1]';
% x = [x1'; x2'];

%non-lin sep'ble movie rec data
x1 = [1 1 1.5 1 1 1.5 2.5 2.5 2.5 2.5 3.5 3.5 4.5 4.5]';
x2 = [1 1.5 1 5 4 4 5 1 1.5 3 5 4 5 4]';
y = [1 1 1 0 0 0 1 0 0 0 1 1 1 1]';


g = @(z) 1./(1+exp(-z));
dg = @(z) (1-g(z)).*g(z);
lc = @(thetaA, thetaB, B, term1, term2) thetaA*term1 + thetaB*term2 + B;

delta = 0.05;

theta = random('Normal',0,1,1,6);
b = random('Normal',0,1,1,3);
l1 = 'liked'; l2 = 'did not like';

xp = linspace(0,6,10);

xx = [0:0.125:6];
yy = [0:0.125:6];
[X,Y] = meshgrid(xx,yy);
Z = predictorNN_1Layer_2Neurons(X,Y,theta,b);
v = [1/2,1/2];
c0 = find(y==0); c1 = find(y==1); 
figure(2)
plot(x1(c1),x2(c1),'ro','MarkerSize',10,'LineWidth', 3); hold on;
plot(x1(c0),x2(c0),'bx','MarkerSize',10,'LineWidth', 3); hold on;
contour(X,Y,Z,v);  
hold off; hold off;
axis([0 6 0 6]);
title('Movie Predictions')
xlabel('movie score #1'); ylabel('movie score #2');
legend(l1,l2)


sprintf('press any key to start the process')
pause
outer_max = 15000;

for j =1:outer_max 
    r = randperm(length(x1));
    for i = 1:length(x1)
        [gradh_theta, gradh_b] = gradNN_1Layer_2Neurons(x1(r(i)),x2(r(i)),y(r(i)),theta,b);
        
        theta = theta - delta*gradh_theta;
        b = b - delta*gradh_b;
         
        
    end
    %plot curve every 500 outer iterations
    if rem(j,500) == 0
        figure(2)
        plot(x1(c1),x2(c1),'ro','MarkerSize',10,'LineWidth', 3); hold on;
        plot(x1(c0),x2(c0),'bx','MarkerSize',10,'LineWidth', 3); hold on;      
        Z = predictorNN_1Layer_2Neurons(X,Y,theta,b);
        v = [1/2,1/2];
        contour(X,Y,Z,v)
        hold off; hold off;
        axis([0 6 0 6])
        title('Movie Predictions')
        xlabel('movie score #1'); ylabel('movie score #2');
        legend(l1,l2)
        pause(0.1)
        j
    end
end


figure(3)
plot(x1(c1),x2(c1),'ro','MarkerSize',10,'LineWidth', 3); hold on;
plot(x1(c0),x2(c0),'bx','MarkerSize',10,'LineWidth', 3); hold on;
title('Movie Predictions')
xlabel('movie score #1'); ylabel('movie score #2');
Z = predictorNN_1Layer_2Neurons(X,Y,theta,b);
v = [1/2,1/2];
contour(X,Y,Z,v)
hold off; hold off;
axis([0 6 0 6])
title('Movie Predictions')
xlabel('movie score #1'); ylabel('movie score #2');
legend(l1,l2)



function z = predictorNN_1Layer_2Neurons(X,Y,theta,b)
%This calculayes the predictor for a NN of 1 layer and 3 neurons
t = num2cell(theta);
[t1,t2,t3,t4,w1,w2] = deal(t{:});

tb = num2cell(b);
[b1,b2,c] = deal(tb{:});

%functions
g = @(z) 1./(1+exp(-z));
lc = @(thetaA, thetaB, B, term1, term2) thetaA*term1 + thetaB*term2 + B;

%forward pass
A1 = g(lc(t1,t2,b1,X,Y));
A2 = g(lc(t3,t4,b2,X,Y));
A3 = g(lc(w1,w2,c,A1,A2));

z = A3;
end



function [gradh_theta, gradh_b] = gradNN_1Layer_2Neurons(x1,x2,y,theta,b)
%This calculates backpropagation for a NN of 1 hidden layer and 3 neurons
%Backpropagation returns the gradient in terms of theta and b

t = num2cell(theta);
[t1,t2,t3,t4,w1,w2] = deal(t{:});

tb = num2cell(b);
[b1,b2,c] = deal(tb{:});

%functions
g = @(z) 1./(1+exp(-z));
dg = @(z) (1-z).*z;
lc = @(thetaA, thetaB, B, term1, term2) thetaA*term1 + thetaB*term2 + B;

% forward pass
A1 = g(lc(t1,t2,b1,x1,x2));
A2 = ?
A3 = ?
Dg = ?
Dg1 = ?
Dg2 = ?
A4 = A3-y;

%backward pass
Dt1 = A4.*Dg.*(w1*Dg1*x1);
Dt2 = ?
Dt3 = ?
Dt4 = ?
Dw1 = ?
Dw2 = ?
Db1 = ?
Db2 = ?
Dc = A4.*Dg.*1;

%gradient
gradh_theta = [Dt1,Dt2,Dt3,Dt4,Dw1,Dw2];
gradh_b = [Db1,Db2,Dc];

end





