%% Setup the parameters you will use for this part of the exercise

input_layer_size  = 400;  % 20x20 Input Images of Digits

num_labels = 10;          % 10 labels, from 1 to 10

                          % (note that we have mapped "0" to label 10)

% Load Training Data

fprintf('Loading and Visualizing Data ...\n')

load('ex3data1.mat'); % training data stored in arrays X, y

m = size(X, 1);

% Randomly select 100 data points to display

rand_indices = randperm(m);

sel = X(rand_indices(1:100), :);

displayData(sel);

function [h, display_array] = displayData(X, example_width)

% Set example_width automatically if not passed in

if ~exist('example_width', 'var') || isempty(example_width)

	example_width = round(sqrt(size(X, 2)));

end

% Gray Image

colormap(gray);

% Compute rows, cols

[m n] = size(X);

example_height = (n / example_width);

% Compute number of items to display

display_rows = floor(sqrt(m));

display_cols = ceil(m / display_rows);

% Between images padding

pad = 1;

% Setup blank display

display_array = - ones(pad + display_rows * (example_height + pad), ...

                       pad + display_cols * (example_width + pad));

% Copy each example into a patch on the display array

curr_ex = 1;

for j = 1:display_rows

	for i = 1:display_cols

		if curr_ex > m,

			break;

		end

		% Copy the patch

		% Get the max value of the patch

		max_val = max(abs(X(curr_ex, :)));

		display_array(pad + (j - 1) * (example_height + pad) + (1:example_height), ...

		              pad + (i - 1) * (example_width + pad) + (1:example_width)) = ...

						reshape(X(curr_ex, :), example_height, example_width) / max_val;

		curr_ex = curr_ex + 1;

	end

	if curr_ex > m,

		break;

	end

end

% Display Image

h = imagesc(display_array, [-1 1]);

% Do not show axis

axis image off

drawnow;

end

lambda = 0.1;

[all_theta] = oneVsAll(X, y, num_labels, lambda);

function [all_theta] = oneVsAll(X, y, num_labels, lambda)

% Some useful variables

m = size(X, 1);

n = size(X, 2);

% You need to return the following variables correctly

all_theta = zeros(num_labels, n + 1);

% Add ones to the X data matrix

X = [ones(m, 1) X];

for c = 1:num_labels,

    initial_theta = zeros(n+1, 1);

    options = optimset('GradObj', 'on', 'MaxIter', 50);

    [theta] = ...

        fmincg(@(t)(lrCostFunction(t, X, (y==c), lambda)), initial_theta, options);

    all_theta(c,:) = theta;

end;

end

function [J, grad] = lrCostFunction(theta, X, y, lambda)

% Initialize some useful values

m = length(y); % number of training examples

% You need to return the following variables correctly

J = 0;

grad = zeros(size(theta));

theta2 = theta(2:end,1);

h = sigmoid(X*theta);

J = 1/m*(-y'*log(h)-(1-y')*log(1-h)) + lambda/(2*m)*sum(theta2.^2);

theta(1,1) = 0;

grad = 1/m*(X'*(h-y)) + lambda/m*theta;

grad = grad(:);

end

function g = sigmoid(z)

g = 1.0 ./ (1.0 + exp(-z));

end

pred = predictOneVsAll(all_theta, X);

fprintf('\nTraining Set Accuracy: %f\n', mean(double(pred == y)) * 100);

function p = predictOneVsAll(all_theta, X)

m = size(X, 1);

num_labels = size(all_theta, 1);

% You need to return the following variables correctly

p = zeros(size(X, 1), 1);

% Add ones to the X data matrix

X = [ones(m, 1) X];

g = zeros(size(X, 1), num_labels);

for c = 1: num_labels,

    theta = all_theta(c, :);

    g(:, c) = sigmoid(X*theta');

end

[value, p] = max(g, [], 2);

end

%% Setup the parameters you will use for this exercise

input_layer_size  = 400;  % 20x20 Input Images of Digits

hidden_layer_size = 25;   % 25 hidden units

num_labels = 10;          % 10 labels, from 1 to 10

                          % (note that we have mapped "0" to label 10)

% Load Training Data

fprintf('Loading and Visualizing Data ...\n')

load('ex3data1.mat');

m = size(X, 1);

% Randomly select 100 data points to display

sel = randperm(size(X, 1));

sel = sel(1:100);

displayData(X(sel, :));

% Load the weights into variables Theta1 and Theta2

load('ex3weights.mat');

pred = predict(Theta1, Theta2, X);

fprintf('\nTraining Set Accuracy: %f\n', mean(double(pred == y)) * 100);

function p = predict(Theta1, Theta2, X)

% Useful values

m = size(X, 1);

num_labels = size(Theta2, 1);

% You need to return the following variables correctly

p = zeros(size(X, 1), 1);

X = [ones(m,1) X];

z2 = X*Theta1';

a2 = sigmoid(z2);

a2 = [ones(size(a2, 1), 1) a2];

z3 = a2*Theta2';

a3 = sigmoid(z3)

[values, p] = max(a3, [], 2)

end

rp = randperm(m);

for i = 1:m

    % Display

    fprintf('\nDisplaying Example Image\n');

    displayData(X(rp(i), :));

    pred = predict(Theta1, Theta2, X(rp(i),:));

    fprintf('\nNeural Network Prediction: %d (digit %d)\n', pred, mod(pred, 10));

    % Pause with quit option

    s = input('Paused - press enter to continue, q to exit:','s');

    if s == 'q'

      break

    end

end