机器学习13：神经网络的反向传播实现

# Importing pandas and numpy
import pandas as pd
import numpy as np

# Reading the csv file into a pandas DataFrame
data = pd.read_csv('student_data.csv')

# Printing out the first 10 rows of our data
data[:10]

# Importing matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
# Function to help us plot
def plot_points(data):
    X = np.array(data[["gre","gpa"]])
    y = np.array(data["admit"])
    admitted = X[np.argwhere(y==1)]
    rejected = X[np.argwhere(y==0)]
    plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'red', edgecolor = 'k')
    plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'cyan', edgecolor = 'k')
    plt.xlabel('Test (GRE)')
    plt.ylabel('Grades (GPA)')
    
# Plotting the points
plot_points(data)
plt.show()

# Separating the ranks
data_rank1 = data[data["rank"]==1]
data_rank2 = data[data["rank"]==2]
data_rank3 = data[data["rank"]==3]
data_rank4 = data[data["rank"]==4]

# Plotting the graphs
plot_points(data_rank1)
plt.title("Rank 1")
plt.show()
plot_points(data_rank2)
plt.title("Rank 2")
plt.show()
plot_points(data_rank3)
plt.title("Rank 3")
plt.show()
plot_points(data_rank4)
plt.title("Rank 4")
plt.show()

# Make dummy variables for rank
one_hot_data = pd.concat([data, pd.get_dummies(data['rank'], prefix='rank')], axis=1)
one_hot_data[:10]

# Drop the previous rank column
one_hot_data = one_hot_data.drop('rank', axis=1)

# Print the first 10 rows of our data
one_hot_data[:10]

# Making a copy of our data
processed_data = one_hot_data[:]

# TODO: Scale the columns
processed_data['gre'] = processed_data['gre']/800

processed_data['gpa'] = processed_data['gpa']/800

# Printing the first 10 rows of our procesed data
processed_data[:10]

sample = np.random.choice(processed_data.index, size=int(len(processed_data)*0.9), replace=False)
train_data, test_data = processed_data.iloc[sample], processed_data.drop(sample)

print("Number of training samples is", len(train_data))
print("Number of testing samples is", len(test_data))
print(train_data[:10])
print(test_data[:10])

Number of training samples is 360
Number of testing samples is 40
     admit    gre       gpa  rank_1  rank_2  rank_3  rank_4
7        0  0.500  0.003850       0       1       0       0
9        0  0.875  0.004900       0       1       0       0
165      0  0.875  0.005000       1       0       0       0
158      0  0.825  0.004363       0       1       0       0
211      0  0.725  0.003775       0       1       0       0
327      1  0.700  0.004350       0       1       0       0
132      0  0.725  0.004250       0       1       0       0
151      0  0.500  0.004225       0       1       0       0
78       0  0.675  0.003900       1       0       0       0
350      1  0.975  0.005000       0       1       0       0
    admit    gre       gpa  rank_1  rank_2  rank_3  rank_4
13      0  0.875  0.003850       0       1       0       0
15      0  0.600  0.004300       0       0       1       0
18      0  1.000  0.004687       0       1       0       0
21      1  0.825  0.004537       0       1       0       0
30      0  0.675  0.004725       0       0       0       1
54      0  0.825  0.004175       0       0       1       0
55      1  0.925  0.005000       0       0       1       0
56      0  0.700  0.003987       0       0       1       0
66      0  0.925  0.004525       0       0       0       1
69      0  1.000  0.004662       1       0       0       0

features = train_data.drop('admit', axis=1)
targets = train_data['admit']
features_test = test_data.drop('admit', axis=1)
targets_test = test_data['admit']

print(features[:10])
print(targets[:10])

       gre       gpa  rank_1  rank_2  rank_3  rank_4
7    0.500  0.003850       0       1       0       0
9    0.875  0.004900       0       1       0       0
165  0.875  0.005000       1       0       0       0
158  0.825  0.004363       0       1       0       0
211  0.725  0.003775       0       1       0       0
327  0.700  0.004350       0       1       0       0
132  0.725  0.004250       0       1       0       0
151  0.500  0.004225       0       1       0       0
78   0.675  0.003900       1       0       0       0
350  0.975  0.005000       0       1       0       0
7      0
9      0
165    0
158    0
211    0
327    1
132    0
151    0
78     0
350    1
Name: admit, dtype: int64

# Activation (sigmoid) function
def sigmoid(x):
    return 1 / (1 + np.exp(-x))
def sigmoid_prime(x):
    return sigmoid(x) * (1-sigmoid(x))
def error_formula(y, output):
    return - y*np.log(output) - (1 - y) * np.log(1-output)

# TODO: Write the error term formula
def error_term_formula(y, output):
    return (y-output)

# Neural Network hyperparameters
epochs = 1000
learnrate = 0.5

# Training function
def train_nn(features, targets, epochs, learnrate):
    
    # Use to same seed to make debugging easier
    np.random.seed(42)

    n_records, n_features = features.shape
    last_loss = None

    # Initialize weights
    weights = np.random.normal(scale=1 / n_features**.5, size=n_features)
    print(weights.shape)
    
    

    for e in range(epochs):
        del_w = np.zeros(weights.shape)
        for x, y in zip(features.values, targets):
            # Loop through all records, x is the input, y is the target

            # Activation of the output unit
            #   Notice we multiply the inputs and the weights here 
            #   rather than storing h as a separate variable 
            output = sigmoid(np.dot(x, weights))

            # The error, the target minus the network output
            error = error_formula(y, output)

            # The error term
            #   Notice we calulate f'(h) here instead of defining a separate
            #   sigmoid_prime function. This just makes it faster because we
            #   can re-use the result of the sigmoid function stored in
            #   the output variable
            error_term = error_term_formula(y, output)
            #print("error_term:",error_term)

            # The gradient descent step, the error times the gradient times the inputs
            del_w += error_term * x

        # Update the weights here. The learning rate times the 
        # change in weights, divided by the number of records to average
        weights += learnrate * del_w / n_records

        # Printing out the error on the training set
        if e % (epochs / 10) == 0:
            out = sigmoid(np.dot(features, weights))
            #print(out)
            loss = np.mean((out - targets) ** 2)
            print("Epoch:", e)
            if last_loss and last_loss < loss:
                print("Train loss: ", loss, "  WARNING - Loss Increasing")
            else:
                print("Train loss: ", loss)
            last_loss = loss
            print("=========")
    print("Finished training!")
    return weights
    
weights = train_nn(features, targets, epochs, learnrate)

(6,)
Epoch: 0
Train loss:  0.273075830588848
=========
Epoch: 100
Train loss:  0.20520354225199638
=========
Epoch: 200
Train loss:  0.20393525728827555
=========
Epoch: 300
Train loss:  0.2030136874864369
=========
Epoch: 400
Train loss:  0.2022100443775751
=========
Epoch: 500
Train loss:  0.2015047057211576
=========
Epoch: 600
Train loss:  0.20088633117322183
=========
Epoch: 700
Train loss:  0.2003443983488469
=========
Epoch: 800
Train loss:  0.1998693847500299
=========
Epoch: 900
Train loss:  0.19945286989297628
=========
Finished training!

# Calculate accuracy on test data
tes_out = sigmoid(np.dot(features_test, weights))
predictions = tes_out > 0.5
accuracy = np.mean(predictions == targets_test)
print("Prediction accuracy: {:.3f}".format(accuracy))

Prediction accuracy: 0.675