线性回归 numpy normal equation & tensorflow gradient descent

Linear Regression

normal equation

最简单的线性回归就是求出w，b使得 y = w x + b与真实值均方误差最小

于是很直接的可以直接令cost fuction 导数等于0， 然后就有可以求出closed solution

唯一需要注意的是把b吸收进w中 也就是 X = [X,1], Y = W * X 方便计算

train_x = (np.arange(10000)/100)
train_x=np.column_stack((train_x,np.ones(10000)))

其实核心代码就很简单：

def linearRegression(self):
xTx = train_x.T.dot(train_x)
if np.linalg.det(xTx) ==0.0:
print("singular")
return
w = np.linalg.inv(xTx).dot(self.train_x.T).dot(self.train_y)
pre_y = self.test_x.dot(w)
return pre_y

但是要注意：要先确定xTx非奇异才能继续往下做。

整体(顺便加了bias variance decomposition)

import numpy as np
import matplotlib.pyplot as plt

def f(x):
return x+x*2*np.sin(x)

class regression:
def __init__(self,train_x,train_y,test_x,test_y):
self.test_x = test_x
self.train_x = train_x
self.train_y = train_y
self.test_y = test_y

def linearRegression(self):
xTx = train_x.T.dot(train_x)
if np.linalg.det(xTx) ==0.0:
print("singular")
return
w = np.linalg.inv(xTx).dot(self.train_x.T).dot(self.train_y)
pre_y = self.test_x.dot(w)
return pre_y

def bias_variance(self,pre_y):
var = np.mean((pre_y - np.mean(pre_y)) ** 2)
bias2 = np.mean(np.mean(pre_y) - self.test_y) ** 2
noise = np.mean((pre_y - self.test_y) ** 2)
print("var = ", var)
print("bias2 = ", bias2)
print("noise = ", noise)
def plotAns(self,pre_y):
plt.plot(self.test_y, 'ro', label='Original data')
plt.plot(pre_y, label='Fitted line')
plt.show()

if __name__=="__main__":
train_x = (np.arange(10000)/100)
train_y = f(train_x)
test_x = train_x + np.random.random()
train_x = np.column_stack((train_x,np.ones(10000)))

test_x = np.column_stack((test_x, np.ones(10000)))
test_y = f(test_x[:,0])

regressor = regression(train_x,train_y,test_x,test_y)
pre_y = regressor.linearRegression()
regressor.bias_variance(pre_y)
regression(train_x, train_y, test_x, test_y).LWLR(0.1)
regressor.plotAns(pre_y)

上图

gradient descent

直接用tensorflow就很简单了，直接暴力优化均方误差就可以

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import random
from time import time
rng = np.random

# Training Data
train_size = 2000
train_X = np.linspace(0,10,train_size)
test_Y = train_X+2*np.sin(1.5*train_X)
train_Y = train_X+2*np.sin(1.5*train_X) + random.gauss(0,0.2)

learning_rate = 0.01
training_epochs = 2000
display_step = 50

class linearRegression:

def VectorLR(train_X,train_Y,learning_rate,training_epochs,display_step):
n_samples = train_X.shape[0]
X = tf.placeholder("float",[n_samples])
Y = tf.placeholder("float",[n_samples])

W = tf.Variable(tf.random_normal([1]),name = "weight")
b = tf.Variable(tf.random_normal([1]),name = "bias")

activation = X*W+b

cost = tf.reduce_sum(tf.pow(activation-Y,2))/(2*n_samples)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

init = tf.initialize_all_variables()

sess = tf.InteractiveSession()
sess.run(init)
for epoch in range(training_epochs):
sess.run(optimizer,feed_dict={X:train_X,Y:train_Y})
return sess.run(W) * train_X + sess.run(b)