# 家乐的深度学习笔记「4」 - softmax回归

## softmax回归

### 分类问题

。如此，一张图像的标签为1、2和3这3个数值中的一个。虽然仍可以使用回归模型来进行建模，并将预测值就近定点化到1、2和3这3个离散值之一，但这种连续值到离散值的转换通常会影响到分类质量。因此一般使用更加适合离散值输出的模型来解决分类问题。

### softmax回归模型

softmax与线性回归一样将输入特征与权重做线性叠加。其输出值个数等于标签里的类别数。由于每个输出的计算都要依赖于所有的输入，所以其输出层也是一个全连接层。

#### softmax运算

，会存在两个问题：其一，由于输出层的输出值的范围不确定，很难直观上判断这些值的意义；另一方面，由于真实标签为离散值，这些离散值与不确定范围的输出值之间的误差难以衡量。 ### 矢量表达式

#### 单样本分类的矢量计算表达式 #### 小批量样本分类的矢量计算表达式 ### 交叉熵损失函数

softmax运算将输出变换成一个合法的类别预测分布，而真实标签也可以用类别分布表达：对于样本 ，我们构造向量 （ 为前文提到的输出个数（类别数）），使其第 个（样本i类别的离散数值）个元素为1，其余为0。这样训练目标就可以设为使预测概率分布 尽可能接近真实的标签概率分布 。也就是说，交叉熵只关心对正确类别的预测概率，因为只要其值足够大，就可以确保分类结果正确。然而，当一个样本具有多个标签时，例如图像中不止一个物体时，并不能做这一步简化。但即便对于这种情况，交叉熵同样只关心对图像中出现的物体类别的预测概率。 ，即最小化交叉熵损失函数等价于最大化训练数据集所有标签类别的联合预测概率。

，自己概念绕混了，已更新在前一篇的损失函数章节上。

## 图像分类数据集（Fashion-MNIST）

Fashion-MNIST，顾名思义，时尚的MNIST，那MNIST是什么呢？MNIST数据集来自美国国家标准与技术研究所，由来自 250 个不同人手写的数字构成，全都是压缩过的手写数字图片，由于已经被玩烂了（大部分模型都可以做到95%以上的准确率），所以新的入门数据集被推了出来，即Fashion-MNIST。

Fashion-MNIST一共包括十个类别，分别为't-shirt', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot'，即T恤、裤子、套衫、连衣裙、外套、凉鞋、衬衫、运动鞋、包和短靴，由于存储空间原因，标签仅记录了数值标签0-9。

### 获取数据集

#### 必要的包

from mxnet.gluon import data as gdata
import d2lzh as d2l
import time
import sys

#### 训练与测试数据集

mnist_train = gdata.vision.FashionMNIST(train=True)
mnist_test = gdata.vision.FashionMNIST(train=False)

#### 训练数据集样本数

len(mnist_train), len(mnist_test)

(60000, 10000)

### 查看样本

feature, label = mnist_train
print(feature.shape, feature.dtype)
print(label, type(label), label.dtype)

(28, 28, 1) <class 'numpy.uint8'>
2 <class 'numpy.int32'> int32

#### 获取标签

def get_fashion_mnist_labels(labels):
text_labels = ['t-shirt', 'trouser', 'pullover', 'dress', 'coat',
'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
return [text_labels[int(i)] for i in labels]

#### 画出图像

def show_fashion_mnist(images, labels):
d2l.use_svg_display()
_, figs = d2l.plt.subplots(1, len(images), figsize=(12, 12))
for f, img, lbl in zip(figs, images, labels):
f.imshow(img.reshape((28, 28)).asnumpy())
f.set_title(lbl)
f.axes.get_xaxis().set_visible(False)
f.axes.get_yaxis().set_visible(False)

X, y = mnist_train[:9]
show_fashion_mnist(X, get_fashion_mnist_labels(y)) #### 读取小批量

batch_size = 256
transformer = gdata.vision.transforms.ToTensor()
if sys.platform.startswith('win'):
num_workers = 0
else:
num_workers = 80

batch_size=batch_size, shuffle=True,
num_workers=num_workers)
batch_size=batch_size, shuffle=False,
num_workers=num_workers)

start = time.time()
for X, y in train_iter:
continue
'%.2f sec' % (time.time() - start)

'0.74 sec'

## softmax回归的从零实现

### 必要的包

from mxnet import autograd, gpu, nd
import d2lzh as d2l

ctx = gpu(3)

### 读取数据集

batch_size = 512
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)

### 初始化模型参数

num_inputs = 28 * 28
num_outputs = 10

W = nd.random.normal(scale=0.01, shape=(num_inputs, num_outputs), ctx=ctx)
b = nd.zeros(num_outputs, ctx=ctx)

b.attach_grad()

### 实现softmax运算

X = nd.array([[1, 2, 3], [3, 4, 5]], ctx=ctx)
print(X.sum(axis=0, keepdims=True), X.sum(axis=1, keepdims=True))
print(X.sum(axis=0), X.sum(axis=1, keepdims=False))

[[4. 6. 8.]]
<NDArray 1x3 @gpu(3)>
[[ 6.]
[12.]]
<NDArray 2x1 @gpu(3)>

[4. 6. 8.]
<NDArray 3 @gpu(3)>
[ 6. 12.]
<NDArray 2 @gpu(3)>

def softmax(X):
X_exp = X.exp()
partition = X_exp.sum(axis=1, keepdims=True)
return X_exp / partition

softmax(X)
[[0.09003057 0.24472848 0.66524094]
[0.09003057 0.24472846 0.66524094]]
<NDArray 2x3 @gpu(3)>

a = nd.array(, ctx=ctx)
a.exp()
[5.1847055e+21]
<NDArray 1 @gpu(3)>

X = nd.random.normal(shape=(2, 5), ctx=ctx)
X_prob = softmax(X)
print(X)
print(X_prob, X_prob.sum(axis=1))
[[-1.1795309   1.7976178   1.52335    -1.3275213  -0.21527036]
[ 0.35299432  1.0368916  -1.1053166   0.08840044 -1.0292935 ]]
<NDArray 2x5 @gpu(3)>

[[0.02561495 0.5028665  0.38224316 0.02209134 0.0671841 ]
[0.23625386 0.46815717 0.05495946 0.18132897 0.05930058]]
<NDArray 2x5 @gpu(3)>
[1. 1.]
<NDArray 2 @gpu(3)>

### 定义模型

def net(X):
return softmax(nd.dot(X.reshape((-1, num_inputs)).as_in_context(ctx), W) + b)

### 定义损失函数

y_hat = nd.array([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])
y = nd.array([0, 2], dtype='int32')
nd.pick(y_hat, y)
[0.1 0.5]
<NDArray 2 @cpu(0)>

def cross_entropy(y_hat, y):
return -nd.pick(y_hat, y).log()

a = nd.array([0.000001, 0.999999], ctx=ctx)
print(a / a)
[999999.]
<NDArray 1 @gpu(3)>

### 计算分类准确率

def accuracy(y_hat, y):
return (y_hat.argmax(axis=1) == y.astype('float32')).mean().asscalar()

accuracy(y_hat, y)
0.5

def evaluate_accuracy(data_iter, net):
acc_sum, n = 0.0, 0
for X, y in data_iter:
y = y.astype('float32').as_in_context(ctx)
acc_sum += (net(X).argmax(axis=1) == y).sum().asscalar()
n += y.size
return acc_sum / n

evaluate_accuracy(test_iter, net)
0.0608

### 训练模型

num_epochs, lr = 5, 0.2

def train(net, train_iter, test_iter, loss, num_epochs,
batch_size, params=None, lr=None, trainer=None):
for epoch in range(num_epochs):
train_l_sum, train_acc_sum, n = 0.0, 0.0, 0
for X, y in train_iter:
y_hat = net(X)
l = loss(y_hat, y.as_in_context(ctx)).sum()
l.backward()
if trainer is None:
d2l.sgd(params, lr, batch_size)
else:
traniner.step(batch_size)
y = y.astype('float32')
train_l_sum += l.asscalar()
train_acc_sum += (y_hat.argmax(axis=1) == y.as_in_context(ctx)).sum().asscalar()
n += y.size
test_acc = evaluate_accuracy(test_iter, net)
print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
% (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))

train(net, train_iter, test_iter, cross_entropy, num_epochs, batch_size, [W, b], lr)
epoch 1, loss 0.9251, train acc 0.703, test acc 0.778
epoch 2, loss 0.6221, train acc 0.789, test acc 0.805
epoch 3, loss 0.5600, train acc 0.810, test acc 0.825
epoch 4, loss 0.5325, train acc 0.819, test acc 0.826
epoch 5, loss 0.5085, train acc 0.827, test acc 0.821

### 预测效果

count = 0
for X, y in test_iter:
if count == 233:
break
count += 1

true_labels = d2l.get_fashion_mnist_labels(y.asnumpy())
pred_labels = d2l.get_fashion_mnist_labels(net(X).argmax(axis=1).asnumpy())
titles = [true + '\n' + pred for true, pred in zip(true_labels, pred_labels)]

d2l.show_fashion_mnist(X[:10], titles[:10]) ## softmax简洁实现

### 必要的包

from mxnet.gluon import loss as gloss, nn
from mxnet import autograd, gluon, init, gpu
import d2lzh as d2l

ctx = gpu(3)

### 读取数据集

batch_size = 512
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)

### 定义和初始化模型

net = nn.Sequential()
net.initialize(init.Normal(sigma=0.01), ctx=ctx)

### softmax和交叉熵损失函数

loss = gloss.SoftmaxCrossEntropyLoss()

### 训练模型

trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.2})

def evaluate_accuracy(data_iter, net):
acc_sum, n = 0.0, 0
for X, y in data_iter:
X = X.as_in_context(ctx)
y = y.astype('float32').as_in_context(ctx)
acc_sum += (net(X).argmax(axis=1) == y).sum().asscalar()
n += y.size
return acc_sum / n

num_epochs = 5

def train(net, train_iter, test_iter, loss, num_epochs, batch_size, trainer):
for epoch in range(num_epochs):
train_l_sum, train_acc_sum, n = 0.0, 0.0, 0
for X, y in train_iter:
y_hat = net(X.as_in_context(ctx))
l = loss(y_hat, y.as_in_context(ctx)).sum()
l.backward()
trainer.step(batch_size)
y = y.astype('float32')
train_l_sum += l.asscalar()
train_acc_sum += (y_hat.argmax(axis=1) == y.as_in_context(ctx)).sum().asscalar()
n += y.size
test_acc = evaluate_accuracy(test_iter, net)
print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
% (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))

train(net, train_iter, test_iter, loss, num_epochs, batch_size, trainer)

epoch 1, loss 0.9137, train acc 0.703, test acc 0.788
epoch 2, loss 0.6413, train acc 0.784, test acc 0.820
epoch 3, loss 0.5698, train acc 0.805, test acc 0.816
epoch 4, loss 0.5274, train acc 0.822, test acc 0.833
epoch 5, loss 0.5265, train acc 0.819, test acc 0.840
{"ft_lang_label":"__label__zh","ft_lang_prob":0.5388094,"math_prob":0.99234414,"size":10419,"snap":"2020-24-2020-29","text_gpt3_token_len":5837,"char_repetition_ratio":0.11349016,"word_repetition_ratio":0.20361991,"special_character_ratio":0.3191285,"punctuation_ratio":0.24468665,"nsfw_num_words":0,"has_unicode_error":false,"math_prob_llama3":0.99588543,"pos_list":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22],"im_url_duplicate_count":[null,1,null,1,null,1,null,1,null,1,null,1,null,1,null,1,null,1,null,1,null,1,null],"WARC_HEADER":"{\"WARC-Type\":\"response\",\"WARC-Date\":\"2020-06-07T06:35:01Z\",\"WARC-Record-ID\":\"<urn:uuid:2e787f80-5bbe-462f-9095-d1deb63966d6>\",\"Content-Length\":\"48925\",\"Content-Type\":\"application/http; msgtype=response\",\"WARC-Warcinfo-ID\":\"<urn:uuid:5a97dc04-3c60-4b23-aaa5-59999b628e8c>\",\"WARC-Concurrent-To\":\"<urn:uuid:3dbc2533-e7ac-4bf4-b2cc-1aa4a423e549>\",\"WARC-IP-Address\":\"39.106.8.162\",\"WARC-Target-URI\":\"https://www.tuicool.com/articles/VrENZzA\",\"WARC-Payload-Digest\":\"sha1:WONQIBBP3OFD5DDHTDRXZMDHYD5PRVFF\",\"WARC-Block-Digest\":\"sha1:3RZ7NQBC6HSRUJ2VULUON6OGGGORADVU\",\"WARC-Identified-Payload-Type\":\"text/html\",\"warc_filename\":\"/cc_download/warc_2020/CC-MAIN-2020-24/CC-MAIN-2020-24_segments_1590348523564.99_warc_CC-MAIN-20200607044626-20200607074626-00284.warc.gz\"}"}