解析mnist数据库

Python解析数据库成图片

import struct
import numpy as np
#import matplotlib.pyplot as plt
import Image
import sys

input_path = sys.argv[1] #mnist数据库解压后的所在路径
output_path = sys.argv[2] #生成的图片所在的路径

# =====read labels=====
label_file = input_path + '/train-labels.idx1-ubyte'
label_fp = open(label_file, 'rb')
label_buf = label_fp.read()

label_index=0
label_magic, label_numImages = struct.unpack_from('>II', label_buf, label_index)
label_index += struct.calcsize('>II')
labels = struct.unpack_from('>60000B', label_buf, label_index)

# =====read train images=====
label_map = {}
train_file = input_path + '/train-images.idx3-ubyte'
train_fp = open(train_file, 'rb')
buf = train_fp.read()

index=0
magic,numImages,numRows,numColumns=struct.unpack_from('>IIII',buf,index)
index+=struct.calcsize('>IIII')
k = 0
for image in range(0,numImages):
label = labels[k]
if(label_map.has_key(label)):
ids = label_map[label] + 1
label_map[label] += 1

else:
label_map[label] = 0
ids = 0
k += 1
if(label_map[label] > 50):
continue
im=struct.unpack_from('>784B',buf,index)
index+=struct.calcsize('>784B')

im=np.array(im,dtype='uint8')
im=im.reshape(28,28)
#fig=plt.figure()
#plotwindow=fig.add_subplot(111)
#plt.imshow(im,cmap='gray')
#plt.show()
im=Image.fromarray(im)
im.save(output_path + '/%s_%s.bmp'%(label, ids),'bmp')

Matlab解析

trainImages = loadMNISTImages('train-images.idx3-ubyte');
trainLabels = loadMNISTLabels('train-labels.idx1-ubyte');
N = 784;
K = 100;% can be any other value
testImages = loadMNISTImages('t10k-images.idx3-ubyte');
testLabels = loadMNISTLabels('t10k-labels.idx1-ubyte');
trainLength = length(trainImages);
testLength = length(testImages);
testResults = linspace(0,0,length(testImages));
compLabel = linspace(0,0,K);
tic;
for i=1:testLength
curImage = repmat(testImages(:,i),1,trainLength);
curImage = abs(trainImages-curImage);
comp=sum(curImage);
[sortedComp,ind] = sort(comp);
for j = 1:K
compLabel(j) = trainLabels(ind(j));
end
table = tabulate(compLabel);
[maxCount,idx] = max(table(:,2));
testResults(i) = table(idx);

disp(testResults(i));
disp(testLabels(i));
end
% Compute the error on the test set
error=0;
for i=1:testLength
if (testResults(i) ~= testLabels(i))
error=error+1;
end
end

%Print out the classification error on the test set
error/testLength
toc;
disp(toc-tic);

function images = loadMNISTImages(filename)
%loadMNISTImages returns a 28x28x[number of MNIST images] matrix containing
%the raw MNIST images
fp = fopen(filename, 'rb');
assert(fp ~= -1, ['Could not open ', filename, '']);
magic = fread(fp, 1, 'int32', 0, 'ieee-be');
assert(magic == 2051, ['Bad magic number in ', filename, '']);
numImages = fread(fp, 1, 'int32', 0, 'ieee-be');
numRows = fread(fp, 1, 'int32', 0, 'ieee-be');
numCols = fread(fp, 1, 'int32', 0, 'ieee-be');
images = fread(fp, inf, 'unsigned char');
images = reshape(images, numCols, numRows, numImages);
images = permute(images,[2 1 3]);
fclose(fp);
% Reshape to #pixels x #examples
images = reshape(images, size(images, 1) * size(images, 2), size(images, 3));
% Convert to double and rescale to [0,1]
images = double(images) / 255;
end

function labels = loadMNISTLabels(filename)
%loadMNISTLabels returns a [number of MNIST images]x1 matrix containing
%the labels for the MNIST images
fp = fopen(filename, 'rb');
assert(fp ~= -1, ['Could not open ', filename, '']);
magic = fread(fp, 1, 'int32', 0, 'ieee-be');
assert(magic == 2049, ['Bad magic number in ', filename, '']);
numLabels = fread(fp, 1, 'int32', 0, 'ieee-be');
labels = fread(fp, inf, 'unsigned char');
assert(size(labels,1) == numLabels, 'Mismatch in label count');
fclose(fp);
end

C++解析

#include <iostream>
#include <fstream>

#include "opencv2/core/core.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"

using namespace std;

int ReverseInt(int i)
{
unsigned char ch1, ch2, ch3, ch4;
ch1 = i & 255;
ch2 = (i >> 8) & 255;
ch3 = (i >> 16) & 255;
ch4 = (i >> 24) & 255;
return((int) ch1 << 24) + ((int)ch2 << 16) + ((int)ch3 << 8) + ch4;
}

void read_Mnist(string filename, vector<cv::Mat> &vec)
{
ifstream file (filename, ios::binary);
if (file.is_open()) {
int magic_number = 0;
int number_of_images = 0;
int n_rows = 0;
int n_cols = 0;
file.read((char*) &magic_number, sizeof(magic_number));
magic_number = ReverseInt(magic_number);
file.read((char*) &number_of_images,sizeof(number_of_images));
number_of_images = ReverseInt(number_of_images);
file.read((char*) &n_rows, sizeof(n_rows));
n_rows = ReverseInt(n_rows);
file.read((char*) &n_cols, sizeof(n_cols));
n_cols = ReverseInt(n_cols);

for(int i = 0; i < number_of_images; ++i) {
cv::Mat tp = cv::Mat::zeros(n_rows, n_cols, CV_8UC1);
for(int r = 0; r < n_rows; ++r) {
for(int c = 0; c < n_cols; ++c) {
unsigned char temp = 0;
file.read((char*) &temp, sizeof(temp));
tp.at<uchar>(r, c) = (int) temp;
}
}
vec.push_back(tp);
}
}
}

void read_Mnist_Label(string filename, vector<int> &vec)
{
ifstream file (filename, ios::binary);
if (file.is_open()) {
int magic_number = 0;
int number_of_images = 0;
int n_rows = 0;
int n_cols = 0;
file.read((char*) &magic_number, sizeof(magic_number));
magic_number = ReverseInt(magic_number);
file.read((char*) &number_of_images,sizeof(number_of_images));
number_of_images = ReverseInt(number_of_images);

for(int i = 0; i < number_of_images; ++i) {
unsigned char temp = 0;
file.read((char*) &temp, sizeof(temp));
vec[i]= (int)temp;
}
}
}

string GetImageName(int number, int arr[])
{
string str1, str2;

for (int i = 0; i < 10; i++) {
if (number == i) {
arr[i]++;
char ch1[10];
sprintf(ch1, "%d", arr[i]);
str1 = std::string(ch1);

if (arr[i] < 10) {
str1 = "0000" + str1;
} else if (arr[i] < 100) {
str1 = "000" + str1;
} else if (arr[i] < 1000) {
str1 = "00" + str1;
} else if (arr[i] < 10000) {
str1 = "0" + str1;
}

break;
}
}

char ch2[10];
sprintf(ch2, "%d", number);
str2 = std::string(ch2);

str2 = str2 + "_" + str1;

return str2;
}

int main()
{
//reference: http://eric-yuan.me/cpp-read-mnist/ //test images and test labels
//read MNIST image into OpenCV Mat vector
string filename_test_images = "D:/Download/t10k-images-idx3-ubyte/t10k-images.idx3-ubyte";
int number_of_test_images = 10000;
vector<cv::Mat> vec_test_images;

read_Mnist(filename_test_images, vec_test_images);

//read MNIST label into int vector
string filename_test_labels = "D:/Download/t10k-labels-idx1-ubyte/t10k-labels.idx1-ubyte";
vector<int> vec_test_labels(number_of_test_images);

read_Mnist_Label(filename_test_labels, vec_test_labels);

if (vec_test_images.size() != vec_test_labels.size()) {
cout<<"parse MNIST test file error"<<endl;
return -1;
}

//save test images
int count_digits[10];
for (int i = 0; i < 10; i++)
count_digits[i] = 0;

string save_test_images_path = "D:/Download/MNIST/test_images/";

for (int i = 0; i < vec_test_images.size(); i++) {
int number = vec_test_labels[i];
string image_name = GetImageName(number, count_digits);
image_name = save_test_images_path + image_name + ".jpg";

cv::imwrite(image_name, vec_test_images[i]);
}

//train images and train labels
//read MNIST image into OpenCV Mat vector
string filename_train_images = "D:/Download/train-images-idx3-ubyte/train-images.idx3-ubyte";
int number_of_train_images = 60000;
vector<cv::Mat> vec_train_images;

read_Mnist(filename_train_images, vec_train_images);

//read MNIST label into int vector
string filename_train_labels = "D:/Download/train-labels-idx1-ubyte/train-labels.idx1-ubyte";
vector<int> vec_train_labels(number_of_train_images);

read_Mnist_Label(filename_train_labels, vec_train_labels);

if (vec_train_images.size() != vec_train_labels.size()) {
cout<<"parse MNIST train file error"<<endl;
return -1;
}

//save train images
for (int i = 0; i < 10; i++)
count_digits[i] = 0;

string save_train_images_path = "D:/Download/MNIST/train_images/";

for (int i = 0; i < vec_train_images.size(); i++) {
int number = vec_train_labels[i];
string image_name = GetImageName(number, count_digits);
image_name = save_train_images_path + image_name + ".jpg";

cv::imwrite(image_name, vec_train_images[i]);
}

return 0;
}