I am trying to convert a set of grayscale images and ground truth images to LMDB for segmentation task. I have created 4 text files (train.txt
, train_label.txt
, val.txt
, val_label.txt
) indicating path to each image in each folder.
For many days, it is taking my time and still I am not successful. I am a beginner in python
and caffe
I will be thankful if somebody helps me here.
I tried the following code:
import os
import numpy as np
from scipy import io
import lmdb
import caffe
from PIL import Image
NUM_IDX_DIGITS = 10
IDX_FMT = '{:0>%d' % NUM_IDX_DIGITS + 'd}'
# src
def img_to_lmdb(paths_src_file, path_dst):
"""
paths_src: src img paths
path_dst: src img lmdb file
"""
print("Creating Training Data LMDB File ..... ")
paths_src = []
with open(paths_src_file) as f:
for line in f.readlines():
#print line
line = line.strip('\n')
paths_src.append(line)
#path_dst = 'train-lmdb'
in_db = lmdb.open(path_dst, map_size=int(1e12))
with in_db.begin(write=True) as in_txn:
for in_idx, in_ in enumerate(paths_src):
print in_idx, in_
# load image:
# - as np.uint8 {0, ..., 255}
# - in BGR (switch from RGB)
# - in Channel x Height x Width order (switch from H x W x C)
im = np.array(Image.open(in_)) # or load whatever ndarray you need
###############
print im.shape
im = np.expand_dims(im, axis=0)
print im.shape
###################
im = im[:,:,::-1]
#im = im[:,:]
im = im.transpose((2,0,1)) # convert to CxHxW
im_dat = caffe.io.array_to_datum(im)
in_txn.put('{:0>10d}'.format(in_idx), im_dat.SerializeToString())
in_db.close()
print 'end.'
# label
def label_to_lmdb(paths_label_file, path_dst):
"""
paths_src: label img paths
path_dst: label lmdb file
"""
print("Creating Training Label LMDB File ..... ")
paths_src = []
with open(paths_label_file) as f:
for line in f.readlines():
line = line.strip('\n')
paths_src.append(line)
in_db = lmdb.open(path_dst, map_size=int(1e12))
with in_db.begin(write=True) as in_txn:
for in_idx, in_ in enumerate(paths_src):
print in_idx, in_
# load image:
# - as np.uint8 {0, ..., 255}
# - in BGR (switch from RGB)
# - in Channel x Height x Width order (switch from H x W x C)
im = np.array(Image.open(in_)) # or load whatever ndarray you need
#
im = im.reshape(im.shape[0], im.shape[1], 1)
im = im.transpose((2,0,1))
##
##im = np.expand_dims(im, axis=0)
##print im.shape
#im = np.expand_dims(im, axis=0)
# create the dataset
im_dat = caffe.io.array_to_datum(im)
in_txn.put('{:0>10d}'.format(in_idx), im_dat.SerializeToString())
in_db.close()
print 'end.'
# train
train_img_paths = 'train.txt'
train_img_lmdb = 'train_img_lmdb'
train_label_paths = 'train_label.txt'
train_label_lmdb = 'train_label_lmdb'
img_to_lmdb(train_img_paths, train_img_lmdb)
label_to_lmdb(train_label_paths, train_label_lmdb)
# val
val_img_paths = 'val.txt'
val_img_lmdb = 'val_img_lmdb'
val_label_paths = 'val_label.txt'
val_label_lmdb = 'val_label_lmdb1'
img_to_lmdb(val_img_paths, val_img_lmdb)
label_to_lmdb(val_label_paths, val_label_lmdb)
I was getting an error:
im = im[:,:,::-1]
IndexError: too many indices for array
And:
im = im.transpose((2,0,1)) # convert to CxHxW
ValueError: axes don't match array
Since my images and labels are in grayscale (i.e., only one channel), I added im = np.expand_dims(im, axis=0)
, and now I am getting the following error:
Creating Training Data LMDB File .....
0 /home/user2/caffe-pascalcontext-fcn32s/dataset/Train/PNG/image-001-001.png
(281, 389)
(1, 281, 389)
Traceback (most recent call last):
File "lmdb_data.py", line 104, in <module>
img_to_lmdb(train_img_paths, train_img_lmdb)
File "lmdb_data.py", line 46, in img_to_lmdb
im_dat = caffe.io.array_to_datum(im)
File "/home/user2/caffe-master/python/caffe/io.py", line 78, in array_to_datum
datum.float_data.extend(arr.flat)
File "/home/user2/anaconda2/envs/caffeenv/lib/python2.7/site-packages/google/protobuf/internal/containers.py", line 275, in extend
new_values = [self._type_checker.CheckValue(elem) for elem in elem_seq_iter]
File "/home/user2/anaconda2/envs/caffeenv/lib/python2.7/site-packages/google/protobuf/internal/type_checkers.py", line 108, in CheckValue
raise TypeError(message)
TypeError: 66 has type <type 'numpy.int32'>, but expected one of: ((<type 'float'>, <type 'int'>, <type 'long'>),)
- How can I fix this error?
- How can I convert these four image sets into LMDB?
- Is it possible I use
convert_imageset
(Caffe built-in converter) for this task? If yes, please let me know how can I do that?
Your help is really appreciated.