FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fix pycaffe input processing · mirsys/CaffeOnACL@d5c3cef · GitHub

  1. mirsys
  2. CaffeOnACL

Repository navigation

Commit d5c3cef

Browse files
shelhamer
committed
fix pycaffe input processing
- load an image as [0,1] single / np.float32 according to Python convention - fix input scaling during preprocessing: - scale input for preprocessing by `raw_scale` e.g. to map an image to [0, 255] for the CaffeNet and AlexNet ImageNet models - scale feature space by `input_scale` after mean subtraction - switch examples to raw scale for ImageNet models - fix #525 - preserve type after resizing. - resize 1, 3, or K channel images with special casing between skimage.transform (1 and 3) and scipy.ndimage (K) for speed
1 parent f1eb982 commit d5c3cef

11 files changed

Lines changed: 98 additions & 55 deletions

‎examples/detection.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
"input": [
3737
"!mkdir -p _temp\n",
3838
"!echo `pwd`/images/fish-bike.jpg > _temp/det_input.txt\n",
39-
"!../python/detect.py --crop_mode=selective_search --pretrained_model=imagenet/caffe_rcnn_imagenet_model --model_def=imagenet/rcnn_imagenet_deploy.prototxt --gpu _temp/det_input.txt _temp/det_output.h5"
39+
"!../python/detect.py --crop_mode=selective_search --pretrained_model=imagenet/caffe_rcnn_imagenet_model --model_def=imagenet/rcnn_imagenet_deploy.prototxt --gpu --raw_scale=255 _temp/det_input.txt _temp/det_output.h5"
4040
],
4141
"language": "python",
4242
"metadata": {},

‎examples/filter_visualization.ipynb‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,8 @@
6666
"net.set_mode_cpu()\n",
6767
"# input preprocessing: 'data' is the name of the input blob == net.inputs[0]\n",
6868
"net.set_mean('data', caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy') # ImageNet mean\n",
69-
"net.set_channel_swap('data', (2,1,0)) # the reference model has channels in BGR order instead of RGB\n",
70-
"net.set_input_scale('data', 255) # the reference model operates on images in [0,255] range instead of [0,1]"
69+
"net.set_raw_scale('data', 255) # the reference model operates on images in [0,255] range instead of [0,1]\n",
70+
"net.set_channel_swap('data', (2,1,0)) # the reference model has channels in BGR order instead of RGB"
7171
],
7272
"language": "python",
7373
"metadata": {},
@@ -178,12 +178,6 @@
178178
"cell_type": "code",
179179
"collapsed": false,
180180
"input": [
181-
"# our network takes BGR images, so we need to switch color channels\n",
182-
"def showimage(im):\n",
183-
" if im.ndim == 3:\n",
184-
" im = im[:, :, ::-1]\n",
185-
" plt.imshow(im)\n",
186-
" \n",
187181
"# take an array of shape (n, height, width) or (n, height, width, channels)\n",
188182
"# and visualize each (height, width) thing in a grid of size approx. sqrt(n) by sqrt(n)\n",
189183
"def vis_square(data, padsize=1, padval=0):\n",
@@ -199,7 +193,7 @@
199193
" data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3) + tuple(range(4, data.ndim + 1)))\n",
200194
" data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:])\n",
201195
" \n",
202-
" showimage(data)"
196+
" imshow(data)"
203197
],
204198
"language": "python",
205199
"metadata": {},
@@ -218,10 +212,7 @@
218212
"collapsed": false,
219213
"input": [
220214
"# index four is the center crop\n",
221-
"image = net.blobs['data'].data[4].copy()\n",
222-
"image -= image.min()\n",
223-
"image /= image.max()\n",
224-
"showimage(image.transpose(1, 2, 0))"
215+
"imshow(net.deprocess('data', net.blobs['data'].data[4]))"
225216
],
226217
"language": "python",
227218
"metadata": {},
@@ -606,4 +597,4 @@
606597
"metadata": {}
607598
}
608599
]
609-
}
600+
}

‎examples/imagenet_classification.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
"cell_type": "markdown",
5454
"metadata": {},
5555
"source": [
56-
"Loading a network is easy. `caffe.Classifier` takes care of everything. Note the arguments for configuring input preprocessing: mean subtraction switched on by giving a mean file, input channel swapping takes care of mapping RGB into the reference ImageNet model's BGR order, and input scaling multiplies the feature scale from the input [0,1] to [0,255]."
56+
"Loading a network is easy. `caffe.Classifier` takes care of everything. Note the arguments for configuring input preprocessing: mean subtraction switched on by giving a mean file, input channel swapping takes care of mapping RGB into the reference ImageNet model's BGR order, and raw scaling multiplies the feature scale from the input [0,1] to the ImageNet model's [0,255]."
5757
]
5858
},
5959
{
@@ -63,7 +63,7 @@
6363
"net = caffe.Classifier(MODEL_FILE, PRETRAINED,\n",
6464
" mean_file=caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy',\n",
6565
" channel_swap=(2,1,0),\n",
66-
" input_scale=255,\n",
66+
" raw_scale=255,\n",
6767
" image_dims=(256, 256))"
6868
],
6969
"language": "python",

‎examples/net_surgery.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@
277277
"plt.imshow(im)\n",
278278
"net_full_conv.set_mean('data', '../python/caffe/imagenet/ilsvrc_2012_mean.npy')\n",
279279
"net_full_conv.set_channel_swap('data', (2,1,0))\n",
280-
"net_full_conv.set_input_scale('data', 255.0)\n",
280+
"net_full_conv.set_raw_scale('data', 255.0)\n",
281281
"# make classification map by forward pass and show top prediction index per location\n",
282282
"out = net_full_conv.forward_all(data=np.asarray([net_full_conv.preprocess('data', im)]))\n",
283283
"out['prob'][0].argmax(axis=0)"
@@ -331,4 +331,4 @@
331331
"metadata": {}
332332
}
333333
]
334-
}
334+
}

‎python/caffe/_caffe.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,7 @@ struct CaffeNet {
278278
// Input preprocessing configuration attributes.
279279
dict mean_;
280280
dict input_scale_;
281+
dict raw_scale_;
281282
dict channel_swap_;
282283
// if taking input from an ndarray, we need to hold references
283284
object input_data_;
@@ -329,6 +330,7 @@ BOOST_PYTHON_MODULE(_caffe) {
329330
.add_property("outputs", &CaffeNet::outputs)
330331
.add_property("mean", &CaffeNet::mean_)
331332
.add_property("input_scale", &CaffeNet::input_scale_)
333+
.add_property("raw_scale", &CaffeNet::raw_scale_)
332334
.add_property("channel_swap", &CaffeNet::channel_swap_)
333335
.def("_set_input_arrays", &CaffeNet::set_input_arrays)
334336
.def("save", &CaffeNet::save);

‎python/caffe/classifier.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,14 @@ class Classifier(caffe.Net):
1414
by scaling, center cropping, or oversampling.
1515
"""
1616
def __init__(self, model_file, pretrained_file, image_dims=None,
17-
gpu=False, mean_file=None, input_scale=None, channel_swap=None):
17+
gpu=False, mean_file=None, input_scale=None, raw_scale=None,
18+
channel_swap=None):
1819
"""
1920
Take
2021
image_dims: dimensions to scale input for cropping/sampling.
21-
Default is to scale to net input size for whole-image crop.
22-
gpu, mean_file, input_scale, channel_swap: convenience params for
23-
setting mode, mean, input scale, and channel order.
22+
Default is to scale to net input size for whole-image crop.
23+
gpu, mean_file, input_scale, raw_scale, channel_swap: params for
24+
preprocessing options.
2425
"""
2526
caffe.Net.__init__(self, model_file, pretrained_file)
2627
self.set_phase_test()
@@ -32,9 +33,11 @@ def __init__(self, model_file, pretrained_file, image_dims=None,
3233

3334
if mean_file:
3435
self.set_mean(self.inputs[0], mean_file)
35-
if input_scale:
36+
if input_scale is not None:
3637
self.set_input_scale(self.inputs[0], input_scale)
37-
if channel_swap:
38+
if raw_scale is not None:
39+
self.set_raw_scale(self.inputs[0], raw_scale)
40+
if channel_swap is not None:
3841
self.set_channel_swap(self.inputs[0], channel_swap)
3942

4043
self.crop_dims = np.array(self.blobs[self.inputs[0]].data.shape[2:])

‎python/caffe/detector.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,12 @@ class Detector(caffe.Net):
2525
selective search proposals.
2626
"""
2727
def __init__(self, model_file, pretrained_file, gpu=False, mean_file=None,
28-
input_scale=None, channel_swap=None, context_pad=None):
28+
input_scale=None, raw_scale=None, channel_swap=None,
29+
context_pad=None):
2930
"""
3031
Take
31-
gpu, mean_file, input_scale, channel_swap: convenience params for
32-
setting mode, mean, input scale, and channel order.
32+
gpu, mean_file, input_scale, raw_scale, channel_swap: params for
33+
preprocessing options.
3334
context_pad: amount of surrounding context to take s.t. a `context_pad`
3435
sized border of pixels in the network input image is context, as in
3536
R-CNN feature extraction.
@@ -44,9 +45,11 @@ def __init__(self, model_file, pretrained_file, gpu=False, mean_file=None,
4445

4546
if mean_file:
4647
self.set_mean(self.inputs[0], mean_file)
47-
if input_scale:
48+
if input_scale is not None:
4849
self.set_input_scale(self.inputs[0], input_scale)
49-
if channel_swap:
50+
if raw_scale is not None:
51+
self.set_raw_scale(self.inputs[0], raw_scale)
52+
if channel_swap is not None:
5053
self.set_channel_swap(self.inputs[0], channel_swap)
5154

5255
self.configure_crop(context_pad)
@@ -180,12 +183,12 @@ def configure_crop(self, context_pad):
180183
"""
181184
self.context_pad = context_pad
182185
if self.context_pad:
183-
input_scale = self.input_scale.get(self.inputs[0])
186+
raw_scale = self.raw_scale.get(self.inputs[0])
184187
channel_order = self.channel_swap.get(self.inputs[0])
185188
# Padding context crops needs the mean in unprocessed input space.
186189
self.crop_mean = self.mean[self.inputs[0]].copy()
187190
self.crop_mean = self.crop_mean.transpose((1,2,0))
188191
channel_order_inverse = [channel_order.index(i)
189192
for i in range(self.crop_mean.shape[2])]
190193
self.crop_mean = self.crop_mean[:,:, channel_order_inverse]
191-
self.crop_mean /= input_scale
194+
self.crop_mean /= raw_scale

‎python/caffe/io.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import numpy as np
22
import skimage.io
3-
import skimage.transform
3+
from scipy.ndimage import zoom
4+
from skimage.transform import resize
45

56
from caffe.proto import caffe_pb2
67

@@ -15,7 +16,8 @@ def load_image(filename, color=True):
1516
loads as intensity (if image is already grayscale).
1617
1718
Give
18-
image: an image with type np.float32 of size (H x W x 3) in RGB or
19+
image: an image with type np.float32 in range [0, 1]
20+
of size (H x W x 3) in RGB or
1921
of size (H x W x 1) in grayscale.
2022
"""
2123
img = skimage.img_as_float(skimage.io.imread(filename)).astype(np.float32)
@@ -40,7 +42,17 @@ def resize_image(im, new_dims, interp_order=1):
4042
Give
4143
im: resized ndarray with shape (new_dims[0], new_dims[1], K)
4244
"""
43-
return skimage.transform.resize(im, new_dims, order=interp_order)
45+
if im.shape[-1] == 1 or im.shape[-1] == 3:
46+
# skimage is fast but only understands {1,3} channel images in [0, 1].
47+
im_min, im_max = im.min(), im.max()
48+
im_std = (im - im_min) / (im_max - im_min)
49+
resized_std = resize(im_std, new_dims, order=interp_order)
50+
resized_im = resized_std * (im_max - im_min) + im_min
51+
else:
52+
# ndimage interpolates anything but more slowly.
53+
scale = tuple(np.array(new_dims) / np.array(im.shape[:2]))
54+
resized_im = zoom(im, scale + (1,), order=interp_order)
55+
return resized_im.astype(np.float32)
4456

4557

4658
def oversample(images, crop_dims):

‎python/caffe/pycaffe.py‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -216,23 +216,22 @@ def _Net_set_mean(self, input_, mean_f, mode='elementwise'):
216216
in_shape = self.blobs[input_].data.shape
217217
mean = np.load(mean_f)
218218
if mode == 'elementwise':
219-
if mean.shape != in_shape[1:]:
220-
# Resize mean (which requires H x W x K input in range [0,1]).
221-
m_min, m_max = mean.min(), mean.max()
222-
normal_mean = (mean - m_min) / (m_max - m_min)
223-
mean = caffe.io.resize_image(normal_mean.transpose((1,2,0)),
224-
in_shape[2:]).transpose((2,0,1)) * (m_max - m_min) + m_min
219+
if mean.shape[1:] != in_shape[2:]:
220+
# Resize mean (which requires H x W x K input).
221+
mean = caffe.io.resize_image(mean.transpose((1,2,0)),
222+
in_shape[2:]).transpose((2,0,1))
225223
self.mean[input_] = mean
226224
elif mode == 'channel':
227225
self.mean[input_] = mean.mean(1).mean(1).reshape((in_shape[1], 1, 1))
228226
else:
229227
raise Exception('Mode not in {}'.format(['elementwise', 'channel']))
230228

231229

232-
233230
def _Net_set_input_scale(self, input_, scale):
234231
"""
235-
Set the input feature scaling factor s.t. input blob = input * scale.
232+
Set the scale of preprocessed inputs s.t. the blob = blob * scale.
233+
N.B. input_scale is done AFTER mean subtraction and other preprocessing
234+
while raw_scale is done BEFORE.
236235
237236
Take
238237
input_: which input to assign this scale factor
@@ -243,6 +242,22 @@ def _Net_set_input_scale(self, input_, scale):
243242
self.input_scale[input_] = scale
244243

245244

245+
def _Net_set_raw_scale(self, input_, scale):
246+
"""
247+
Set the scale of raw features s.t. the input blob = input * scale.
248+
While Python represents images in [0, 1], certain Caffe models
249+
like CaffeNet and AlexNet represent images in [0, 255] so the raw_scale
250+
of these models must be 255.
251+
252+
Take
253+
input_: which input to assign this scale factor
254+
scale: scale coefficient
255+
"""
256+
if input_ not in self.inputs:
257+
raise Exception('Input not in {}'.format(self.inputs))
258+
self.raw_scale[input_] = scale
259+
260+
246261
def _Net_set_channel_swap(self, input_, order):
247262
"""
248263
Set the input channel order for e.g. RGB to BGR conversion
@@ -263,10 +278,11 @@ def _Net_preprocess(self, input_name, input_):
263278
Format input for Caffe:
264279
- convert to single
265280
- resize to input dimensions (preserving number of channels)
266-
- scale feature
267281
- reorder channels (for instance color to BGR)
268-
- subtract mean
282+
- scale raw input (e.g. from [0, 1] to [0, 255] for ImageNet models)
269283
- transpose dimensions to K x H x W
284+
- subtract mean
285+
- scale feature
270286
271287
Take
272288
input_name: name of input blob to preprocess for
@@ -275,20 +291,23 @@ def _Net_preprocess(self, input_name, input_):
275291
Give
276292
caffe_inputs: (K x H x W) ndarray
277293
"""
278-
caffe_in = input_.astype(np.float32)
294+
caffe_in = input_.astype(np.float32, copy=False)
279295
mean = self.mean.get(input_name)
280296
input_scale = self.input_scale.get(input_name)
297+
raw_scale = self.raw_scale.get(input_name)
281298
channel_order = self.channel_swap.get(input_name)
282299
in_size = self.blobs[input_name].data.shape[2:]
283300
if caffe_in.shape[:2] != in_size:
284301
caffe_in = caffe.io.resize_image(caffe_in, in_size)
285-
if input_scale is not None:
286-
caffe_in *= input_scale
287302
if channel_order is not None:
288303
caffe_in = caffe_in[:, :, channel_order]
289304
caffe_in = caffe_in.transpose((2, 0, 1))
305+
if raw_scale is not None:
306+
caffe_in *= raw_scale
290307
if mean is not None:
291308
caffe_in -= mean
309+
if input_scale is not None:
310+
caffe_in *= input_scale
292311
return caffe_in
293312

294313

@@ -299,16 +318,19 @@ def _Net_deprocess(self, input_name, input_):
299318
decaf_in = input_.copy().squeeze()
300319
mean = self.mean.get(input_name)
301320
input_scale = self.input_scale.get(input_name)
321+
raw_scale = self.raw_scale.get(input_name)
302322
channel_order = self.channel_swap.get(input_name)
323+
if input_scale is not None:
324+
decaf_in /= input_scale
303325
if mean is not None:
304326
decaf_in += mean
327+
if raw_scale is not None:
328+
decaf_in /= raw_scale
305329
decaf_in = decaf_in.transpose((1,2,0))
306330
if channel_order is not None:
307331
channel_order_inverse = [channel_order.index(i)
308332
for i in range(decaf_in.shape[2])]
309333
decaf_in = decaf_in[:, :, channel_order_inverse]
310-
if input_scale is not None:
311-
decaf_in /= input_scale
312334
return decaf_in
313335

314336

@@ -364,6 +386,7 @@ def _Net_batch(self, blobs):
364386
Net.forward_backward_all = _Net_forward_backward_all
365387
Net.set_mean = _Net_set_mean
366388
Net.set_input_scale = _Net_set_input_scale
389+
Net.set_raw_scale = _Net_set_raw_scale
367390
Net.set_channel_swap = _Net_set_channel_swap
368391
Net.preprocess = _Net_preprocess
369392
Net.deprocess = _Net_deprocess

‎python/classify.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,12 @@ def main(argv):
6666
parser.add_argument(
6767
"--input_scale",
6868
type=float,
69-
default=255,
70-
help="Multiply input features by this scale before input to net"
69+
help="Multiply input features by this scale to finish input preprocessing."
70+
)
71+
parser.add_argument(
72+
"--raw_scale",
73+
type=float,
74+
help="Multiply raw input by this scale before preprocessing."
7175
)
7276
parser.add_argument(
7377
"--channel_swap",

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL