-
Notifications
You must be signed in to change notification settings - Fork 10
/
Copy pathbackwardpass.m
158 lines (154 loc) · 6.35 KB
/
backwardpass.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
function [dE,delta]=backwardpass(delta,XX,F,dropprob)
% ****** ERROR BACK PROPAGATION ******
Nlayers=length(F);
if Nlayers==0
dE=[];
else
% XX is of length Nlayers+1.
% Last layer, must be non-convolutional.
j=Nlayers; N=size(delta,1);
% To take into account the scale issue.
delta=delta/(1-dropprob(j+1)); T=XX{j+1}*(1-dropprob(j+1)); delta(T==0)=0;
switch lower(F{j}.type)
case 'linear',
% Do nothing.
case 'relu',
delta(T<=0)=0;
case 'leaky_relu',
delta(T<=0)=delta(T<=0)*0.1;
case 'cubic',
delta=delta./(1+T.^2);
case 'sigmoid',
delta=delta.*T.*(1-T);
case 'sigmoid_zero_mean',
delta=delta.*(0.5+T).*(0.5-T);
case 'tanh',
delta=delta.*(1-T.^2);
case 'logistic',
delta=delta.*T.*(1-T);
case 'softmax',
delta=delta.*T - repmat(sum(delta.*T,2),1,size(T,2)).*T;
otherwise,
error('Invalid layer type: %s\n',F{j}.type);
end
if Nlayers>1 && strcmpi(F{Nlayers-1}.type,'conv');
de=[XX{j}.respfull ones(N,1)]'*delta;
else
de=[XX{j} ones(N,1)]'*delta;
end
de(1:end-1,:)=de(1:end-1,:) + 2*F{j}.l*F{j}.W(1:end-1,:);
dE=de(:);
% Prepare the delta for next layer.
delta=delta * F{j}.W(1:end-1,:)';
% Other layers.
for j=Nlayers-1:-1:1
T=XX{j+1};
if strcmp(F{j}.type,'conv')
% Reshape the deltas to the size of output feature maps. We start from here.
delta=reshape(delta,N,F{j}.sizeout1,F{j}.sizeout2,F{j}.numoutputmaps);
% Stretch. Inverse the pooling process.
delta=delta(:,repmat(1:F{j}.sizeout1,F{j}.rowpoolratio,1),...
repmat(1:F{j}.sizeout2,F{j}.colpoolratio,1),:);
delta=delta(:,1:F{j}.sizeout_prepool1,1:F{j}.sizeout_prepool2,:);
delta=delta.*T.respidx;
switch F{j}.pooling
case 'average',
delta=delta./(F{j}.rowpoolratio*F{j}.colpoolratio);
end
% Inverse the nonlinearity.
switch F{j}.sigmoid
case 'sigmoid',
delta=delta.*(T.respfull_prepool).*(1-T.respfull_prepool);
case 'sigmoid_zero_mean',
delta=delta.*(0.5+T.respfull_prepool).*(0.5-T.respfull_prepool);
case 'tanh',
delta=delta.*(1-(T.respfull_prepool).^2);
case 'relu',
delta(T.respfull_prepool==0)=0; % j, length(find(T.respfull_prepool==0))
case 'leaky_relu',
delta(T.respfull_prepool==0)=delta(T.respfull_prepool==0)*0.1; % j, length(find(T.respfull_prepool==0))
end
% delta is now of size [N, sizeout_prepool1, sizeout_prepool2, numoutputmaps].
dbias=sum(reshape(delta,N*F{j}.sizeout_prepool1*F{j}.sizeout_prepool2,F{j}.numoutputmaps),1);
% Fetch the layer below and reshape it to the size of input feature maps.
if (j>1) && strcmp(F{j-1}.type,'conv') % There are still conv layers below.
lowerlayeroutput=XX{j}.respfull;
else % It is the first layer, lower layer output is the input.
lowerlayeroutput=XX{j};
end
lowerlayeroutput=reshape(lowerlayeroutput,N,F{j}.inputnumrows,F{j}.inputnumcols,F{j}.numinputmaps);
lowerlayeroutput=repmat(lowerlayeroutput,[1 1 1 1 F{j}.numoutputmaps]);
% lowerlayeroutput is of size [N inputnumrows intputnumcols numinputmaps numoutputmaps]
rcW=F{j}.W(1:end-1,:);
rcW=reshape(rcW,[1, F{j}.filternumrows, F{j}.filternumcols, F{j}.numinputmaps, F{j}.numoutputmaps]);
rfilter=repmat(rcW,[N, 1, 1, 1, 1]);
% rfilter is of size [N filternumrow filternumcols numinputmaps numoutputmaps].
de=zeros(1, F{j}.filternumrows*F{j}.filternumcols*F{j}.numinputmaps,...
F{j}.numoutputmaps, F{j}.sizeout_prepool1*F{j}.sizeout_prepool2);
% de is of size [1 filternumrow*filternumcols numinputmaps prepoolimagesize].
% if (j>1) && strcmp(F{j-1}.type,'conv') % There are still conv layers below.
delta_lower=zeros(N,F{j}.inputnumrows,F{j}.inputnumcols,F{j}.numinputmaps);
% end
for ai=1:F{j}.sizeout_prepool1
for aj=1:F{j}.sizeout_prepool2
% for each pixel in the convolved image.
acts=reshape(delta(:,ai,aj,:),[N,1,F{j}.numoutputmaps]);
% find subimage that contribute to the convolution.
rowstart=(ai-1)*F{j}.rowstride+1;
rowend=(ai-1)*F{j}.rowstride+F{j}.filternumrows;
colstart=(aj-1)*F{j}.colstride+1;
colend=(aj-1)*F{j}.colstride+F{j}.filternumcols;
inblock=lowerlayeroutput(:, rowstart:rowend, colstart:colend, :, :);
inblock=bsxfun(@times,reshape(inblock,[N,...
F{j}.filternumrows*F{j}.filternumcols*F{j}.numinputmaps, F{j}.numoutputmaps]),acts);
de(:,:,:, (ai-1)*F{j}.sizeout_prepool2+aj)=sum(inblock,1);
% if (j>1) && strcmp(F{j-1}.type,'conv') % There are still conv layers below.
delta_lower(:,rowstart:rowend,colstart:colend,:)=...
delta_lower(:,rowstart:rowend,colstart:colend,:) + ...
sum(bsxfun(@times,rfilter,reshape(acts,N,1,1,1,F{j}.numoutputmaps)), 5);
% end
end
end
de=reshape(sum(de,4),F{j}.filternumrows*F{j}.filternumcols*F{j}.numinputmaps,F{j}.numoutputmaps);
de=[de; dbias];
de(1:end-1,:)=de(1:end-1,:) + 2*F{j}.l*F{j}.W(1:end-1,:);
dE=[de(:); dE];
% Prepare the delta for next layer.
delta=delta_lower;
if j==1 delta=reshape(delta,N,F{j}.inputnumrows*F{j}.inputnumcols*F{j}.numinputmaps); end
else
% Non-convolutional layers consider drop out.
delta=delta/(1-dropprob(j+1)); T=T*(1-dropprob(j+1)); delta(T==0)=0;
switch lower(F{j}.type)
case 'linear',
% Do nothing.
case 'relu',
delta(T<=0)=0;
case 'leaky_relu',
delta(T<=0)=delta(T<=0)*0.1;
case 'cubic',
delta=delta./(1+T.^2);
case 'sigmoid',
delta=delta.*T.*(1-T);
case 'sigmoid_zero_mean',
delta=delta.*(0.5+T).*(0.5-T);
case 'tanh',
delta=delta.*(1-T.^2);
case 'logistic',
delta=delta.*T.*(1-T);
otherwise,
error('Invalid layer type: %s\n',F{j}.type);
end
if j>1 && strcmpi(F{j-1}.type,'conv');
de=[XX{j}.respfull ones(N,1)]'*delta;
else
de=[XX{j} ones(N,1)]'*delta;
end
de(1:end-1,:)=de(1:end-1,:) + 2*F{j}.l*F{j}.W(1:end-1,:);
dE=[de(:); dE];
% Prepare the delta for next layer.
delta=delta * F{j}.W(1:end-1,:)';
end
end
% ****** END OF ERROR BACKPROPAGATION ******
end