【FPGA】手写数字识别2-卷积层

2022-12-31

字数统计: 2.9k字 | 阅读时长≈ 15分

卷积模块

多个卷积核的卷积层

完成一个卷积核对image的卷积运算，输出feature map。

一个卷积层包含多个filter，也可称为卷积核。image输入到不同的Filter，得到包含不同特征的feature map。filterSet通过选择不同的卷积核并行完成两个特征图的运算。

image是32x32的，filter是5x5的。

第一层卷积层C1，有filters长度为2400。第一层有6个卷积核，每个卷积核是5x5的，位宽是FP16，故线宽为6x5x5x16=2400。

image线宽为32x32x16=16384。

输出的feature map线宽为6x28x28x16=75264。

单个卷积核的卷积层

执行单个filter对image的卷积，输出对应的feature map（总结果）。

filter为单个卷积核线宽为5x5x16=400。

输出的feature map为28x28x16=12544

卷积单元

实现一次单个卷积核和image的卷积，输出对应的feature map（单次窗口卷积运算结果）

输出的feature map线宽为1x16，一次卷积是乘加，只得到一个FP16的结果。

基本运算单元：乘加

实现一次FP16的乘加，加法器是累加器。

功能分析

半精度浮点数（FP16）

$$
number=(-1)^s\times(1.M)\times2^{E-15}
$$

s：符号位；E：指数位；M：尾数位

浮点数加减法

本部分内容属于计算机组成原理，可参考网上资料得到算法。

`timescale 100 ns / 10 ps

module floatAdd16 (floatA,floatB,sum);
	
input [15:0] floatA, floatB;
output reg [15:0] sum;

reg sign;
reg signed [5:0] exponent; //fifth bit is sign
reg [9:0] mantissa;
reg [4:0] exponentA, exponentB;
reg [10:0] fractionA, fractionB, fraction;	//fraction = {1,mantissa}
reg [7:0] shiftAmount;
reg cout;

always @ (floatA or floatB) begin
	exponentA = floatA[14:10];
	exponentB = floatB[14:10];
	fractionA = {1'b1,floatA[9:0]};
	fractionB = {1'b1,floatB[9:0]}; 
	
	exponent = exponentA;

	if (floatA == 0) begin						//special case (floatA = 0)
		sum = floatB;
	end else if (floatB == 0) begin					//special case (floatB = 0)
		sum = floatA;
	end else if (floatA[14:0] == floatB[14:0] && floatA[15]^floatB[15]==1'b1) begin
		sum=0;
	end else begin // 1、对阶，保证小数部分正确对齐相加
		if (exponentB > exponentA) begin
			shiftAmount = exponentB - exponentA;
			fractionA = fractionA >> (shiftAmount);
			exponent = exponentB;
		end else if (exponentA > exponentB) begin 
			shiftAmount = exponentA - exponentB;
			fractionB = fractionB >> (shiftAmount);
			exponent = exponentA;
		end
		if (floatA[15] == floatB[15]) begin			//same sign
            {cout,fraction} = fractionA + fractionB;  // 2、尾数求和
            if (cout == 1'b1) begin  // 3、规格化
				{cout,fraction} = {cout,fraction} >> 1;
				exponent = exponent + 1;
			end
			sign = floatA[15];
		end else begin						//different signs
			if (floatA[15] == 1'b1) begin
				{cout,fraction} = fractionB - fractionA;
			end else begin
				{cout,fraction} = fractionA - fractionB;
			end
			sign = cout;
			if (cout == 1'b1) begin
				fraction = -fraction;
			end else begin
			end
			if (fraction [10] == 0) begin
				if (fraction[9] == 1'b1) begin
					fraction = fraction << 1;
					exponent = exponent - 1;
				end else if (fraction[8] == 1'b1) begin
					fraction = fraction << 2;
					exponent = exponent - 2;
				end else if (fraction[7] == 1'b1) begin
					fraction = fraction << 3;
					exponent = exponent - 3;
				end else if (fraction[6] == 1'b1) begin
					fraction = fraction << 4;
					exponent = exponent - 4;
				end else if (fraction[5] == 1'b1) begin
					fraction = fraction << 5;
					exponent = exponent - 5;
				end else if (fraction[4] == 1'b1) begin
					fraction = fraction << 6;
					exponent = exponent - 6;
				end else if (fraction[3] == 1'b1) begin
					fraction = fraction << 7;
					exponent = exponent - 7;
				end else if (fraction[2] == 1'b1) begin
					fraction = fraction << 8;
					exponent = exponent - 8;
				end else if (fraction[1] == 1'b1) begin
					fraction = fraction << 9;
					exponent = exponent - 9;
				end else if (fraction[0] == 1'b1) begin
					fraction = fraction << 10;
					exponent = exponent - 10;
				end 
			end
		end
		mantissa = fraction[9:0];
		if(exponent[5]==1'b1) begin //exponent is negative
			sum = 16'b0000000000000000;
		end
		else begin
			sum = {sign,exponent[4:0],mantissa};
		end		
	end		
end

endmodule

浮点数乘法

本部分内容属于计算机组成原理，可参考网上资料得到算法。

`timescale 100 ns / 10 ps

module floatMult16 (floatA,floatB,product);

input [15:0] floatA, floatB;
output reg [15:0] product;

reg sign;
reg signed [5:0] exponent; //6th bit is the sign
reg [9:0] mantissa;
reg [10:0] fractionA, fractionB;	//fraction = {1,mantissa}
reg [21:0] fraction;


always @ (floatA or floatB) begin
	if (floatA == 0 || floatB == 0) begin
		product = 0;
	end else begin
		sign = floatA[15] ^ floatB[15];
		exponent = floatA[14:10] + floatB[14:10] - 5'd15 + 5'd2;
	
		fractionA = {1'b1,floatA[9:0]};
		fractionB = {1'b1,floatB[9:0]};
		fraction = fractionA * fractionB;
		
		if (fraction[21] == 1'b1) begin
			fraction = fraction << 1;
			exponent = exponent - 1; 
		end else if (fraction[20] == 1'b1) begin
			fraction = fraction << 2;
			exponent = exponent - 2;
		end else if (fraction[19] == 1'b1) begin
			fraction = fraction << 3;
			exponent = exponent - 3;
		end else if (fraction[18] == 1'b1) begin
			fraction = fraction << 4;
			exponent = exponent - 4;
		end else if (fraction[17] == 1'b1) begin
			fraction = fraction << 5;
			exponent = exponent - 5;
		end else if (fraction[16] == 1'b1) begin
			fraction = fraction << 6;
			exponent = exponent - 6;
		end else if (fraction[15] == 1'b1) begin
			fraction = fraction << 7;
			exponent = exponent - 7;
		end else if (fraction[14] == 1'b1) begin
			fraction = fraction << 8;
			exponent = exponent - 8;
		end else if (fraction[13] == 1'b1) begin
			fraction = fraction << 9;
			exponent = exponent - 9;
		end else if (fraction[12] == 1'b0) begin
			fraction = fraction << 10;
			exponent = exponent - 10;
		end 
	
		mantissa = fraction[21:12];
		if(exponent[5]==1'b1) begin //exponent is negative
			product=16'b0000000000000000;
		end
		else begin
			product = {sign,exponent[4:0],mantissa};
		end
	end
end

endmodule

FP16转hex的在线转换器：https://evanw.github.io/float-toy/

基本运算单元：乘加模块

完成FP16和FP16的乘加。代码使用串行的方式累加，而非加法树形式。因此只需每次计算出一个乘法结果，就自动累加。

`timescale 100 ns / 10 ps

module processingElement16(clk,reset,floatA,floatB,result);

parameter DATA_WIDTH = 16;

input clk, reset;
input [DATA_WIDTH-1:0] floatA, floatB;
output reg [DATA_WIDTH-1:0] result;

wire [DATA_WIDTH-1:0] multResult;
wire [DATA_WIDTH-1:0] addResult;

floatMult16 FM (floatA,floatB,multResult);
floatAdd16 FADD (multResult,result,addResult);

always @ (posedge clk or posedge reset) begin
	if (reset == 1'b1) begin
		result = 0;
	end else begin
		result = addResult;
	end
end

endmodule

卷积单元

完成5x5的FP16和5x5的FP16的乘加，即一个窗口的卷积。

根据上一个模块的分析，本模块只需要完成连续的5x5个FP16的乘加，输出的和即总的卷积结果。代码中使用循环变量i来依次切割相对应的image和filter做乘加。只有reset能让i置0，因此本模块的reset其实是开始信号。所花费的时钟周期为：5x5个乘加+1个输出=26（或者认为加法在乘法的下一个周期）。

要注意，image和filter都是每FP16为一个独立数据。

`timescale 100 ns / 10 ps

module convUnit(clk,reset,image,filter,result);

parameter DATA_WIDTH = 16;
parameter D = 1; //depth of the filter
parameter F = 5; //size of the filter

input clk, reset;
input [0:D*F*F*DATA_WIDTH-1] image, filter;
output [0:DATA_WIDTH-1] result;

reg [DATA_WIDTH-1:0] selectedInput1, selectedInput2;

integer i;


processingElement16 PE
	(
		.clk(clk),
		.reset(reset),
		.floatA(selectedInput1),
		.floatB(selectedInput2),
		.result(result)
	);

// The convolution is calculated in a sequential process to save hardware
// The result of the element wise matrix multiplication is finished after (F*F+2) cycles (2 cycles to reset the processing element and F*F cycles to accumulate the result of the F*F multiplications) 
always @ (posedge clk, posedge reset) begin
	if (reset == 1'b1) begin // reset
		i = 0;
		selectedInput1 = 0;
		selectedInput2 = 0;
	end else if (i > D*F*F-1) begin // if the convolution is finished but we still wait for other blocks to finsih, send zeros to the conv unit (in case of pipelining)
		selectedInput1 = 0;
		selectedInput2 = 0;
	end else begin // send one element of the image part and one element of the filter to be multiplied and accumulated
		selectedInput1 = image[DATA_WIDTH*i+:DATA_WIDTH];
		selectedInput2 = filter[DATA_WIDTH*i+:DATA_WIDTH];
		i = i + 1;
	end
end

endmodule

RFSelector模块和convLayerSingle模块

本模块十分重要，也比较难理解。

1、将一幅图像分成左右两部分

2、给定一个rowNumber和column（这个column就是左边部分0或者右边部分1），输出这个rowNumber的所有卷积所需的image部分（即FxFxD的大小，一个四方体），这个image部分是先按行排，再按depth排，再按列排。

3、输出即一个column的一行中包含的所有卷积所需的image部分。

每个column需要有（H-F+1)个rowNumber，每一个row有（W-F+1)/2个列。因此在convLayerSingle模块里

要例化（W-F+1)/2次卷积单元（第一层需14个），也即给定某行，其每一列的卷积是并行的通过截取receptiveField的部分来取出每个并行卷积所需要的image部分。
要循环rowNumber (H-F+1)次，即遍历每行，每行之间的运算是串行的。
在convLayerSingle模块，先算完一行中的两个column，再换到下一行计算。即column之间的卷积是串行的。
convLayerSingle模块完成一个filter对整幅图像的卷积

`timescale 100 ns / 10 ps

module RFselector(image,rowNumber, column,receptiveField);

parameter DATA_WIDTH = 16;
parameter D = 1; //Depth of the filter
parameter H = 32; //Height of the image
parameter W = 32; //Width of the image
parameter F = 5; //Size of the filter

input [0:D*H*W*DATA_WIDTH-1] image;
input [5:0] rowNumber, column;
output reg [0:(((W-F+1)/2)*D*F*F*DATA_WIDTH)-1] receptiveField;

integer address, c, k, i;

always @ (image or rowNumber or column) begin
	address = 0;
	if (column == 0) begin
		for (c = 0; c < (W-F+1)/2; c = c + 1) begin
			for (k = 0; k < D; k = k + 1) begin
				for (i = 0; i < F; i = i + 1) begin
					receptiveField[address*F*DATA_WIDTH+:F*DATA_WIDTH] = image[rowNumber*W*DATA_WIDTH+c*DATA_WIDTH+k*H*W*DATA_WIDTH+i*W*DATA_WIDTH+:F*DATA_WIDTH];
					address = address + 1;
				end
			end
		end
	end else begin
		for (c = (W-F+1)/2; c < (W-F+1); c = c + 1) begin
			for (k = 0; k < D; k = k + 1) begin
				for (i = 0; i < F; i = i + 1) begin
					receptiveField[address*F*DATA_WIDTH+:F*DATA_WIDTH] = image[rowNumber*W*DATA_WIDTH+c*DATA_WIDTH+k*H*W*DATA_WIDTH+i*W*DATA_WIDTH+:F*DATA_WIDTH];
					address = address + 1;
				end
			end
		end
	end
	
end

endmodule

`timescale 100 ns / 10 ps

module convLayerSingle(clk,reset,image,filter,outputConv);

parameter DATA_WIDTH = 16;
parameter D = 1; //Depth of the filter
parameter H = 32; //Height of the image
parameter W = 32; //Width of the image
parameter F = 5; //Size of the filter

input clk, reset;
input [0:D*H*W*DATA_WIDTH-1] image;
input [0:D*F*F*DATA_WIDTH-1] filter;
output reg [0:(H-F+1)*(W-F+1)*DATA_WIDTH-1] outputConv; // output of the module

wire [0:((W-F+1)/2)*DATA_WIDTH-1] outputConvUnits; // output of the conv units and input to the row selector

reg internalReset;
wire [0:(((W-F+1)/2)*D*F*F*DATA_WIDTH)-1] receptiveField; // array of the matrices to be sent to conv units


integer counter, outputCounter;
//counter: number of clock cycles need for the conv unit to finsish
//outputCounter: index to map the output of the conv units to the output of the module

reg [5:0] rowNumber, column; 
//rowNumber: determines the row that is calculated by the conv units
//column: determines if we are calculating the first or the second 14 pixels of the output row

RFselector
#(
	.DATA_WIDTH(DATA_WIDTH),
	.D(D),
	.H(H),
	.W(W),
	.F(F)
) RF
(
	.image(image),
	.rowNumber(rowNumber),
	.column(column),
	.receptiveField(receptiveField)
);

genvar n;

generate //generating n convolution units where n is half the number of pixels in one row of the output image
	for (n = 0; n < (H-F+1)/2; n = n + 1) begin 
		convUnit
		#(
			.D(D),
			.F(F)
		) CU
		(
			.clk(clk),
			.reset(internalReset),
			.image(receptiveField[n*D*F*F*DATA_WIDTH+:D*F*F*DATA_WIDTH]),
			.filter(filter),
			.result(outputConvUnits[n*DATA_WIDTH+:DATA_WIDTH])
		);
	end
endgenerate

always @ (posedge clk or posedge reset) begin
	if (reset == 1'b1) begin
		internalReset = 1'b1;
		rowNumber = 0;
		column = 0;
		counter = 0;
		outputCounter = 0;
	end else if (rowNumber < H-F+1) begin
		if (counter == D*F*F+2) begin //The conv unit finishes ater 1*5*5+2 clock cycles
			outputCounter = outputCounter + 1;
			counter = 0;
			internalReset = 1'b1;
			if (column == 0) begin
				column = (H-F+1)/2;
			end else begin
				rowNumber = rowNumber + 1;
				column = 0;
			end
		end else begin
			internalReset = 0;
			counter = counter + 1;
		end
	end
end

always @ (*) begin
	outputConv[outputCounter*((W-F+1)/2)*DATA_WIDTH+:((W-F+1)/2)*DATA_WIDTH] = outputConvUnits;
end

endmodule

convLayerMulti模块

这个模块例化了2个convLayerSingle模块，实现了对两个filter核的并行卷积。

`timescale 100 ns / 10 ps

module convLayerMulti(clk,reset,image,filters,outputConv);

parameter DATA_WIDTH = 16;
parameter D = 1; //Depth of image and filter
parameter H = 32; //Height of image
parameter W = 32; //Width of image
parameter F = 5; //Size of filter
parameter K = 6; //Number of filters applied

input clk, reset;
input [0:D*H*W*DATA_WIDTH-1] image;
input [0:K*D*F*F*DATA_WIDTH-1] filters;
output reg [0:K*(H-F+1)*(W-F+1)*DATA_WIDTH-1] outputConv;

reg [0:2*D*F*F*DATA_WIDTH-1] inputFilters;
wire [0:2*(H-F+1)*(W-F+1)*DATA_WIDTH-1] outputSingleLayers;

reg internalReset;

integer filterSet, counter, outputCounter;

genvar i;

generate
	for (i = 0; i < 2; i = i + 1) begin 
		convLayerSingle #(
		  .DATA_WIDTH(DATA_WIDTH),
		  .D(D),
		  .H(H),
		  .W(W),
		  .F(F)
		) UUT 
		(
			.clk(clk),
	     		.reset(internalReset),
	     		.image(image),
	    		.filter(inputFilters[i*D*F*F*DATA_WIDTH+:D*F*F*DATA_WIDTH]),
	     		.outputConv(outputSingleLayers[i*(H-F+1)*(W-F+1)*DATA_WIDTH+:(H-F+1)*(W-F+1)*DATA_WIDTH])
      		);
  	end
endgenerate

always @ (posedge clk or posedge reset) begin
	if (reset == 1'b1) begin
		internalReset = 1'b1;
		filterSet = 0;
		counter = 0;
		outputCounter = 0;		
	end else if (filterSet < K/2) begin
		if (counter == ((((H-F+1)*(W-F+1))/((H-F+1)/2))*(D*F*F+3)+1)) begin
			outputCounter = outputCounter + 1;
			counter = 0;
			internalReset = 1'b1;
			filterSet = filterSet + 1;
		end else begin
			internalReset = 0;
			counter = counter + 1;
		end
	end
end

always @ (*) begin
	inputFilters = filters[filterSet*2*D*F*F*DATA_WIDTH+:2*D*F*F*DATA_WIDTH];
	outputConv[outputCounter*2*(H-F+1)*(W-F+1)*DATA_WIDTH+:2*(H-F+1)*(W-F+1)*DATA_WIDTH] = outputSingleLayers;
end

endmodule

至此，一个卷积层所需的实现单元就完成了。剩下的问题就是怎么让卷积的image和filter数据对应输入并计算。

第三个卷积层后接FC层，每个核卷积只输出一个值（因为H=W=F），因此只需convLayerSingle即可（输出降了一维）。