Merge pull request #2 from atomiechen/dev

Bump version to 0.4.1

matsense/blank_template.yaml

@@ -77,7 +77,10 @@ process:
   R0_RECI: ~
   ## convert voltage to resistance: true
   convert: ~
-
+  ## convert voltage to resistance's opposite number (otherwise reciprocal): false
+  resi_opposite: ~
+  ## convert voltage to delta_R / R0: true
+  resi_delta: ~
   ### server data processing
   ## no filtering and calibration
   raw: ~
@@ -99,6 +102,10 @@ process:
   cali_frames: ~
   ## calibration frame window size, 0 for static and >0 for dynamic: 0, 10000
   cali_win_size: ~
+  ## calibration threshold, max data above it maintain, others add to cali_win
+  cali_threshold: 2
+  ## calibration buffer size: 5
+  cali_win_buffer_size: 5
   ## intermediate result: 0, 1, 2
     ## 0: convert voltage to reciprocal resistance
     ## 1: convert & spatial filter

matsense/data.py

@@ -90,6 +90,8 @@ def main():
 			V0=config['process']['V0'],
 			R0_RECI=config['process']['R0_RECI'],
 			convert=config['process']['convert'],
+			resi_opposite=config['process']['resi_opposite'],
+			resi_delta=config['process']['resi_delta'],
 			mask=config['sensor']['mask'],
 			filter_spatial=config['process']['filter_spatial'],
 			filter_spatial_cutoff=config['process']['filter_spatial_cutoff'],
@@ -99,6 +101,8 @@ def main():
 			rw_cutoff=config['process']['rw_cutoff'],
 			cali_frames=config['process']['cali_frames'],
 			cali_win_size=config['process']['cali_win_size'],
+            cali_threshold=config['process']['cali_threshold'],
+            cali_win_buffer_size=config['process']['cali_win_buffer_size'],
 			intermediate=config['process']['intermediate'],
 		)
 		my_processor = Processor(

matsense/process/data_handler.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from math import exp, hypot, pi, sin
 import numpy as np
-
+from collections import deque
 from ..tools import check_shape
 
 
@@ -44,8 +44,11 @@ class DataHandlerPressure(DataHandler):
 
 	## voltage-resistance conversion
 	my_convert = True
+	my_resi_opposite = True
+	my_resi_delta = True # count pressure as delta_R / R0
 	V0 = 255
 	R0_RECI = 1  ## a constant to multiply the value
+	R0_START = 0 ## start resistance
 
 	## process parameters
 	my_SF_D0 = 3.5
@@ -54,6 +57,8 @@ class DataHandlerPressure(DataHandler):
 	my_LP_W = 0.04
 	my_INIT_CALI_FRAMES = 200
 	my_WIN_SIZE = 0
+	my_WIN_BUFFER_SIZE = 5
+	my_CALI_THRESHOLD = 3
 
 	def __init__(self, **kwargs):
 		self.mask = None
@@ -72,8 +77,9 @@ def config(self, *, n, raw=None, V0=None, R0_RECI=None, convert=None,
 		mask=None, filter_spatial=None, filter_spatial_cutoff=None, 
 		butterworth_order=None, filter_temporal=None, 
 		filter_temporal_size=None, rw_cutoff=None, cali_frames=None, 
-		cali_win_size=None, 
-		intermediate=None,
+		cali_win_size=None, cali_win_buffer_size=None,
+		intermediate=None, 
+		resi_opposite=None, resi_delta=None, cali_threshold=None,
 		**kwargs):
 
 		self.n = check_shape(n)
@@ -88,6 +94,10 @@ def config(self, *, n, raw=None, V0=None, R0_RECI=None, convert=None,
 			self.R0_RECI = R0_RECI
 		if convert is not None:
 			self.my_convert = convert
+		if resi_opposite is not None:
+			self.my_resi_opposite = resi_opposite
+		if resi_delta is not None:
+			self.my_resi_delta = resi_delta
 		if mask is not None:
 			self.mask = mask
 		if filter_spatial is not None:
@@ -114,6 +124,10 @@ def config(self, *, n, raw=None, V0=None, R0_RECI=None, convert=None,
 			self.my_WIN_SIZE = cali_win_size
 		if intermediate is not None:
 			self.intermediate = intermediate
+		if cali_threshold is not None:
+			self.my_CALI_THRESHOLD = cali_threshold
+		if cali_win_buffer_size is not None:
+			self.my_WIN_BUFFER_SIZE = cali_win_buffer_size
 
 	@staticmethod
 	def calReciprocalResistance(voltage, v0, r0_reci):
@@ -127,10 +141,27 @@ def calReci_numpy_array(np_array, v0, r0_reci):
 		np_array /= (v0 - np_array)
 		np_array *= r0_reci
 
+	@staticmethod
+	def calOppo_numpy_array(np_array, v0, r0_reci):
+		np_array[np_array >= v0] = 0
+		np_array /= (v0 - np_array)
+		np_array *= r0_reci
+		np_array[np_array != 0] = -1 / np_array[np_array != 0]
+
 	@staticmethod
 	def getNextIndex(idx, size):
 		return (idx+1) if idx != (size-1) else 0
 
+	def calDelta_numpy_array(self, np_array, v0, r0_reci):
+		np_array[np_array >= v0] = 0
+		np_array /= (v0 - np_array)
+		np_array *= r0_reci
+		np_array[np_array != 0] = 1 / np_array[np_array != 0]
+		# print(np_array)
+		np_array[np_array!=0] = abs(np_array[np_array!=0] - self.R0_START[np_array!=0]) / self.R0_START[np_array!=0]
+		np_array *= 10
+		# print(np_array)
+
 	def handle_raw_frame(self, data):
 		self.data_tmp = data
 		self.data_reshape = self.data_tmp.reshape(self.n[0], self.n[1])
@@ -139,14 +170,20 @@ def handle_raw_frame(self, data):
 		if self.my_convert:
 			# for i in range(self.total):
 			# 	self.data_tmp[i] = self.calReciprocalResistance(self.data_tmp[i], self.V0, self.R0_RECI)
-			self.calReci_numpy_array(self.data_tmp, self.V0, self.R0_RECI)
+			if self.my_resi_opposite:
+				self.calOppo_numpy_array(self.data_tmp, self.V0, self.R0_RECI)
+			elif self.my_resi_delta:
+				self.calDelta_numpy_array(self.data_tmp, self.V0, self.R0_RECI)
+			else:
+				self.calReci_numpy_array(self.data_tmp, self.V0, self.R0_RECI)
 
 	def prepare(self, generator):
 		self.generator = generator
 		if not self.my_raw:
+			self.cal_start_R0() # called if my_resi_delta is True
 			self.prepare_spatial()
 			self.prepare_temporal()
-			self.prepare_cali()
+			self.prepare_cali() # called if my_INIT_CALI_FRAMES > 0 and my_resi_delta is False
 
 	def handle(self, data, data_inter=None):
 		self.handle_raw_frame(data)
@@ -158,19 +195,23 @@ def handle(self, data, data_inter=None):
 
 		if not self.my_raw:
 			self.filter()
-			self.calibrate()
+			if not self.my_resi_delta:
+				self.calibrate()
 		elif self.data_inter is not None and self.intermediate != 0:
 			## output intermediate result, making data_inter not blank
 			self.data_inter[:] = self.data_tmp[:]
 
 	def prepare_cali(self):
-		if self.my_INIT_CALI_FRAMES <= 0:
+		if self.my_INIT_CALI_FRAMES <= 0 or self.my_resi_delta:
 			return
 
 		print("Initiating calibration...")
 		self.data_zero = np.zeros(self.total, dtype=float)
 		self.data_win = np.zeros((self.my_WIN_SIZE, self.total), dtype=float)
 		self.win_frame_idx = 0
+		self.data_win_buffer = deque(maxlen=self.my_WIN_BUFFER_SIZE)
+		self.win_buffer_frame_idx = 0
+		self.need_to_clean_buffer = False # if True then clean the buffer at next full time
 		## for preparing calibration
 		frame_cnt = 0
 		# ## accumulate data
@@ -186,6 +227,13 @@ def prepare_cali(self):
 		self.data_zero /= frame_cnt
 		## calculate data_win
 		self.data_win[:] = self.data_zero
+		for _ in range(self.my_WIN_BUFFER_SIZE):
+			self.data_win_buffer.append(self.data_zero)
+        ## save data_min in last WIN_SIZE frames
+		# self.data_min = deque(maxlen=self.my_WIN_SIZE)
+		# self.data_min.append((self.data_zero.copy(), self.win_frame_idx))
+		self.win_frame_idx = self.getNextIndex(self.win_frame_idx, self.my_WIN_SIZE)
+		self.win_buffer_frame_idx = self.getNextIndex(self.win_buffer_frame_idx, self.my_WIN_BUFFER_SIZE)
 
 	def calibrate(self):
 		if self.my_INIT_CALI_FRAMES <= 0:
@@ -198,10 +246,51 @@ def calibrate(self):
 		## adjust window if using dynamic window
 		if self.my_WIN_SIZE > 0:
 			## update data_zero (zero position) and data_win (history data)
-			self.data_zero += (stored - self.data_win[self.win_frame_idx]) / self.my_WIN_SIZE
-			self.data_win[self.win_frame_idx] = stored
-			## update frame index
-			self.win_frame_idx = self.getNextIndex(self.win_frame_idx, self.my_WIN_SIZE)
+
+			## use average number as data_zero
+			# self.data_zero += (stored - self.data_win[self.win_frame_idx]) / self.my_WIN_SIZE
+
+			## use min number as data_zero
+			# if (self.data_min[0][1] == self.win_frame_idx):
+			# 	self.data_min.popleft()
+			# self.data_zero = self.data_min[0][0]
+			# while (self.data_min[-1][0].sum() > stored.sum()):
+			# 	self.data_min.pop()
+			# 	if (len(self.data_min) == 0):
+			# 		break
+			# self.data_min.append((stored, self.win_frame_idx))
+
+			## use average number as data_zero, but delete the odd ones
+			add_to_data_zero = True
+			for id, stored_data in enumerate(stored):
+				data_at_one_point = stored_data - self.data_zero[id]
+				if (data_at_one_point > self.my_CALI_THRESHOLD):
+					# pressure over data_zero + threshold
+					add_to_data_zero = False
+					self.need_to_clean_buffer = True
+					if len(self.data_win_buffer) == self.my_WIN_BUFFER_SIZE:
+						self.data_win_buffer.clear()
+					break
+			if (add_to_data_zero):
+				if len(self.data_win_buffer) < self.my_WIN_BUFFER_SIZE:
+					self.data_win_buffer.append(stored)
+				else:
+					if self.need_to_clean_buffer:
+						self.data_win_buffer.clear()
+						self.need_to_clean_buffer = False
+						self.data_win_buffer.append(stored)
+					else:
+						cur_data = self.data_win_buffer.popleft()
+						self.data_win_buffer.append(stored)
+						## store in data_win
+						self.data_zero += (cur_data - self.data_win[self.win_frame_idx]) / self.my_WIN_SIZE
+						self.data_win[self.win_frame_idx] = cur_data
+						self.win_frame_idx = self.getNextIndex(self.win_frame_idx, self.my_WIN_SIZE)
+
+			# ## store in data_win
+			# self.data_win[self.win_frame_idx] = stored
+			# ## update frame index
+			# self.win_frame_idx = self.getNextIndex(self.win_frame_idx, self.my_WIN_SIZE)
 
 	def filter(self):
 		self.spatial_filter()
@@ -267,6 +356,26 @@ def temporal_filter(self):
 		## update to next index
 		self.filter_frame_idx = self.getNextIndex(self.filter_frame_idx, self.my_LP_SIZE-1)
 
+	def cal_start_R0(self):
+		if not self.my_resi_delta:
+			return
+		# calcualte average start R0
+		frame_cnt = 0
+		data = next(self.generator)
+		r0 = np.zeros(data.shape)
+		R0_AVE_TIMES = 10
+		# ## accumulate data
+		while frame_cnt < R0_AVE_TIMES: # use 10 frames to calculate
+			data = next(self.generator)
+			data = data.reshape(self.n[0], self.n[1])
+			if self.mask is not None:
+				data *= self.mask
+			self.calOppo_numpy_array(data, self.V0, self.R0_RECI)
+			r0 += -1 * data.flatten()
+			frame_cnt += 1
+		self.R0_START = r0 / R0_AVE_TIMES
+		print("start R0: ", self.R0_START)
+
 	def prepare_spatial(self):
 		def gaussianLP(distance):
 			return exp(-distance**2/(2*(self.my_SF_D0)**2))
@@ -331,15 +440,16 @@ def print_proc(self):
 				arg_list_t["kernel size"] = self.my_LP_SIZE
 
 			## output to screen
+			print(f"  - Resistance mode: {'delta_R / R0' if self.my_resi_delta else ('-R' if self.my_resi_opposite else '1/R')}")
 			print(f"  - Spatial filter: {self.my_filter_spatial.value}")
 			for value, key in arg_list_s.items():
 				print(f"    {value}: {key}")
 			print(f"  - Temporal filter: {self.my_filter_temporal.value}")
 			for value, key in arg_list_t.items():
 				print(f"    {value}: {key}")
 
-			print(f"  - Calibration: {'No' if self.my_INIT_CALI_FRAMES == 0 else ''}")
-			if self.my_INIT_CALI_FRAMES != 0:
+			print(f"  - Calibration: {'No' if self.my_INIT_CALI_FRAMES == 0 or self.my_resi_delta else ''}")
+			if self.my_INIT_CALI_FRAMES != 0 and not self.my_resi_delta:
 				print(f"    Initializing frames:     {self.my_INIT_CALI_FRAMES}")
 				if self.my_WIN_SIZE == 0:
 					print("    Static calibration")

matsense/server.py

@@ -77,6 +77,8 @@ def task_serial(paras):
 			V0=paras['config']['process']['V0'],
 			R0_RECI=paras['config']['process']['R0_RECI'],
 			convert=paras['config']['process']['convert'],
+			resi_opposite=paras['config']['process']['resi_opposite'],
+			resi_delta=paras['config']['process']['resi_delta'],
 			mask=paras['config']['sensor']['mask'],
 			filter_spatial=paras['config']['process']['filter_spatial'],
 			filter_spatial_cutoff=paras['config']['process']['filter_spatial_cutoff'],
@@ -86,6 +88,8 @@ def task_serial(paras):
 			rw_cutoff=paras['config']['process']['rw_cutoff'],
 			cali_frames=paras['config']['process']['cali_frames'],
 			cali_win_size=paras['config']['process']['cali_win_size'],
+            cali_threshold=paras['config']['process']['cali_threshold'],
+            cali_win_buffer_size=paras['config']['process']['cali_win_buffer_size'],
 			pipe_conn=paras['pipe_proc'],
 			copy_tags=False,
 			imu=paras['config']['serial']['imu'],
@@ -147,6 +151,8 @@ def task_file(paras):
 		V0=paras['config']['process']['V0'],
 		R0_RECI=paras['config']['process']['R0_RECI'],
 		convert=paras['config']['process']['convert'],
+		resi_opposite=paras['config']['process']['resi_opposite'],
+		resi_delta=paras['config']['process']['resi_delta'],
 		mask=paras['config']['sensor']['mask'],
 		filter_spatial=paras['config']['process']['filter_spatial'],
 		filter_spatial_cutoff=paras['config']['process']['filter_spatial_cutoff'],
@@ -156,6 +162,8 @@ def task_file(paras):
 		rw_cutoff=paras['config']['process']['rw_cutoff'],
 		cali_frames=paras['config']['process']['cali_frames'],
 		cali_win_size=paras['config']['process']['cali_win_size'],
+        cali_threshold=paras['config']['process']['cali_threshold'],
+        cali_win_buffer_size=paras['config']['process']['cali_win_buffer_size'],
 		pipe_conn=None,
 		output_filename=paras['config']['data']['out_filename'],
 		copy_tags=True,
@@ -185,6 +193,10 @@ def prepare_config(args):
 		config['connection']['server_address'] = args.address
 	if config['process']['convert'] is None or hasattr(args, 'no_convert'+DEST_SUFFIX):
 		config['process']['convert'] = not args.no_convert
+	if config['process']['resi_opposite'] is None or hasattr(args, 'resi_opposite'+DEST_SUFFIX):
+		config['process']['resi_opposite'] = args.resi_opposite
+	if config['process']['resi_delta'] is None or hasattr(args, 'resi_delta'+DEST_SUFFIX):
+		config['process']['resi_delta'] = args.resi_delta
 	if config['visual']['zlim'] is None or hasattr(args, 'zlim'+DEST_SUFFIX):
 		config['visual']['zlim'] = args.zlim
 	if config['visual']['fps'] is None or hasattr(args, 'fps'+DEST_SUFFIX):
@@ -311,6 +323,8 @@ def main():
 	parser.add_argument('-u', '--udp', dest='udp', action=make_action('store_true'), default=UDP, help="use UDP protocol")
 	parser.add_argument('-r', '--raw', dest='raw', action=make_action('store_true'), default=False, help="raw data mode")
 	parser.add_argument('-nc', '--no_convert', dest='no_convert', action=make_action('store_true'), default=NO_CONVERT, help="do not apply voltage-resistance conversion")
+	parser.add_argument('-ro', '--resi_opposite', dest='resi_opposite', action=make_action('store_true'), default=False, help="turn voltage to the opposite of resistance")
+	parser.add_argument('-rd', '--resi_delta', dest='resi_delta', action=make_action('store_true'), default=True, help="turn voltage to the delta_R / R0")
 	parser.add_argument('-v', '--visualize', dest='visualize', action=make_action('store_true'), default=False, help="enable visualization")
 	parser.add_argument('-z', '--zlim', dest='zlim', action=make_action('store'), default=ZLIM, type=float, help="z-axis limit")
 	parser.add_argument('-f', dest='fps', action=make_action('store'), default=FPS, type=int, help="frames per second")

matsense/serverkit/proc.py

@@ -151,7 +151,7 @@ def gen_imu():
 						ret = (1, config_new)
 						break
 					if flag in (FLAG.FLAG_REC_DATA, FLAG.FLAG_REC_RAW):
-						self.record_raw = True if flag == FLAG.FLAG_REC_RAW else True
+						self.record_raw = True if flag == FLAG.FLAG_REC_RAW else False
 						filename = msg[1]
 						if filename == "":
 							if flag == FLAG.FLAG_REC_RAW:

setup.py

@@ -27,7 +27,7 @@
     # For a discussion on single-sourcing the version across setup.py and the
     # project code, see
     # https://packaging.python.org/guides/single-sourcing-package-version/
-    version='0.4.0',  # Required
+    version='0.4.1',  # Required
 
     # This is a one-line description or tagline of what your project does. This
     # corresponds to the "Summary" metadata field:
@@ -158,7 +158,7 @@
         "pyserial>=3.5",
         "PyYAML>=5.4.1",
         "pyparsing>=2.4.7",
-        "matplotlib>=3.3",
+        "matplotlib>=3.3,<=3.4",
     ],
 
     # List additional groups of dependencies here (e.g. development