From 6ff002371b0a5c283be96536fd91abebbb01f1e6 Mon Sep 17 00:00:00 2001
From: Naoya Yamaguchi <708yamaguchi@gmail.com>
Date: Mon, 23 May 2022 13:49:33 +0900
Subject: [PATCH 1/3] [mongodb_store] Throttle noisy error log

---
 mongodb_store/launch/mongodb_store_inc.launch |  2 ++
 mongodb_store/scripts/replicator_node.py      | 11 +++++++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/mongodb_store/launch/mongodb_store_inc.launch b/mongodb_store/launch/mongodb_store_inc.launch
index 1dce50f..22e02a5 100644
--- a/mongodb_store/launch/mongodb_store_inc.launch
+++ b/mongodb_store/launch/mongodb_store_inc.launch
@@ -20,6 +20,7 @@
   <arg name="repl_set" default="rs0" />
   <arg name="queue_size" default="100" />
   <arg name="bind_to_host" default="false" />
+  <arg name="logerr_period" default="0" />
 
   <arg name="use_localdatacenter" default="true" />
 
@@ -66,6 +67,7 @@
   <group if="$(arg launch_replicator)">
     <node name="replicator_node" pkg="mongodb_store" type="replicator_node.py" machine="$(arg machine)">
       <param name="replicator_dump_path" value="$(arg replicator_dump_path)"/>
+      <param name="logerr_period" value="$(arg logerr_period)"/>
     </node>
   </group>
 
diff --git a/mongodb_store/scripts/replicator_node.py b/mongodb_store/scripts/replicator_node.py
index 251189d..e2c283c 100755
--- a/mongodb_store/scripts/replicator_node.py
+++ b/mongodb_store/scripts/replicator_node.py
@@ -29,11 +29,12 @@
 
 
 class Process(object):
-    def __init__(self, cmd):
+    def __init__(self, cmd, logerr_period):
         self.lock = Lock()
         self.cmd = cmd
         self.process = None
         self.threads = []
+        self.logerr_period = logerr_period
 
     def _message_callback(self, stream, callback):
         buf = str()
@@ -126,7 +127,7 @@ def on_stdout(self, msg):
         rospy.loginfo('[{}] {}'.format(self.cmd[0], msg))
 
     def on_stderr(self, msg):
-        rospy.logerr('[{}] {}'.format(self.cmd[0], msg))
+        rospy.logerr_throttle(self.logerr_period, '[{}] {}'.format(self.cmd[0], msg))
 
 
 class MongoProcess(Process):
@@ -151,7 +152,7 @@ def progress(self):
 
 
 class MongoDumpProcess(MongoProcess):
-    def __init__(self, host, port, db, collection, dump_path, less_time=None, query=None):
+    def __init__(self, host, port, db, collection, dump_path, less_time=None, query=None, logerr_period=0):
         cmd = [
             'mongodump', '--verbose', '-o', dump_path,
             '--host', host, '--port', str(port),
@@ -195,6 +196,7 @@ def __init__(self):
         if use_connection_string:
             use_daemon = True
             rospy.loginfo('Using connection string: %s', connection_string)
+        self.logerr_period = rospy.get_param('logerr_period', 0)
 
         self.connection_string = ''
 
@@ -343,7 +345,8 @@ def do_dump(self, collection, master, less_time_time=None, db='message_store', q
 
         self.dump_process = MongoDumpProcess(host=host, port=port, db=db, collection=collection,
                                              dump_path=self.dump_path,
-                                             less_time=less_time_time, query=query)
+                                             less_time=less_time_time, query=query,
+                                             logerr_period=self.logerr_period)
         self.dump_process.start()
         self.dump_process.wait()
         self.dump_process = None

From af454a9c146216b67945c6ca0edb63f1b79f7be7 Mon Sep 17 00:00:00 2001
From: Naoya Yamaguchi <708yamaguchi@gmail.com>
Date: Tue, 24 May 2022 09:25:39 +0900
Subject: [PATCH 2/3] [mongodb_store] Fix missing init arguments

---
 mongodb_store/scripts/replicator_node.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/mongodb_store/scripts/replicator_node.py b/mongodb_store/scripts/replicator_node.py
index e2c283c..5f092bb 100755
--- a/mongodb_store/scripts/replicator_node.py
+++ b/mongodb_store/scripts/replicator_node.py
@@ -170,11 +170,11 @@ def __init__(self, host, port, db, collection, dump_path, less_time=None, query=
             query = json_util.dumps(query)
             cmd += ['--query', query]
 
-        super(MongoDumpProcess, self).__init__(cmd=cmd)
+        super(MongoDumpProcess, self).__init__(cmd=cmd, logerr_period=logerr_period)
 
 
 class MongoRestoreProcess(MongoProcess):
-    def __init__(self, host, port, dump_path, db=None, collection=None):
+    def __init__(self, host, port, dump_path, db=None, collection=None, logerr_period=0):
         cmd = [
             'mongorestore', '--verbose', '--host', host, '--port', str(port),
         ]
@@ -183,7 +183,7 @@ def __init__(self, host, port, dump_path, db=None, collection=None):
         if collection is not None:
             cmd += [ '--collection', collection]
         cmd += [dump_path]
-        super(MongoRestoreProcess, self).__init__(cmd=cmd)
+        super(MongoRestoreProcess, self).__init__(cmd=cmd, logerr_period=logerr_period)
 
 
 class Replicator(object):
@@ -317,7 +317,7 @@ def do_restore(self, extras, db='message_store'):
             except pymongo.errors.ServerSelectionTimeoutError:
                 rospy.logerr('Failed to connect to the extra server {}'.format(extra))
                 continue
-            self.restore_process = MongoRestoreProcess(host=host, port=port, dump_path=self.dump_path)
+            self.restore_process = MongoRestoreProcess(host=host, port=port, dump_path=self.dump_path, logerr_period=self.logerr_period)
             self.restore_process.start()
             self.restore_process.wait()
             self.restore_process = None

From 269d642df964f548eeba23a4d6db5c70b96abf86 Mon Sep 17 00:00:00 2001
From: Naoya Yamaguchi <708yamaguchi@gmail.com>
Date: Tue, 24 May 2022 14:40:04 +0900
Subject: [PATCH 3/3] [mongodb_store] Add namespace to logerr_period rosparam

---
 mongodb_store/scripts/replicator_node.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongodb_store/scripts/replicator_node.py b/mongodb_store/scripts/replicator_node.py
index 5f092bb..89685b0 100755
--- a/mongodb_store/scripts/replicator_node.py
+++ b/mongodb_store/scripts/replicator_node.py
@@ -196,7 +196,7 @@ def __init__(self):
         if use_connection_string:
             use_daemon = True
             rospy.loginfo('Using connection string: %s', connection_string)
-        self.logerr_period = rospy.get_param('logerr_period', 0)
+        self.logerr_period = rospy.get_param('~logerr_period', 0)
 
         self.connection_string = ''