Merge pull request #2881 from Drakkar-Software/dev

Update master

README.md

@@ -121,23 +121,30 @@ Please feel free to read the source code and understand the mechanism of this bo
 ## License
 GNU General Public License v3.0 or later.
 
-See [GPL-3.0 LICENSE](https://octobot.click/gh-license) to see the full text.
+See [GPL-3.0 LICENSE](https://github.com/Drakkar-Software/OctoBot/blob/master/LICENSE) to see the full text.
 
 ## Institutionals
-If you are an institutional interested by a white label solution, a commercial license, or custom development to suit your specific needs, have a look at [our institutional offers](https://www.octobot.cloud/features/institutional-investors?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=for_institutionals). 
+If you are an institutional interested in a commercial license or custom development to suit your specific needs please contact us at <a href="mailto:[email protected]">[email protected]</a>. 
+
+## Market making
+A version of OctoBot adapted for market making is available at [business.octobot.cloud](https://business.octobot.cloud/). Feel free to contact us if you have any questions about it.
 
 ## Sponsors
 <table>
 <tr>
-<td><a href="https://www.jetbrains.com" target="_blank">JetBrains</a> with PyCharm Pro.</td>
+<td>Thank you <a href="https://www.jetbrains.com" target="_blank">JetBrains</a> with PyCharm Pro for allowing us to develop the new features of OctoBot under the best conditions.</td>
 <td><a href="https://www.jetbrains.com" target="_blank"><p align="center"><img src="https://resources.jetbrains.com/storage/products/pycharm/img/meta/pycharm_logo_300x300.png" width="100px"></p></a></td>
 </tr>
 <tr>
-<td>Special thanks to <a href="https://www.chatwoot.com/" target="_blank">Chatwoot</a> for providing their customer engagement platform.</td>
+<td>Special thanks to <a href="https://www.chatwoot.com/" target="_blank">Chatwoot</a> for helping us assist the users of OctoBot.</td>
 <td><a href="https://github.com/chatwoot/chatwoot" target="_blank"><p align="center"><img src="https://raw.githubusercontent.com/chatwoot/chatwoot/develop/public/brand-assets/logo.svg" width="500px"></p></a></td>
 </tr>
 <tr>
-<td>Special thanks to <a href="https://www.scaleway.com" target="_blank">Scaleway</a> for hosting OctoBot's cloud services.</td>
+<td>Huge thank you to <a href="https://www.scaleway.com" target="_blank">Scaleway</a> for hosting OctoBot's cloud services.</td>
 <td><a href="https://www.scaleway.com" target="_blank"><p align="center"><img src="https://raw.githubusercontent.com/Drakkar-Software/OctoBot/assets/scaleway.svg" width="500px"></p></a></td>
 </tr>
+<tr>
+<td>A big thank you to <a href="https://sentry.io/welcome/" target="_blank">Sentry</a> for helping us identify and understand errors in OctoBot to make it better.</td>
+<td><a href="https://sentry.io/welcome/" target="_blank"><p align="center"><img src="https://raw.githubusercontent.com/Drakkar-Software/OctoBot/assets/sentry.png" width="500px"></p></a></td>
+</tr>
 </table>

octobot/community/supabase_backend/community_supabase_client.py

@@ -761,19 +761,25 @@ async def _paginated_fetch_historical_data(
         self, client, table_name: str, select: str, matcher: dict,
         first_open_time: float, last_open_time: float
     ) -> list:
-        def request_factory(table: postgrest.AsyncRequestBuilder, select_count):
-            return (
-                table.select(select, count=select_count)
-                .match(matcher).gte(
+        def request_factory(table: postgrest.AsyncRequestBuilder, select_count, last_fetched_row):
+            request = table.select(select, count=select_count)
+            if last_fetched_row is None:
+                request = request.match(matcher).gte(
                     "timestamp", self.get_formatted_time(first_open_time)
-                ).lte(
+                )
+            else:
+                request = request.match(matcher).gt(
+                    "timestamp", last_fetched_row["timestamp"]
+                )
+            return (
+                request.lte(
                     "timestamp", self.get_formatted_time(last_open_time)
                 ).order(
                     "timestamp", desc=False
                 )
             )
 
-        return await self.paginated_fetch(
+        return await self.cursor_paginated_fetch(
             client, table_name, request_factory
         )
 
@@ -820,6 +826,47 @@ async def paginated_fetch(
             )
         return total_elements
 
+    async def cursor_paginated_fetch(
+        self,
+        client,
+        table_name: str,
+        request_factory: typing.Callable[
+            [postgrest.AsyncRequestBuilder, postgrest.types.CountMethod, typing.Optional[dict]],
+            postgrest.AsyncSelectRequestBuilder
+        ],
+    ) -> list:
+        max_size_per_fetch = 0
+        total_elements = []
+        request_count = 0
+        total_elements_count = 0
+        while request_count < self.MAX_PAGINATED_REQUESTS_COUNT:
+            request = request_factory(
+                client.table(table_name),
+                None if total_elements_count else postgrest.types.CountMethod.exact,
+                total_elements[-1] if total_elements else None
+            )
+            result = await request.execute()
+            fetched_elements = result.data
+            total_elements_count = total_elements_count or result.count   # don't change total count within iteration
+            total_elements += fetched_elements
+            if(
+                len(fetched_elements) == 0 or   # nothing to fetch
+                len(fetched_elements) < max_size_per_fetch or   # fetched the last elements
+                len(fetched_elements) == total_elements_count   # finished fetching
+            ):
+                # fetched everything
+                break
+            if max_size_per_fetch == 0:
+                max_size_per_fetch = len(fetched_elements)
+            request_count += 1
+
+        if request_count == self.MAX_PAGINATED_REQUESTS_COUNT:
+            commons_logging.get_logger(self.__class__.__name__).info(
+                f"Paginated fetch error on {table_name} with request_factory: {request_factory.__name__}: "
+                f"too many requests ({request_count}), fetched: {len(total_elements)} elements"
+            )
+        return total_elements
+
     def _format_gpt_signals(self, signals: list):
         return {
             self.get_parsed_time(signal["timestamp"]).timestamp(): signal["signal"]["content"]

requirements.txt

@@ -1,6 +1,6 @@
 # Drakkar-Software requirements
 OctoBot-Commons==1.9.70
-OctoBot-Trading==2.4.152
+OctoBot-Trading==2.4.153
 OctoBot-Evaluators==1.9.7
 OctoBot-Tentacles-Manager==2.9.16
 OctoBot-Services==1.6.23