From b8ec7955e60e2cf577c7c9d96e34f6b3bd0e90c1 Mon Sep 17 00:00:00 2001 From: bghira Date: Mon, 28 Aug 2023 08:25:32 -0700 Subject: [PATCH 1/3] S3DataBackend: fix connection pool size for backend --- helpers/data_backend/aws.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helpers/data_backend/aws.py b/helpers/data_backend/aws.py index ca4367df..027aeeb6 100644 --- a/helpers/data_backend/aws.py +++ b/helpers/data_backend/aws.py @@ -3,6 +3,7 @@ from torch import Tensor from pathlib import PosixPath import concurrent.futures +from botocore.config import Config from helpers.data_backend.base import BaseDataBackend loggers_to_silence = [ @@ -49,10 +50,12 @@ def __init__( extra_args = { "endpoint_url": endpoint_url, } + s3_config = Config(max_pool_connections=100) self.client = boto3.client( "s3", aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, + config=s3_config, **extra_args, ) From 64da7b00a35047170855f9bd3db5377d052c2673 Mon Sep 17 00:00:00 2001 From: bghira Date: Mon, 28 Aug 2023 08:32:32 -0700 Subject: [PATCH 2/3] Remove diffusers log outputs --- helpers/log_format.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helpers/log_format.py b/helpers/log_format.py index 6dfc391d..d0e41e6c 100644 --- a/helpers/log_format.py +++ b/helpers/log_format.py @@ -32,5 +32,8 @@ def format(self, record): if not logger.handlers: logger.addHandler(new_handler) +forward_logger = logging.getLogger('unet_2d_condition') +forward_logger.setLevel(logging.WARNING) + pil_logger = logging.getLogger("PIL") pil_logger.setLevel(logging.INFO) From cb9d62f946dd616b864984a97b1b2aa1d88861a2 Mon Sep 17 00:00:00 2001 From: bghira Date: Mon, 28 Aug 2023 08:36:51 -0700 Subject: [PATCH 3/3] Remove diffusers log outputs --- helpers/log_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/log_format.py b/helpers/log_format.py index d0e41e6c..c9e13b9c 100644 --- a/helpers/log_format.py +++ b/helpers/log_format.py @@ -32,7 +32,7 @@ def format(self, record): if not logger.handlers: logger.addHandler(new_handler) -forward_logger = logging.getLogger('unet_2d_condition') +forward_logger = logging.getLogger('diffusers.models.unet_2d_condition') forward_logger.setLevel(logging.WARNING) pil_logger = logging.getLogger("PIL")