From 19d1da2aa283bb603933ad9ee5550a9f7184ce5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Juozas=20Janu=C5=A1aitis?= Date: Fri, 14 Jun 2024 09:53:48 +0300 Subject: [PATCH 1/2] feat: allow hourly partitioning --- README.md | 14 +++++++------- lib/fluent/plugin/out_bigquery_base.rb | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d4af6f6..5056139 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Because embbeded gem dependency sometimes restricts ruby environment. | schema_cache_expire | integer | no | no | 600 | Value is second. If current time is after expiration interval, re-fetch table schema definition. | | request_timeout_sec | integer | no | no | nil | Bigquery API response timeout | | request_open_timeout_sec | integer | no | no | 60 | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value. | -| time_partitioning_type | enum | no (either day) | no | nil | Type of bigquery time partitioning feature. | +| time_partitioning_type | enum | no (either day or hour) | no | nil | Type of bigquery time partitioning feature. | | time_partitioning_field | string | no | no | nil | Field used to determine how to create a time-based partition. | | time_partitioning_expiration | time | no | no | nil | Expiration milliseconds for bigquery time partitioning. | | clustering_fields | array(string) | no | no | nil | One or more fields on which data should be clustered. The order of the specified columns determines the sort order of the data. | @@ -194,15 +194,15 @@ For high rate inserts over streaming inserts, you should specify flush intervals ```apache @type bigquery_insert - + flush_interval 0.1 # flush as frequent as possible - + total_limit_size 10g - + flush_thread_count 16 - + auth_method private_key # default email xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxx@developer.gserviceaccount.com private_key_path /home/username/.keys/00000000000000000000000000000000-privatekey.p12 @@ -543,7 +543,7 @@ The second method is to specify a path to a BigQuery schema file instead of list @type bigquery_insert ... - + schema_path /path/to/httpd.schema ``` @@ -556,7 +556,7 @@ The third method is to set `fetch_schema` to `true` to enable fetch a schema usi @type bigquery_insert ... - + fetch_schema true # fetch_schema_table other_table # if you want to fetch schema from other table diff --git a/lib/fluent/plugin/out_bigquery_base.rb b/lib/fluent/plugin/out_bigquery_base.rb index 6e8d8dc..d83aceb 100644 --- a/lib/fluent/plugin/out_bigquery_base.rb +++ b/lib/fluent/plugin/out_bigquery_base.rb @@ -69,7 +69,7 @@ class BigQueryBaseOutput < Output config_param :request_open_timeout_sec, :time, default: 60 ## Partitioning - config_param :time_partitioning_type, :enum, list: [:day], default: nil + config_param :time_partitioning_type, :enum, list: [:day, :hour], default: nil config_param :time_partitioning_field, :string, default: nil config_param :time_partitioning_expiration, :time, default: nil From 718fbcaa991fb662122a213031c57c1853d9f778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Juozas=20Janu=C5=A1aitis?= Date: Fri, 14 Jun 2024 14:02:54 +0300 Subject: [PATCH 2/2] chore: bump version to v3.1.1 --- lib/fluent/plugin/bigquery/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fluent/plugin/bigquery/version.rb b/lib/fluent/plugin/bigquery/version.rb index d8bde2f..6c0c2bc 100644 --- a/lib/fluent/plugin/bigquery/version.rb +++ b/lib/fluent/plugin/bigquery/version.rb @@ -1,5 +1,5 @@ module Fluent module BigQueryPlugin - VERSION = "3.1.0".freeze + VERSION = "3.1.1".freeze end end