Skip to content

Commit

Permalink
repo-sync-20240524 (#277)
Browse files Browse the repository at this point in the history
  • Loading branch information
jingshi-ant authored May 14, 2024
1 parent 3a8c663 commit 8f2f3ed
Show file tree
Hide file tree
Showing 174 changed files with 4,575 additions and 2,182 deletions.
110 changes: 55 additions & 55 deletions .ci/broker-docker-compose/postgres/initdb/postgres_carol_init.sql
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ groupby_string_0 varchar(64) NOT NULL DEFAULT '',
compare_string_0 varchar(64) NOT NULL DEFAULT '',
aggregate_string_0 varchar(64) NOT NULL DEFAULT '',
encrypt_string_0 varchar(64) NOT NULL DEFAULT '',
plain_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
join_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
Expand All @@ -51,12 +51,12 @@ groupby_string_1 varchar(64) NOT NULL DEFAULT '',
compare_string_1 varchar(64) NOT NULL DEFAULT '',
aggregate_string_1 varchar(64) NOT NULL DEFAULT '',
encrypt_string_1 varchar(64) NOT NULL DEFAULT '',
plain_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_1 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
join_timestamp_1 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_timestamp_1 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
Expand All @@ -81,12 +81,12 @@ groupby_string_2 varchar(64) NOT NULL DEFAULT '',
compare_string_2 varchar(64) NOT NULL DEFAULT '',
aggregate_string_2 varchar(64) NOT NULL DEFAULT '',
encrypt_string_2 varchar(64) NOT NULL DEFAULT '',
plain_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_2 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
join_timestamp_2 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_timestamp_2 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
Expand Down Expand Up @@ -716,12 +716,12 @@ groupby_string_0 varchar(64) NOT NULL DEFAULT '',
compare_string_0 varchar(64) NOT NULL DEFAULT '',
aggregate_string_0 varchar(64) NOT NULL DEFAULT '',
encrypt_string_0 varchar(64) NOT NULL DEFAULT '',
plain_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
join_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
Expand All @@ -746,12 +746,12 @@ groupby_string_1 varchar(64) NOT NULL DEFAULT '',
compare_string_1 varchar(64) NOT NULL DEFAULT '',
aggregate_string_1 varchar(64) NOT NULL DEFAULT '',
encrypt_string_1 varchar(64) NOT NULL DEFAULT '',
plain_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_1 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
join_timestamp_1 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_timestamp_1 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
Expand All @@ -776,12 +776,12 @@ groupby_string_2 varchar(64) NOT NULL DEFAULT '',
compare_string_2 varchar(64) NOT NULL DEFAULT '',
aggregate_string_2 varchar(64) NOT NULL DEFAULT '',
encrypt_string_2 varchar(64) NOT NULL DEFAULT '',
plain_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_2 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
join_timestamp_2 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_timestamp_2 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
Expand Down Expand Up @@ -1411,12 +1411,12 @@ groupby_string_0 varchar(64) NOT NULL DEFAULT '',
compare_string_0 varchar(64) NOT NULL DEFAULT '',
aggregate_string_0 varchar(64) NOT NULL DEFAULT '',
encrypt_string_0 varchar(64) NOT NULL DEFAULT '',
plain_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
join_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
Expand All @@ -1441,12 +1441,12 @@ groupby_string_1 varchar(64) NOT NULL DEFAULT '',
compare_string_1 varchar(64) NOT NULL DEFAULT '',
aggregate_string_1 varchar(64) NOT NULL DEFAULT '',
encrypt_string_1 varchar(64) NOT NULL DEFAULT '',
plain_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_1 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_1 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_1 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
join_timestamp_1 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_timestamp_1 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
Expand All @@ -1471,12 +1471,12 @@ groupby_string_2 varchar(64) NOT NULL DEFAULT '',
compare_string_2 varchar(64) NOT NULL DEFAULT '',
aggregate_string_2 varchar(64) NOT NULL DEFAULT '',
encrypt_string_2 varchar(64) NOT NULL DEFAULT '',
plain_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_2 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
join_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
compare_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
aggregate_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
encrypt_datetime_2 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_2 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
join_timestamp_2 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
groupby_timestamp_2 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10',
Expand Down Expand Up @@ -2092,7 +2092,7 @@ CREATE TABLE tbl_3 (cp_jn_pl_int_0 integer NOT NULL DEFAULT 0,
cp_jn_pl_int_1 integer NOT NULL DEFAULT 0,
gb_en_pl_int_0 integer NOT NULL DEFAULT 0,
agg_en_pl_int_0 integer NOT NULL DEFAULT 0,
plain_datetime_0 date NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_datetime_0 timestamp with time zone NOT NULL DEFAULT '2020-10-10 10:10:10',
plain_timestamp_0 timestamp NOT NULL DEFAULT '2020-10-10 10:10:10');

INSERT INTO tbl_3 VALUES (-21, -27, 94, 78, '2000-08-03 09:50:21', '2001-11-24 04:52:03');
Expand Down
5 changes: 5 additions & 0 deletions .circleci/release-config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,13 @@ jobs:
cp /tmp/binary/linux/arm64/broker linux/arm64/broker
cp /tmp/binary/linux/arm64/brokerctl linux/arm64/brokerctl
# copy scripts
cp -r ../scripts ./scripts
ls ./linux/amd64
ls ./linux/arm64
ls ./scripts
TAG=$(grep "version" version.txt | awk -F'"' '{print $2}')
echo $TAG
docker buildx create --name scql-image-builder --platform linux/arm64,linux/amd64 --use
Expand Down
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## Staging

## [0.7.0] - 2024-05-14

### Added

- Added CheckAndUpdate API for self-recovery when status is inconsistent in P2P mode.

### Fixed

- Fixed the problem that Broker was unable to detect SCQLEngine crashes or being killed by OOM.

## [0.6.0] - 2024-04-15

### Added
Expand Down
11 changes: 5 additions & 6 deletions api/broker.proto
Original file line number Diff line number Diff line change
Expand Up @@ -196,15 +196,14 @@ service IntraBrokerService {
// Other APIs

// CheckAndUpdateStatus
// (-- NOTE: api not implemented yet --)
//
// Check and update status for Projects
rpc CheckAndUpdateStatus(CheckAndUpdateStatusRequest)
returns (CheckAndUpdateStatusResponse) {
// option (google.api.http) = {
// post: "/intra/status/check_and_update"
// body: "*"
// };
option (google.api.http) = {
post: "/intra/status/check_and_update"
body: "*"
};
}
}

Expand Down Expand Up @@ -479,7 +478,7 @@ message CheckAndUpdateStatusResponse {
Status status = 1;
// Valid only when status.code() == Code::PROJECT_CONFLICT
// maps from project_id to conflicts in the Project
map<string, ProjectConflict> conficts = 2;
map<string, ProjectConflict> conflicts = 2;
}
message ProjectConflict {
message ConflictItem {
Expand Down
71 changes: 51 additions & 20 deletions api/engine.proto
Original file line number Diff line number Diff line change
Expand Up @@ -33,24 +33,27 @@ service SCQLEngineService {
rpc RunExecutionPlan(RunExecutionPlanRequest)
returns (RunExecutionPlanResponse);

// Stop session
rpc StopSession(StopSessionRequest) returns (Status);
// Query Job Status
rpc QueryJobStatus(QueryJobStatusRequest) returns (QueryJobStatusResponse);

// Stop Job
rpc StopJob(StopJobRequest) returns (Status);
};

message StopSessionRequest {
string session_id = 1;
// session stop reason, it maybe
// - "Finish". Session ended normally.
// - "Timeout". Session timeout.
// - "Canceled". Session canceled by client, maybe triggered by CTRL+C
message StopJobRequest {
string job_id = 1;
// job stop reason, it maybe
// - "Finish". Job ended normally.
// - "Timeout". Job timeout.
// - "Canceled". Job canceled by client, maybe triggered by CTRL+C
// - "Error". Exception caught when running.
// - "Killed". Killed by client.
// - or something else.
string reason = 2;
}

// A message carries all session-level (execution plan level) information.
message SessionStartParams {
// A message carries all job session-level (execution plan level) information.
message JobStartParams {
message Party {
// party code
string code = 1;
Expand Down Expand Up @@ -87,42 +90,70 @@ message SessionStartParams {
// All parties that would jointly complete an execution plan.
repeated Party parties = 2;

// The session id
string session_id = 3;
// The job id
string job_id = 3;

// The spu runtime configuration.
spu.RuntimeConfig spu_runtime_cfg = 4;

// The session time zone, only support time offset, like: '+08:00'
// The query job time zone, only support time offset, like: '+08:00'
string time_zone = 5;
}

message GraphChecksum {
bool check_graph_checksum = 1;
// map from rank to checksum
// It could be used to verify and ensure that engines execute the same
// graph.
map<string, string> sub_graph_checksums = 2;
string whole_graph_checksum = 3;
}

message RunExecutionPlanRequest {
SessionStartParams session_params = 1;
JobStartParams job_params = 1;

SubGraph graph = 2;
GraphChecksum graph_checksum = 3;

// Whether the whole execution plan on the engine should be executed
// synchronously or asynchronously. By default, the execution plan is executed
// synchronously.
bool async = 3;
bool async = 4;

// Callback url, e.g.: "http://alice.com:8080/path/to/callback".
string callback_url = 4;
DebugOptions debug_opts = 5;
string callback_url = 5;
DebugOptions debug_opts = 6;
}

message RunExecutionPlanResponse {
Status status = 1;
// Output columns used to store result datas.
repeated Tensor out_columns = 2;
string session_id = 3;
string job_id = 3;
// Code of party which finished the execution plan.
string party_code = 4;
// The number of rows affected by a select into, update, insert, or delete.
int64 num_rows_affected = 5;
}

message QueryJobStatusRequest {
string job_id = 1;
}

message QueryJobStatusResponse {
Status status = 1;
enum JobState {
JOB_STATE_UNSPECIFIED = 0;
INITIALIZED = 1;
RUNNING = 2;
SUCCEEDED = 3;
FAILED = 4;
CANCELED = 5;
}
JobState job_state = 2;
// TODO: Add progress information
}

// EngineResultCallback
service EngineResultCallback {
// Engine report the plan result to scdb/broker/...
Expand All @@ -137,8 +168,8 @@ message ReportRequest {
Status status = 1;
// Output columns.
repeated Tensor out_columns = 2;
// The session_id that this execution plan belongs to.
string session_id = 3;
// The job_id that this execution plan belongs to.
string job_id = 3;
// Code of party which finished the plan run.
string party_code = 4;
// The number of rows affected by a select into, update, insert, or delete.
Expand Down
2 changes: 1 addition & 1 deletion api/interpreter.proto
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ message CompiledPlan {
// The position order matters, the parties[0]'s rank is 0.
repeated PartyId parties = 4;
map<string, SubGraph> sub_graphs = 5;
string whole_graph_checksum = 6;
// TODO: Add field to specify the party who report final query result.

ExplainInfo explain = 1000;
}

Expand Down
4 changes: 2 additions & 2 deletions api/subgraph.proto
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ option go_package = "proto-gen/scql";
message SubGraph {
map<string, ExecNode> nodes = 1;
SchedulingPolicy policy = 2;
// checksum of the whole execution graph
// checksum of the current sub graph
// It could be used to verify and ensure that engines execute the same graph.
bytes graph_checksum = 3;
string sub_graph_checksum = 3;
}

message SubDAG {
Expand Down
Loading

0 comments on commit 8f2f3ed

Please sign in to comment.