From 3e12438c1fb500f11609f0d9af920c14e268cd87 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Mon, 18 Nov 2024 10:00:51 +0800 Subject: [PATCH 01/13] feat: support host monitor --- core/CMakeLists.txt | 1 + core/app_config/AppConfig.cpp | 1 + core/common/FileSystemUtil.cpp | 44 ++++ core/common/FileSystemUtil.h | 17 ++ core/common/StringTools.cpp | 8 + core/common/StringTools.h | 6 + core/common/common.cmake | 2 +- core/common/timer/HostMonitorTimerEvent.cpp | 32 +++ core/common/timer/HostMonitorTimerEvent.h | 60 +++++ core/common/timer/Timer.h | 1 + core/common/timer/TimerEvent.h | 1 + core/constants/EntityConstants.cpp | 48 ++++ core/constants/EntityConstants.h | 48 ++++ core/host_monitor/Constants.cpp | 30 +++ core/host_monitor/Constants.h | 33 +++ core/host_monitor/HostMonitorInputRunner.cpp | 145 ++++++++++++ core/host_monitor/HostMonitorInputRunner.h | 77 ++++++ core/host_monitor/SystemInformationTools.cpp | 50 ++++ core/host_monitor/SystemInformationTools.h | 33 +++ core/host_monitor/collector/BaseCollector.h | 38 +++ .../collector/CollectorManager.cpp | 44 ++++ .../host_monitor/collector/CollectorManager.h | 44 ++++ core/host_monitor/collector/MockCollector.cpp | 33 +++ core/host_monitor/collector/MockCollector.h | 36 +++ .../collector/ProcessCollector.cpp | 223 ++++++++++++++++++ .../host_monitor/collector/ProcessCollector.h | 186 +++++++++++++++ core/models/PipelineEventGroup.h | 4 +- core/pipeline/Pipeline.h | 1 + core/pipeline/PipelineManager.cpp | 4 +- core/pipeline/plugin/PluginRegistry.cpp | 4 + core/pipeline/queue/ProcessQueueManager.h | 1 + core/plugin/input/InputHostMeta.cpp | 65 +++++ core/plugin/input/InputHostMeta.h | 46 ++++ .../inner/ProcessorHostMetaNative.cpp | 117 +++++++++ .../processor/inner/ProcessorHostMetaNative.h | 48 ++++ core/runner/sink/http/HttpSink.cpp | 16 +- core/unittest/CMakeLists.txt | 1 + core/unittest/host_monitor/1/stat | 1 + core/unittest/host_monitor/CMakeLists.txt | 38 +++ .../host_monitor/CollectorManagerUnittest.cpp | 42 ++++ .../HostMonitorInputRunnerUnittest.cpp | 78 ++++++ .../host_monitor/ProcessCollectorUnittest.cpp | 58 +++++ .../SystemInformationToolsUnittest.cpp | 38 +++ core/unittest/host_monitor/stat | 1 + core/unittest/input/CMakeLists.txt | 4 + core/unittest/input/InputHostMetaUnittest.cpp | 131 ++++++++++ core/unittest/processor/CMakeLists.txt | 4 + .../ProcessorHostMetaNativeUnittest.cpp | 120 ++++++++++ 48 files changed, 2052 insertions(+), 11 deletions(-) create mode 100644 core/common/timer/HostMonitorTimerEvent.cpp create mode 100644 core/common/timer/HostMonitorTimerEvent.h create mode 100644 core/constants/EntityConstants.cpp create mode 100644 core/constants/EntityConstants.h create mode 100644 core/host_monitor/Constants.cpp create mode 100644 core/host_monitor/Constants.h create mode 100644 core/host_monitor/HostMonitorInputRunner.cpp create mode 100644 core/host_monitor/HostMonitorInputRunner.h create mode 100644 core/host_monitor/SystemInformationTools.cpp create mode 100644 core/host_monitor/SystemInformationTools.h create mode 100644 core/host_monitor/collector/BaseCollector.h create mode 100644 core/host_monitor/collector/CollectorManager.cpp create mode 100644 core/host_monitor/collector/CollectorManager.h create mode 100644 core/host_monitor/collector/MockCollector.cpp create mode 100644 core/host_monitor/collector/MockCollector.h create mode 100644 core/host_monitor/collector/ProcessCollector.cpp create mode 100644 core/host_monitor/collector/ProcessCollector.h create mode 100644 core/plugin/input/InputHostMeta.cpp create mode 100644 core/plugin/input/InputHostMeta.h create mode 100644 core/plugin/processor/inner/ProcessorHostMetaNative.cpp create mode 100644 core/plugin/processor/inner/ProcessorHostMetaNative.h create mode 100644 core/unittest/host_monitor/1/stat create mode 100644 core/unittest/host_monitor/CMakeLists.txt create mode 100644 core/unittest/host_monitor/CollectorManagerUnittest.cpp create mode 100644 core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp create mode 100644 core/unittest/host_monitor/ProcessCollectorUnittest.cpp create mode 100644 core/unittest/host_monitor/SystemInformationToolsUnittest.cpp create mode 100644 core/unittest/host_monitor/stat create mode 100644 core/unittest/input/InputHostMetaUnittest.cpp create mode 100644 core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index b1b91aaf73..4a8ca74921 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -124,6 +124,7 @@ set(SUB_DIRECTORIES_LIST prometheus prometheus/labels prometheus/schedulers prometheus/async ebpf ebpf/observer ebpf/security ebpf/handler parser sls_control sdk + host_monitor host_monitor/collector ) if (LINUX) if (ENABLE_ENTERPRISE) diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index d492af6b7b..a660601346 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -164,6 +164,7 @@ DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator s DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888); DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000); DEFINE_FLAG_STRING(_pod_name_, "agent pod name", ""); +DEFINE_FLAG_INT32(process_collect_silent_count, "number of process scanned between a sleep", 1000); DEFINE_FLAG_STRING(app_info_file, "", "app_info.json"); DEFINE_FLAG_STRING(crash_stack_file_name, "crash stack back trace file name", "backtrace.dat"); diff --git a/core/common/FileSystemUtil.cpp b/core/common/FileSystemUtil.cpp index 39ef57c067..80d7e24fd0 100644 --- a/core/common/FileSystemUtil.cpp +++ b/core/common/FileSystemUtil.cpp @@ -141,6 +141,50 @@ bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t return true; } +int GetLines(std::istream& is, + bool enableEmptyLine, + const std::function& pushBack, + std::string* errorMessage) { + std::string line; + // 此处必须判断eof，具体原因参见: + // https://stackoverflow.com/questions/40561482/getline-throws-basic-iosclear-exception-after-reading-the-last-line + while (!is.eof() && std::getline(is, line)) { + if (enableEmptyLine || !line.empty()) { + pushBack(line); + } + } + return 0; +} + +int GetLines(const bfs::path& filename, + bool enableEmptyLine, + const std::function& pushBack, + std::string* errorMessage) { + int ret = 0; + std::ifstream fin; + try { + fin.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fin.open(filename.string(), std::ios_base::in); + fin.exceptions(std::ifstream::goodbit); + GetLines(fin, enableEmptyLine, pushBack, errorMessage); + fin.close(); + } catch (const std::exception& fail) { + if (errorMessage != nullptr) { + LOG_ERROR(sLogger, ("open file fail", filename)("errno", strerror(errno))); + ret = -1; + } + fin.close(); + } + return ret; +} + +int GetFileLines(const bfs::path& filename, + std::vector& res, + bool enableEmptyLine, + std::string* errorMessage) { + return GetLines(filename, enableEmptyLine, [&res](const std::string& s) { res.push_back(s); }, errorMessage); +} + bool OverwriteFile(const std::string& fileName, const std::string& content) { FILE* pFile = fopen(fileName.c_str(), "w"); if (pFile == NULL) { diff --git a/core/common/FileSystemUtil.h b/core/common/FileSystemUtil.h index 3ad1b046ed..8e0f523600 100644 --- a/core/common/FileSystemUtil.h +++ b/core/common/FileSystemUtil.h @@ -26,10 +26,14 @@ #elif defined(_MSC_VER) #include #endif +#include + #include "DevInode.h" #include "ErrorUtil.h" #include "LogtailCommonFlags.h" +namespace bfs = boost::filesystem; + // Filesystem utility. namespace logtail { @@ -87,6 +91,19 @@ void TrimLastSeperator(std::string& path); // ReadFileContent reads all content of @fileName to @content. bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t maxFileSize = 8192); +int GetLines(std::istream& is, + bool enableEmptyLine, + const std::function& pushBack, + std::string* errorMessage); +int GetLines(const bfs::path& filename, + bool enableEmptyLine, + const std::function& pushBack, + std::string* errorMessage); +int GetFileLines(const bfs::path& filename, + std::vector& res, + bool enableEmptyLine = true, + std::string* errorMessage = nullptr); + // OverwriteFile overwrides @fileName with @content. bool OverwriteFile(const std::string& fileName, const std::string& content); diff --git a/core/common/StringTools.cpp b/core/common/StringTools.cpp index 0e800cb754..57b29cfde9 100644 --- a/core/common/StringTools.cpp +++ b/core/common/StringTools.cpp @@ -363,4 +363,12 @@ void RemoveFilePathTrailingSlash(std::string& filePath) { filePath = path.string(); } +bool IsInt(const char* sz) { + bool ok = (sz != nullptr && *sz != '\0'); + for (auto* it = reinterpret_cast(sz); ok && *it; ++it) { + ok = (0 != std::isdigit(*it)); + } + return ok; +} + } // namespace logtail diff --git a/core/common/StringTools.h b/core/common/StringTools.h index 4d519dc8fa..da20c45e73 100644 --- a/core/common/StringTools.h +++ b/core/common/StringTools.h @@ -146,6 +146,12 @@ bool NormalizeTopicRegFormat(std::string& regStr); void RemoveFilePathTrailingSlash(std::string& path); +bool IsInt(const char* sz); + +inline bool IsInt(const std::string& str) { + return IsInt(str.c_str()); +} + #if defined(_MSC_VER) // TODO: Test it. #define FNM_PATHNAME 0 diff --git a/core/common/common.cmake b/core/common/common.cmake index cd5e9401c6..94d715bf8d 100644 --- a/core/common/common.cmake +++ b/core/common/common.cmake @@ -29,7 +29,7 @@ list(APPEND THIS_SOURCE_FILES_LIST ${XX_HASH_SOURCE_FILES}) # add memory in common list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/memory/SourceBuffer.h) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpRequest.cpp) -list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp) +list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp ${CMAKE_SOURCE_DIR}/common/timer/HostMonitorTimerEvent.cpp) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/compression/Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/CompressorFactory.cpp ${CMAKE_SOURCE_DIR}/common/compression/LZ4Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/ZstdCompressor.cpp) # remove several files in common list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/BoostRegexValidator.cpp ${CMAKE_SOURCE_DIR}/common/GetUUID.cpp) diff --git a/core/common/timer/HostMonitorTimerEvent.cpp b/core/common/timer/HostMonitorTimerEvent.cpp new file mode 100644 index 0000000000..2c4fcbd355 --- /dev/null +++ b/core/common/timer/HostMonitorTimerEvent.cpp @@ -0,0 +1,32 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "HostMonitorTimerEvent.h" + +#include "HostMonitorInputRunner.h" + +namespace logtail { + +bool HostMonitorTimerEvent::IsValid() const { + return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(mConfigName, mCollectorName); +} + +bool HostMonitorTimerEvent::Execute() { + HostMonitorInputRunner::GetInstance()->ScheduleOnce(this); + return true; +} + +} // namespace logtail diff --git a/core/common/timer/HostMonitorTimerEvent.h b/core/common/timer/HostMonitorTimerEvent.h new file mode 100644 index 0000000000..e9f3c08457 --- /dev/null +++ b/core/common/timer/HostMonitorTimerEvent.h @@ -0,0 +1,60 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "QueueKey.h" +#include "timer/TimerEvent.h" + +namespace logtail { + +class HostMonitorTimerEvent : public TimerEvent { +public: + HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime, + size_t interval, + std::string configName, + std::string collectorName, + QueueKey processQueueKey) + : TimerEvent(execTime), + mConfigName(std::move(configName)), + mCollectorName(collectorName), + mProcessQueueKey(processQueueKey), + mInputIdx(0) { + mInterval = std::chrono::seconds(interval); + } + + bool IsValid() const override; + bool Execute() override; + + const std::string GetConfigName() const { return mConfigName; } + const std::string GetCollectorName() const { return mCollectorName; } + const QueueKey GetProcessQueueKey() const { return mProcessQueueKey; } + int GetInputIndex() const { return mInputIdx; } + const std::chrono::seconds GetInterval() const { return mInterval; } + void ResetForNextExec() { SetExecTime(GetExecTime() + mInterval); } + +private: + std::string mConfigName; + std::string mCollectorName; + QueueKey mProcessQueueKey; + int mInputIdx; + std::chrono::seconds mInterval; +}; + +} // namespace logtail diff --git a/core/common/timer/Timer.h b/core/common/timer/Timer.h index 6825e8443d..4b241e91d5 100644 --- a/core/common/timer/Timer.h +++ b/core/common/timer/Timer.h @@ -53,6 +53,7 @@ class Timer { #ifdef APSARA_UNIT_TEST_MAIN friend class TimerUnittest; friend class ScrapeSchedulerUnittest; + friend class HostMonitorInputRunnerUnittest; #endif }; diff --git a/core/common/timer/TimerEvent.h b/core/common/timer/TimerEvent.h index 005d60e882..e6064b667a 100644 --- a/core/common/timer/TimerEvent.h +++ b/core/common/timer/TimerEvent.h @@ -29,6 +29,7 @@ class TimerEvent { virtual bool Execute() = 0; std::chrono::steady_clock::time_point GetExecTime() const { return mExecTime; } + void SetExecTime(std::chrono::steady_clock::time_point nextExecTime) { mExecTime = nextExecTime; } private: std::chrono::steady_clock::time_point mExecTime; diff --git a/core/constants/EntityConstants.cpp b/core/constants/EntityConstants.cpp new file mode 100644 index 0000000000..17df9c2b5c --- /dev/null +++ b/core/constants/EntityConstants.cpp @@ -0,0 +1,48 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "EntityConstants.h" + +namespace logtail { + +const std::string DEFAULT_ENV_KEY_HOST_TYPE = "HOST_TYPE"; +const std::string DEFAULT_ENV_VALUE_ECS = "ecs"; +const std::string DEFAULT_ENV_VALUE_HOST = "host"; +const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE = "__entity_type__"; +const std::string DEFAULT_CONTENT_KEY_ENTITY_ID = "__entity_id__"; +const std::string DEFAULT_CONTENT_KEY_DOMAIN = "__domain__"; +const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS = "acs"; +const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA = "infra"; +const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME = "__first_observed_time__"; +const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME = "__last_observed_time__"; +const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS = "__keep_alive_seconds__"; +const std::string DEFAULT_CONTENT_KEY_METHOD = "__method__"; +const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE = "update"; +const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE = "expire"; + +// for process entity +const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS = "process"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_PID = "process_pid"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID = "process_ppid"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_USER = "process_user"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM = "process_comm"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME = "process_create_time"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD = "process_cwd"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY = "process_binary"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS = "process_arguments"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE = "process_language"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID = "process_container_id"; +} // namespace logtail diff --git a/core/constants/EntityConstants.h b/core/constants/EntityConstants.h new file mode 100644 index 0000000000..0da3a13920 --- /dev/null +++ b/core/constants/EntityConstants.h @@ -0,0 +1,48 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +namespace logtail { + +extern const std::string DEFAULT_ENV_KEY_HOST_TYPE; +extern const std::string DEFAULT_ENV_VALUE_ECS; +extern const std::string DEFAULT_ENV_VALUE_HOST; +extern const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE; +extern const std::string DEFAULT_CONTENT_KEY_ENTITY_ID; +extern const std::string DEFAULT_CONTENT_KEY_DOMAIN; +extern const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS; +extern const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA; +extern const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME; +extern const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME; +extern const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS; +extern const std::string DEFAULT_CONTENT_KEY_METHOD; +extern const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE; +extern const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE; + +// for process entity +extern const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PID; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_USER; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID; +} // namespace logtail diff --git a/core/host_monitor/Constants.cpp b/core/host_monitor/Constants.cpp new file mode 100644 index 0000000000..366b314480 --- /dev/null +++ b/core/host_monitor/Constants.cpp @@ -0,0 +1,30 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Constants.h" + + +namespace logtail { + +#ifndef APSARA_UNIT_TEST_MAIN +const bfs::path PROCESS_DIR = "/proc"; +#else +bfs::path PROCESS_DIR = "/proc"; +#endif + +const bfs::path PROCESS_STAT = "stat"; + +} // namespace logtail diff --git a/core/host_monitor/Constants.h b/core/host_monitor/Constants.h new file mode 100644 index 0000000000..788065b81e --- /dev/null +++ b/core/host_monitor/Constants.h @@ -0,0 +1,33 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace bfs = boost::filesystem; + +namespace logtail { + +#ifndef APSARA_UNIT_TEST_MAIN +extern const bfs::path PROCESS_DIR; +#else +extern bfs::path PROCESS_DIR; +#endif + +const extern bfs::path PROCESS_STAT; + +} // namespace logtail diff --git a/core/host_monitor/HostMonitorInputRunner.cpp b/core/host_monitor/HostMonitorInputRunner.cpp new file mode 100644 index 0000000000..ab2618623e --- /dev/null +++ b/core/host_monitor/HostMonitorInputRunner.cpp @@ -0,0 +1,145 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "HostMonitorInputRunner.h" + +#include +#include + +#include "CollectorManager.h" +#include "Lock.h" +#include "LogEvent.h" +#include "Logger.h" +#include "ProcessQueueItem.h" +#include "ProcessQueueManager.h" +#include "ThreadPool.h" +#include "timer/HostMonitorTimerEvent.h" +#include "timer/TimerEvent.h" + + +namespace logtail { + +HostMonitorInputRunner::HostMonitorInputRunner() { + mTimer = std::make_shared(); + mThreadPool = std::make_shared(3); +} + +void HostMonitorInputRunner::UpdateCollector(const std::string& configName, + const std::vector& collectorNames, + QueueKey processQueueKey) { + WriteLock lock(mCollectorMapRWLock); + mCollectorMap[configName] = collectorNames; + for (const auto& collectorName : collectorNames) { + LOG_INFO(sLogger, ("add new host monitor collector", configName)("collector", collectorName)); + mTimer->PushEvent(BuildTimerEvent(configName, collectorName, processQueueKey)); + } +} + +void HostMonitorInputRunner::RemoveCollector(const std::string& configName) { + WriteLock lock(mCollectorMapRWLock); + mCollectorMap.erase(configName); +} + +void HostMonitorInputRunner::Init() { + std::lock_guard lock(mStartMutex); + if (mIsStarted) { + return; + } + LOG_INFO(sLogger, ("HostMonitorInputRunner", "Start")); + mIsStarted = true; + +#ifndef APSARA_UNIT_TEST_MAIN + mThreadPool->Start(); + mTimer->Init(); +#endif +} + +void HostMonitorInputRunner::Stop() { + std::lock_guard lock(mStartMutex); + if (!mIsStarted) { + return; + } + + mIsStarted = false; +#ifndef APSARA_UNIT_TEST_MAIN + mTimer->Stop(); + mThreadPool->Stop(); +#endif + LOG_INFO(sLogger, ("HostMonitorInputRunner", "Stop")); +} + +bool HostMonitorInputRunner::HasRegisteredPlugins() const { + ReadLock lock(mCollectorMapRWLock); + return !mCollectorMap.empty(); +} + +bool HostMonitorInputRunner::IsCollectTaskValid(const std::string& configName, const std::string& collectorName) const { + ReadLock lock(mCollectorMapRWLock); + auto collectors = mCollectorMap.find(configName); + if (collectors == mCollectorMap.end()) { + return false; + } + for (const auto& collectorName : collectors->second) { + if (collectorName == collectorName) { + return true; + } + } + return false; +} + +void HostMonitorInputRunner::ScheduleOnce(HostMonitorTimerEvent* event) { + // TODO: reuse event + HostMonitorTimerEvent eventCopy = *event; + mThreadPool->Add([this, eventCopy]() mutable { + auto configName = eventCopy.GetConfigName(); + auto collectorName = eventCopy.GetCollectorName(); + auto processQueueKey = eventCopy.GetProcessQueueKey(); + PipelineEventGroup group(std::make_shared()); + auto collector = CollectorManager::GetInstance()->GetCollector(collectorName); + collector->Collect(group); + + std::unique_ptr item + = std::make_unique(std::move(group), eventCopy.GetInputIndex()); + if (ProcessQueueManager::GetInstance()->IsValidToPush(processQueueKey)) { + ProcessQueueManager::GetInstance()->PushQueue(processQueueKey, std::move(item)); + } else { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + // try again + if (ProcessQueueManager::GetInstance()->IsValidToPush(processQueueKey)) { + ProcessQueueManager::GetInstance()->PushQueue(processQueueKey, std::move(item)); + } else { + LOG_WARNING(sLogger, ("process queue is full", "discard data")("config", configName)); + } + } + + LOG_DEBUG(sLogger, ("schedule host monitor collector again", configName)("collector", collectorName)); + + eventCopy.ResetForNextExec(); + mTimer->PushEvent(std::make_unique(eventCopy)); + }); +} + +std::unique_ptr HostMonitorInputRunner::BuildTimerEvent(const std::string& configName, + const std::string& collectorName, + QueueKey processQueueKey) { + auto now = std::chrono::steady_clock::now(); + auto event = std::make_unique( + now, DEFAULT_SCHEDULE_INTERVAL, configName, collectorName, processQueueKey); + return event; +} + + +} // namespace logtail diff --git a/core/host_monitor/HostMonitorInputRunner.h b/core/host_monitor/HostMonitorInputRunner.h new file mode 100644 index 0000000000..5aceb92c30 --- /dev/null +++ b/core/host_monitor/HostMonitorInputRunner.h @@ -0,0 +1,77 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "InputRunner.h" +#include "Lock.h" +#include "QueueKey.h" +#include "ThreadPool.h" +#include "timer/HostMonitorTimerEvent.h" +#include "timer/Timer.h" + +namespace logtail { + +const int DEFAULT_SCHEDULE_INTERVAL = 10; + +class HostMonitorInputRunner : public InputRunner { +public: + HostMonitorInputRunner(const HostMonitorInputRunner&) = delete; + HostMonitorInputRunner(HostMonitorInputRunner&&) = delete; + HostMonitorInputRunner& operator=(const HostMonitorInputRunner&) = delete; + HostMonitorInputRunner& operator=(HostMonitorInputRunner&&) = delete; + ~HostMonitorInputRunner() override = default; + static HostMonitorInputRunner* GetInstance() { + static HostMonitorInputRunner sInstance; + return &sInstance; + } + + void UpdateCollector(const std::string& configName, + const std::vector& collectorNames, + QueueKey processQueueKey); + void RemoveCollector(const std::string& configName); + + void Init() override; + void Stop() override; + bool HasRegisteredPlugins() const override; + + bool IsCollectTaskValid(const std::string& configName, const std::string& collectorName) const; + void ScheduleOnce(HostMonitorTimerEvent* event); + +private: + HostMonitorInputRunner(); + std::unique_ptr + BuildTimerEvent(const std::string& configName, const std::string& collectorName, QueueKey processQueueKey); + + bool mIsStarted = false; + std::mutex mStartMutex; + + std::shared_ptr mTimer; + std::shared_ptr mThreadPool; + + mutable ReadWriteLock mCollectorMapRWLock; + std::unordered_map> mCollectorMap; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class HostMonitorInputRunnerUnittest; +#endif +}; + +} // namespace logtail diff --git a/core/host_monitor/SystemInformationTools.cpp b/core/host_monitor/SystemInformationTools.cpp new file mode 100644 index 0000000000..edbad31cab --- /dev/null +++ b/core/host_monitor/SystemInformationTools.cpp @@ -0,0 +1,50 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SystemInformationTools.h" + +#include "Logger.h" +#include "host_monitor/Constants.h" + +namespace logtail { + +int64_t GetSystemBootSeconds() { + static int64_t systemBootSeconds; + if (systemBootSeconds != 0) { + return systemBootSeconds; + } + + std::vector cpuLines = {}; + std::string errorMessage; + int ret = GetFileLines(PROCESS_DIR / PROCESS_STAT, cpuLines, true, &errorMessage); + LOG_WARNING(sLogger, ("failed to get cpu lines", errorMessage)("ret", ret)("cpuLines", cpuLines.size())); + if (ret != 0 || cpuLines.empty()) { + return duration_cast(system_clock::now().time_since_epoch()).count(); + } + + for (auto const& cpuLine : cpuLines) { + auto cpuMetric = SplitString(cpuLine); + if (cpuMetric.size() >= 2 && cpuMetric[0] == "btime") { + constexpr size_t bootTimeIndex = 1; + return bootTimeIndex < cpuMetric.size() ? StringTo(cpuMetric[bootTimeIndex]) : 0; + } + } + + systemBootSeconds = duration_cast(system_clock::now().time_since_epoch()).count(); + return systemBootSeconds; +} + +} // namespace logtail diff --git a/core/host_monitor/SystemInformationTools.h b/core/host_monitor/SystemInformationTools.h new file mode 100644 index 0000000000..c0e1e5189b --- /dev/null +++ b/core/host_monitor/SystemInformationTools.h @@ -0,0 +1,33 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "FileSystemUtil.h" +#include "StringTools.h" +#include "constants/EntityConstants.h" + +using namespace std::chrono; + +namespace logtail { + +int64_t GetSystemBootSeconds(); + +} // namespace logtail diff --git a/core/host_monitor/collector/BaseCollector.h b/core/host_monitor/collector/BaseCollector.h new file mode 100644 index 0000000000..73115e1a53 --- /dev/null +++ b/core/host_monitor/collector/BaseCollector.h @@ -0,0 +1,38 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "PipelineEventGroup.h" + +namespace logtail { + +class BaseCollector { +public: + BaseCollector() = default; + virtual ~BaseCollector() = default; + + bool IsValid() const { return mValidState; } + virtual void Collect(PipelineEventGroup& group) = 0; + + const std::string GetName() const { return mName; } + +protected: + std::string mName; + bool mValidState = false; +}; + +} // namespace logtail diff --git a/core/host_monitor/collector/CollectorManager.cpp b/core/host_monitor/collector/CollectorManager.cpp new file mode 100644 index 0000000000..9a9c865b85 --- /dev/null +++ b/core/host_monitor/collector/CollectorManager.cpp @@ -0,0 +1,44 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CollectorManager.h" + +#include "BaseCollector.h" +#include "MockCollector.h" +#include "host_monitor/collector/ProcessCollector.h" + +namespace logtail { + +CollectorManager::CollectorManager() { + RegisterCollector(); + RegisterCollector(); +} + +std::shared_ptr CollectorManager::GetCollector(const std::string& collectorName) { + auto it = mCollectorMap.find(collectorName); + if (it == mCollectorMap.end()) { + return nullptr; + } + return it->second; +} + +template +void CollectorManager::RegisterCollector() { + auto collector = std::make_shared(); + mCollectorMap[collector->GetName()] = collector; +} + +} // namespace logtail diff --git a/core/host_monitor/collector/CollectorManager.h b/core/host_monitor/collector/CollectorManager.h new file mode 100644 index 0000000000..2882bc5ef2 --- /dev/null +++ b/core/host_monitor/collector/CollectorManager.h @@ -0,0 +1,44 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "host_monitor/collector/BaseCollector.h" +namespace logtail { + +class CollectorManager { +public: + static CollectorManager* GetInstance() { + static CollectorManager sInstance; + return &sInstance; + } + + std::shared_ptr GetCollector(const std::string& collectorName); + +private: + CollectorManager(); + ~CollectorManager() = default; + + template + void RegisterCollector(); + + std::unordered_map> mCollectorMap; +}; + +} // namespace logtail diff --git a/core/host_monitor/collector/MockCollector.cpp b/core/host_monitor/collector/MockCollector.cpp new file mode 100644 index 0000000000..c9046b5d43 --- /dev/null +++ b/core/host_monitor/collector/MockCollector.cpp @@ -0,0 +1,33 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "MockCollector.h" + +#include + +#include "PipelineEventGroup.h" +#include "constants/EntityConstants.h" + +namespace logtail { + +void MockCollector::Collect(PipelineEventGroup& group) { + auto event = group.AddLogEvent(); + event->SetTimestamp(time(nullptr)); + std::string content = "mock content"; + event->SetContent("mock", content); +} + +} // namespace logtail diff --git a/core/host_monitor/collector/MockCollector.h b/core/host_monitor/collector/MockCollector.h new file mode 100644 index 0000000000..48f8292ae8 --- /dev/null +++ b/core/host_monitor/collector/MockCollector.h @@ -0,0 +1,36 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "constants/EntityConstants.h" +#include "host_monitor/collector/BaseCollector.h" + +namespace logtail { + +class MockCollector : public BaseCollector { +public: + MockCollector() { mName = "mock"; }; + ~MockCollector() override = default; + + void Collect(PipelineEventGroup& group) override; + +private: +}; + +} // namespace logtail diff --git a/core/host_monitor/collector/ProcessCollector.cpp b/core/host_monitor/collector/ProcessCollector.cpp new file mode 100644 index 0000000000..5479f7909b --- /dev/null +++ b/core/host_monitor/collector/ProcessCollector.cpp @@ -0,0 +1,223 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ProcessCollector.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "FileSystemUtil.h" +#include "Logger.h" +#include "PipelineEventGroup.h" +#include "StringTools.h" +#include "constants/EntityConstants.h" +#include "host_monitor/SystemInformationTools.h" + +namespace logtail { + +const size_t ProcessTopN = 20; + +void ProcessCollector::Collect(PipelineEventGroup& group) { + std::lock_guard lock(mCollectLock); + + group.SetMetadata(EventGroupMetaKey::HOST_MONITOR_COLLECT_TIME, std::to_string(time(nullptr))); + std::vector processes; + SortProcessByCpu(processes, ProcessTopN); + for (auto process : processes) { + auto event = group.AddLogEvent(); + time_t logtime = time(nullptr); + event->SetTimestamp(logtime); + + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PID, std::to_string(process->pid)); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PPID, std::to_string(process->parentPid)); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME, + std::to_string(duration_cast(process->startTime.time_since_epoch()).count())); + } +} + +void ProcessCollector::SortProcessByCpu(std::vector& processStats, size_t topN) { + steady_clock::time_point now = steady_clock::now(); + auto compare = [](const std::pair& a, const std::pair& b) { + return a.second < b.second; + }; + std::priority_queue, + std::vector>, + decltype(compare)> + queue(compare); + + int readCount = 0; + WalkAllProcess(PROCESS_DIR, [&](const std::string& dirName) { + if (++readCount > mProcessSilentCount) { + readCount = 0; + std::this_thread::sleep_for(milliseconds{100}); + } + auto pid = StringTo(dirName); + if (pid != 0) { + bool isFirstCollect = false; + auto ptr = GetProcessStat(pid, isFirstCollect); + if (ptr && !isFirstCollect) { + queue.push(std::make_pair(ptr, ptr->cpuInfo.total)); + } + if (queue.size() > topN) { + queue.pop(); + } + } + }); + + processStats.clear(); + processStats.reserve(queue.size()); + while (!queue.empty()) { + processStats.push_back(queue.top().first); + queue.pop(); + } + std::reverse(processStats.begin(), processStats.end()); + + if (processStats.empty()) { + LOG_INFO(sLogger, ("first collect Process Cpu info", "empty")); + } + LOG_DEBUG(sLogger, ("collect Process Cpu info, top", processStats.size())); + + mProcessSortTime = now; + mSortProcessStats = processStats; +} + +ProcessStatPtr ProcessCollector::GetProcessStat(pid_t pid, bool& isFirstCollect) { + const auto now = steady_clock::now(); + + // TODO: more accurate cache + auto prev = GetPreProcessStat(pid); + isFirstCollect = prev == nullptr || prev->lastTime.time_since_epoch().count() == 0; + // proc/[pid]/stat的统计粒度通常为10ms，两次采样之间需要足够大才能平滑。 + if (prev && now < prev->lastTime + seconds{1}) { + return prev; + } + auto ptr = ReadProcessStat(pid); + if (!ptr) { + return nullptr; + } + + // calculate CPU related fields + { + ptr->lastTime = now; + ptr->cpuInfo.user = ptr->utime.count(); + ptr->cpuInfo.sys = ptr->stime.count(); + ptr->cpuInfo.total = ptr->cpuInfo.user + ptr->cpuInfo.sys; + if (isFirstCollect || ptr->cpuInfo.total <= prev->cpuInfo.total) { + // first time called + ptr->cpuInfo.percent = 0.0; + } else { + auto totalDiff = static_cast(ptr->cpuInfo.total - prev->cpuInfo.total); + auto timeDiff = static_cast(ptr->lastTime.time_since_epoch().count() + - prev->lastTime.time_since_epoch().count()); + ptr->cpuInfo.percent = totalDiff / timeDiff * 100; + } + } + + prev = ptr; + mPrevProcessStat[pid] = ptr; + return ptr; +} + +ProcessStatPtr ProcessCollector::ReadProcessStat(pid_t pid) { + LOG_DEBUG(sLogger, ("read process stat", pid)); + auto processStat = PROCESS_DIR / std::to_string(pid) / PROCESS_STAT; + + std::string line; + if (!ReadFileContent(processStat.string(), line)) { + LOG_ERROR(sLogger, ("read process stat", "fail")("file", processStat)); + return nullptr; + } + return ParseProcessStat(pid, line); +} + + +// 数据样例: /proc/1/stat +// 1 (cat) R 0 1 1 34816 1 4194560 1110 0 0 0 1 1 0 0 20 0 1 0 18938584 4505600 171 18446744073709551615 4194304 4238788 +// 140727020025920 0 0 0 0 0 0 0 0 0 17 3 0 0 0 0 0 6336016 6337300 21442560 140727020027760 140727020027777 +// 140727020027777 140727020027887 0 +ProcessStatPtr ProcessCollector::ParseProcessStat(pid_t pid, std::string& line) { + ProcessStatPtr ptr = std::make_shared(); + ptr->pid = pid; + auto nameStartPos = line.find_first_of('('); + auto nameEndPos = line.find_last_of(')'); + if (nameStartPos == std::string::npos || nameEndPos == std::string::npos) { + LOG_ERROR(sLogger, ("can't find process name", pid)("stat", line)); + return nullptr; + } + nameStartPos++; // 跳过左括号 + ptr->name = line.substr(nameStartPos, nameEndPos - nameStartPos); + line = line.substr(nameEndPos + 2); // 跳过右括号及空格 + + std::vector words = SplitString(line); + + constexpr const EnumProcessStat offset = EnumProcessStat::state; // 跳过pid, comm + constexpr const int minCount = EnumProcessStat::processor - offset + 1; // 37 + if (words.size() < minCount) { + LOG_ERROR(sLogger, ("unexpected item count", pid)("stat", line)); + return nullptr; + } + + ptr->state = words[EnumProcessStat::state - offset].front(); + ptr->parentPid = StringTo(words[EnumProcessStat::ppid - offset]); + ptr->tty = StringTo(words[EnumProcessStat::tty_nr - offset]); + ptr->minorFaults = StringTo(words[EnumProcessStat::minflt - offset]); + ptr->majorFaults = StringTo(words[EnumProcessStat::majflt - offset]); + + ptr->utime = static_cast(StringTo(words[EnumProcessStat::utime - offset])); + ptr->stime = static_cast(StringTo(words[EnumProcessStat::stime - offset])); + ptr->cutime = static_cast(StringTo(words[EnumProcessStat::cutime - offset])); + ptr->cstime = static_cast(StringTo(words[EnumProcessStat::cstime - offset])); + + ptr->priority = StringTo(words[EnumProcessStat::priority - offset]); + ptr->nice = StringTo(words[EnumProcessStat::nice - offset]); + ptr->numThreads = StringTo(words[EnumProcessStat::num_threads - offset]); + + ptr->startTime = system_clock::time_point{ + static_cast(StringTo(words[EnumProcessStat::starttime - offset])) + + milliseconds{GetSystemBootSeconds() * 1000}}; + ptr->vSize = StringTo(words[EnumProcessStat::vsize - offset]); + ptr->rss = StringTo(words[EnumProcessStat::rss - offset]) << (getpagesize()); + ptr->processor = StringTo(words[EnumProcessStat::processor - offset]); + return ptr; +} + +bool ProcessCollector::WalkAllProcess(const bfs::path& root, const std::function& callback) { + if (!bfs::exists(root) || !bfs::is_directory(root)) { + LOG_ERROR(sLogger, ("ProcessCollector", "root path is not a directory or not exist")("root", root)); + return false; + } + + for (const auto& dirEntry : bfs::directory_iterator{root, bfs::directory_options::skip_permission_denied}) { + std::string filename = dirEntry.path().filename().string(); + if (IsInt(filename)) { + callback(filename); + } + } + return true; +} + +} // namespace logtail diff --git a/core/host_monitor/collector/ProcessCollector.h b/core/host_monitor/collector/ProcessCollector.h new file mode 100644 index 0000000000..836f106812 --- /dev/null +++ b/core/host_monitor/collector/ProcessCollector.h @@ -0,0 +1,186 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Flags.h" +#include "Logger.h" +#include "MachineInfoUtil.h" +#include "StringTools.h" +#include "constants/EntityConstants.h" +#include "host_monitor/Constants.h" +#include "host_monitor/collector/BaseCollector.h" + +DECLARE_FLAG_INT32(process_collect_silent_count); + +namespace bfs = boost::filesystem; +using namespace std::chrono; + +namespace logtail { + + +struct ProcessCpuInfo { + uint64_t user = 0; + uint64_t sys = 0; + uint64_t total = 0; + double percent = 0.0; +}; + +struct ProcessStat { + pid_t pid = 0; + uint64_t vSize = 0; + uint64_t rss = 0; + uint64_t minorFaults = 0; + uint64_t majorFaults = 0; + pid_t parentPid = 0; + int tty = 0; + int priority = 0; + int nice = 0; + int numThreads = 0; + system_clock::time_point startTime; + steady_clock::time_point lastTime; + + milliseconds utime{0}; + milliseconds stime{0}; + milliseconds cutime{0}; + milliseconds cstime{0}; + ProcessCpuInfo cpuInfo; + + std::string name; + char state = '\0'; + int processor = 0; + + int64_t startMillis() const { return duration_cast(startTime.time_since_epoch()).count(); } +}; + +typedef std::shared_ptr ProcessStatPtr; + +// See https://man7.org/linux/man-pages/man5/proc.5.html +enum class EnumProcessStat : int { + pid, // 0 + comm, // 1 + state, // 2 + ppid, // 3 + pgrp, // 4 + session, // 5 + tty_nr, // 6 + tpgid, // 7 + flags, // 8 + minflt, // 9 + cminflt, // 10 + majflt, // 11 + cmajflt, // 12 + utime, // 13 + stime, // 14 + cutime, // 15 + cstime, // 16 + priority, // 17 + nice, // 18 + num_threads, // 19 + itrealvalue, // 20 + starttime, // 21 + vsize, // 22 + rss, // 23 + rsslim, // 24 + startcode, // 25 + endcode, // 26 + startstack, // 27 + kstkesp, // 28 + kstkeip, // 29 + signal, // 30 + blocked, // 31 + sigignore, // 32 + sigcatch, // 33 + wchan, // 34 + nswap, // 35 + cnswap, // 36 + exit_signal, // 37 + processor, // 38 <--- 至少需要有该字段 + rt_priority, // 39 + policy, // 40 + delayacct_blkio_ticks, // 41 + guest_time, // 42 + cguest_time, // 43 + start_data, // 44 + end_data, // 45 + start_brk, // 46 + arg_start, // 47 + arg_end, // 48 + env_start, // 49 + env_end, // 50 + exit_code, // 51 + + _count, // 只是用于计数，非实际字段 +}; +static_assert((int)EnumProcessStat::comm == 1, "EnumProcessStat invalid"); +static_assert((int)EnumProcessStat::processor == 38, "EnumProcessStat invalid"); + +constexpr int operator-(EnumProcessStat a, EnumProcessStat b) { + return (int)a - (int)b; +} + +class ProcessCollector : public BaseCollector { +public: + ProcessCollector() : mProcessSilentCount(INT32_FLAG(process_collect_silent_count)) { + mName = "process"; + + // try to read process dir + if (access(PROCESS_DIR.c_str(), R_OK) != 0) { + LOG_ERROR(sLogger, ("process collector init failed", "process dir not exist or ")("dir", PROCESS_DIR)); + mValidState = false; + } else { + mValidState = true; + } + }; + ~ProcessCollector() override = default; + + void Collect(PipelineEventGroup& group) override; + +private: + void SortProcessByCpu(std::vector& processStats, size_t topN); + ProcessStatPtr GetProcessStat(pid_t pid, bool& isFirstCollect); + ProcessStatPtr ReadProcessStat(pid_t pid); + ProcessStatPtr ParseProcessStat(pid_t pid, std::string& line); + + bool WalkAllProcess(const bfs::path& root, const std::function& callback); + + ProcessStatPtr GetPreProcessStat(pid_t pid) { return mPrevProcessStat[pid]; } + + std::mutex mCollectLock; + steady_clock::time_point mProcessSortTime; + std::vector mSortProcessStats; + std::unordered_map mPrevProcessStat; + + const int mProcessSilentCount; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class ProcessCollectorUnittest; +#endif +}; + +} // namespace logtail diff --git a/core/models/PipelineEventGroup.h b/core/models/PipelineEventGroup.h index 5831a581d1..be4508827e 100644 --- a/core/models/PipelineEventGroup.h +++ b/core/models/PipelineEventGroup.h @@ -20,8 +20,8 @@ #include #include "checkpoint/RangeCheckpoint.h" -#include "constants/Constants.h" #include "common/memory/SourceBuffer.h" +#include "constants/Constants.h" #include "models/PipelineEventPtr.h" namespace logtail { @@ -58,6 +58,8 @@ enum class EventGroupMetaKey { PROMETHEUS_SCRAPE_TIMESTAMP_MILLISEC, PROMETHEUS_UP_STATE, + HOST_MONITOR_COLLECT_TIME, + SOURCE_ID }; diff --git a/core/pipeline/Pipeline.h b/core/pipeline/Pipeline.h index d6a55911c7..643efdc506 100644 --- a/core/pipeline/Pipeline.h +++ b/core/pipeline/Pipeline.h @@ -126,6 +126,7 @@ class Pipeline { friend class InputProcessSecurityUnittest; friend class InputNetworkSecurityUnittest; friend class InputNetworkObserverUnittest; + friend class InputHostMetaUnittest; #endif }; diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 57abcde874..183262da9c 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -16,6 +16,7 @@ #include "pipeline/PipelineManager.h" +#include "HostMonitorInputRunner.h" #include "file_server/ConfigManager.h" #include "file_server/FileServer.h" #include "go_pipeline/LogtailPlugin.h" @@ -40,7 +41,8 @@ PipelineManager::PipelineManager() : mInputRunners({ PrometheusInputRunner::GetInstance(), #if defined(__linux__) && !defined(__ANDROID__) - ebpf::eBPFServer::GetInstance(), + ebpf::eBPFServer::GetInstance(), + HostMonitorInputRunner::GetInstance(), #endif }) { } diff --git a/core/pipeline/plugin/PluginRegistry.cpp b/core/pipeline/plugin/PluginRegistry.cpp index bf0d7acbe6..ee6d314fc5 100644 --- a/core/pipeline/plugin/PluginRegistry.cpp +++ b/core/pipeline/plugin/PluginRegistry.cpp @@ -30,7 +30,9 @@ #include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/input/InputContainerStdio.h" #include "plugin/input/InputFile.h" +#include "plugin/input/InputHostMeta.h" #include "plugin/input/InputPrometheus.h" +#include "plugin/processor/inner/ProcessorHostMetaNative.h" #if defined(__linux__) && !defined(__ANDROID__) #include "plugin/input/InputFileSecurity.h" #include "plugin/input/InputInternalMetrics.h" @@ -134,6 +136,7 @@ void PluginRegistry::LoadStaticPlugins() { RegisterInputCreator(new StaticInputCreator()); RegisterInputCreator(new StaticInputCreator()); RegisterInputCreator(new StaticInputCreator()); + RegisterInputCreator(new StaticInputCreator()); #endif RegisterProcessorCreator(new StaticProcessorCreator()); @@ -151,6 +154,7 @@ void PluginRegistry::LoadStaticPlugins() { RegisterProcessorCreator(new StaticProcessorCreator()); RegisterProcessorCreator(new StaticProcessorCreator()); RegisterProcessorCreator(new StaticProcessorCreator()); + RegisterProcessorCreator(new StaticProcessorCreator()); #if defined(__linux__) && !defined(__ANDROID__) && !defined(__EXCLUDE_SPL__) if (BOOL_FLAG(enable_processor_spl)) { RegisterProcessorCreator(new StaticProcessorCreator()); diff --git a/core/pipeline/queue/ProcessQueueManager.h b/core/pipeline/queue/ProcessQueueManager.h index dbe47efeaa..abe0f39c2b 100644 --- a/core/pipeline/queue/ProcessQueueManager.h +++ b/core/pipeline/queue/ProcessQueueManager.h @@ -93,6 +93,7 @@ class ProcessQueueManager : public FeedbackInterface { void Clear(); friend class ProcessQueueManagerUnittest; friend class PipelineUnittest; + friend class HostMonitorInputRunnerUnittest; #endif }; diff --git a/core/plugin/input/InputHostMeta.cpp b/core/plugin/input/InputHostMeta.cpp new file mode 100644 index 0000000000..b8dd6c93fd --- /dev/null +++ b/core/plugin/input/InputHostMeta.cpp @@ -0,0 +1,65 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "InputHostMeta.h" + +#include +#include + +#include "HostMonitorInputRunner.h" +#include "Logger.h" +#include "PluginRegistry.h" +#include "ProcessorInstance.h" +#include "constants/EntityConstants.h" +#include "json/value.h" +#include "pipeline/Pipeline.h" +#include "plugin/processor/inner/ProcessorHostMetaNative.h" + +namespace logtail { + +const std::string InputHostMeta::sName = "input_host_meta"; + +bool InputHostMeta::Init(const Json::Value& config, Json::Value& optionalGoPipeline) { + return CreateInnerProcessors(config); +} + +bool InputHostMeta::Start() { + LOG_INFO(sLogger, ("input host meta start", mContext->GetConfigName())); + HostMonitorInputRunner::GetInstance()->Init(); + HostMonitorInputRunner::GetInstance()->UpdateCollector( + mContext->GetConfigName(), {"process"}, mContext->GetProcessQueueKey()); + return true; +} + +bool InputHostMeta::Stop(bool isPipelineRemoving) { + LOG_INFO(sLogger, ("input host meta stop", mContext->GetConfigName())); + HostMonitorInputRunner::GetInstance()->RemoveCollector(mContext->GetConfigName()); + return true; +} + +bool InputHostMeta::CreateInnerProcessors(const Json::Value& config) { + std::unique_ptr processor; + { + processor = PluginRegistry::GetInstance()->CreateProcessor(ProcessorHostMetaNative::sName, + mContext->GetPipeline().GenNextPluginMeta(false)); + Json::Value detail; + if (!processor->Init(detail, *mContext)) { + return false; + } + mInnerProcessors.emplace_back(std::move(processor)); + } + return true; +} + +} // namespace logtail diff --git a/core/plugin/input/InputHostMeta.h b/core/plugin/input/InputHostMeta.h new file mode 100644 index 0000000000..94587705ba --- /dev/null +++ b/core/plugin/input/InputHostMeta.h @@ -0,0 +1,46 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "json/value.h" +#include "pipeline/plugin/interface/Input.h" + +namespace logtail { + +class InputHostMeta : public Input { +public: + static const std::string sName; + + const std::string& Name() const override { return sName; } + bool Init(const Json::Value& config, Json::Value& optionalGoPipeline) override; + bool Start() override; + bool Stop(bool isPipelineRemoving) override; + bool SupportAck() const override { return false; } + +private: + bool CreateInnerProcessors(const Json::Value& config); + + std::string mDomain; + std::string mEntityType; + std::string mHostEntityID; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class InputHostMetaUnittest; +#endif +}; + +} // namespace logtail diff --git a/core/plugin/processor/inner/ProcessorHostMetaNative.cpp b/core/plugin/processor/inner/ProcessorHostMetaNative.cpp new file mode 100644 index 0000000000..1b8b845e40 --- /dev/null +++ b/core/plugin/processor/inner/ProcessorHostMetaNative.cpp @@ -0,0 +1,117 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ProcessorHostMetaNative.h" + +#include +#include +#include +#include + +#include "Common.h" +#include "LogEvent.h" +#include "Logger.h" +#include "MachineInfoUtil.h" +#include "PipelineEventGroup.h" +#include "StringTools.h" +#include "constants/EntityConstants.h" + +namespace logtail { + +const std::string ProcessorHostMetaNative::sName = "processor_host_meta_native"; + +bool ProcessorHostMetaNative::Init(const Json::Value& config) { + auto hostType = ToString(getenv(DEFAULT_ENV_KEY_HOST_TYPE.c_str())); + std::ostringstream oss; + if (hostType == DEFAULT_ENV_VALUE_ECS) { + oss << DEFAULT_CONTENT_VALUE_DOMAIN_ACS << "." << DEFAULT_ENV_VALUE_ECS << "." + << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; + ECSMeta ecsMeta = FetchECSMeta(); + mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_ACS; + mHostEntityID = ecsMeta.instanceID; + } else { + oss << DEFAULT_CONTENT_VALUE_DOMAIN_INFRA << "." << DEFAULT_ENV_VALUE_HOST << "." + << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; + mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_INFRA; + mHostEntityID = GetHostIp(); + } + mEntityType = oss.str(); + return true; +} + +void ProcessorHostMetaNative::Process(PipelineEventGroup& group) { + EventsContainer& events = group.MutableEvents(); + EventsContainer newEvents; + + for (auto& event : events) { + ProcessEvent(group, std::move(event), newEvents); + } + events.swap(newEvents); +} + +bool ProcessorHostMetaNative::IsSupportedEvent(const PipelineEventPtr& event) const { + return event.Is(); +} + +void ProcessorHostMetaNative::ProcessEvent(PipelineEventGroup& group, + PipelineEventPtr&& e, + EventsContainer& newEvents) { + if (!IsSupportedEvent(e)) { + newEvents.emplace_back(std::move(e)); + return; + } + + auto& sourceEvent = e.Cast(); + std::unique_ptr targetEvent = group.CreateLogEvent(true); + targetEvent->SetTimestamp(sourceEvent.GetTimestamp()); + + // TODO: support host entity + targetEvent->SetContent(DEFAULT_CONTENT_KEY_ENTITY_TYPE, mEntityType); + targetEvent->SetContent(DEFAULT_CONTENT_KEY_ENTITY_ID, + GetProcessEntityID(sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_PID), + sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME))); + targetEvent->SetContent(DEFAULT_CONTENT_KEY_DOMAIN, mDomain); + targetEvent->SetContent(DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME, + sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME)); + targetEvent->SetContent(DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME, + group.GetMetadata(EventGroupMetaKey::HOST_MONITOR_COLLECT_TIME)); + targetEvent->SetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS, "30"); + // TODO: support delete event + targetEvent->SetContent(DEFAULT_CONTENT_KEY_METHOD, DEFAULT_CONTENT_VALUE_METHOD_UPDATE); + + targetEvent->SetContent("pid", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_PID)); + targetEvent->SetContent("ppid", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_PPID)); + targetEvent->SetContent("user", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_USER)); + targetEvent->SetContent("comm", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_COMM)); + targetEvent->SetContent("create_time", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME)); + targetEvent->SetContent("cwd", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CWD)); + targetEvent->SetContent("binary", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_BINARY)); + targetEvent->SetContent("arguments", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS)); + targetEvent->SetContent("language", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE)); + targetEvent->SetContent("containerID", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID)); + newEvents.emplace_back(std::move(targetEvent), true, nullptr); +} + + +const std::string ProcessorHostMetaNative::GetProcessEntityID(StringView pid, StringView createTime) { + std::ostringstream oss; + oss << mHostEntityID << pid << createTime; + auto bigID = sdk::CalcMD5(oss.str()); + std::transform(bigID.begin(), bigID.end(), bigID.begin(), ::tolower); + return bigID; +} + +} // namespace logtail diff --git a/core/plugin/processor/inner/ProcessorHostMetaNative.h b/core/plugin/processor/inner/ProcessorHostMetaNative.h new file mode 100644 index 0000000000..3b32e20b06 --- /dev/null +++ b/core/plugin/processor/inner/ProcessorHostMetaNative.h @@ -0,0 +1,48 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "PipelineEventPtr.h" +#include "Processor.h" + +namespace logtail { +class ProcessorHostMetaNative : public Processor { +public: + static const std::string sName; + + const std::string& Name() const override { return sName; } + bool Init(const Json::Value& config) override; + void Process(PipelineEventGroup& group) override; + +protected: + bool IsSupportedEvent(const PipelineEventPtr& event) const override; + +private: + void ProcessEvent(PipelineEventGroup& group, PipelineEventPtr&& e, EventsContainer& newEvents); + + const std::string GetProcessEntityID(StringView pid, StringView createTime); + + std::string mDomain; + std::string mEntityType; + std::string mHostEntityID; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class ProcessorHostMetaNativeUnittest; +#endif +}; + +} // namespace logtail diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index b7ef0beac7..2bd2e77cb9 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -245,17 +245,17 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { long statusCode = 0; curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); request->mResponse.SetStatusCode(statusCode); - static_cast(request->mItem->mFlusher)->OnSendDone(request->mResponse, request->mItem); - FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); - mOutSuccessfulItemsTotal->Add(1); - mSuccessfulItemTotalResponseTimeMs->Add(responseTime); - mSendingItemsTotal->Sub(1); LOG_DEBUG( sLogger, ("send http request succeeded, item address", request->mItem)( "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( "response time", ToString(responseTimeMs) + "ms")("try cnt", ToString(request->mTryCnt))( "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); + static_cast(request->mItem->mFlusher)->OnSendDone(request->mResponse, request->mItem); + FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); + mOutSuccessfulItemsTotal->Add(1); + mSuccessfulItemTotalResponseTimeMs->Add(responseTime); + mSendingItemsTotal->Sub(1); break; } default: @@ -277,9 +277,6 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { ++runningHandlers; requestReused = true; } else { - static_cast(request->mItem->mFlusher) - ->OnSendDone(request->mResponse, request->mItem); - FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); LOG_DEBUG(sLogger, ("failed to send http request", "abort")("item address", request->mItem)( "config-flusher-dst", @@ -287,6 +284,9 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { "response time", ToString(responseTimeMs) + "ms")("try cnt", ToString(request->mTryCnt))( "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); + static_cast(request->mItem->mFlusher) + ->OnSendDone(request->mResponse, request->mItem); + FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); } mOutFailedItemsTotal->Add(1); mFailedItemTotalResponseTimeMs->Add(responseTime); diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 41f1601069..f77bb6998e 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -58,6 +58,7 @@ macro(add_core_subdir) add_subdirectory(prometheus) add_subdirectory(route) add_subdirectory(task_pipeline) + add_subdirectory(host_monitor) endmacro() macro(add_spl_subdir) diff --git a/core/unittest/host_monitor/1/stat b/core/unittest/host_monitor/1/stat new file mode 100644 index 0000000000..cbda9f715f --- /dev/null +++ b/core/unittest/host_monitor/1/stat @@ -0,0 +1 @@ +1 (cat) R 0 1 1 34816 1 4194560 1110 0 0 0 1 1 0 0 20 0 1 0 18938584 4505600 171 18446744073709551615 4194304 4238788 140727020025920 0 0 0 0 0 0 0 0 0 17 3 0 0 0 0 0 6336016 6337300 21442560 140727020027760 140727020027777 140727020027777 140727020027887 0 \ No newline at end of file diff --git a/core/unittest/host_monitor/CMakeLists.txt b/core/unittest/host_monitor/CMakeLists.txt new file mode 100644 index 0000000000..7a412f8e81 --- /dev/null +++ b/core/unittest/host_monitor/CMakeLists.txt @@ -0,0 +1,38 @@ +# Copyright 2023 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.22) +project(host_monitor_unittest) + +add_executable(collector_manager_unittest CollectorManagerUnittest.cpp) +target_link_libraries(collector_manager_unittest ${UT_BASE_TARGET}) + +add_executable(process_collector_unittest ProcessCollectorUnittest.cpp) +target_link_libraries(process_collector_unittest ${UT_BASE_TARGET}) + +add_executable(host_monitor_input_runner_unittest HostMonitorInputRunnerUnittest.cpp) +target_link_libraries(host_monitor_input_runner_unittest ${UT_BASE_TARGET}) + +add_executable(system_information_tools_unittest SystemInformationToolsUnittest.cpp) +target_link_libraries(system_information_tools_unittest ${UT_BASE_TARGET}) + +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/1) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/1/ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/1/) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/stat DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +include(GoogleTest) +gtest_discover_tests(collector_manager_unittest) +gtest_discover_tests(process_collector_unittest) +gtest_discover_tests(host_monitor_input_runner_unittest) +gtest_discover_tests(system_information_tools_unittest) diff --git a/core/unittest/host_monitor/CollectorManagerUnittest.cpp b/core/unittest/host_monitor/CollectorManagerUnittest.cpp new file mode 100644 index 0000000000..595be90ded --- /dev/null +++ b/core/unittest/host_monitor/CollectorManagerUnittest.cpp @@ -0,0 +1,42 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "CollectorManager.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class CollectorManagerUnittest : public testing::Test { +public: + void TestGetCollector() const; +}; + +void CollectorManagerUnittest::TestGetCollector() const { + { + auto collector1 = CollectorManager::GetInstance()->GetCollector("mock"); + APSARA_TEST_NOT_EQUAL_FATAL(nullptr, collector1); + APSARA_TEST_EQUAL_FATAL("mock", collector1->GetName()); + auto collector2 = CollectorManager::GetInstance()->GetCollector("mock"); + APSARA_TEST_EQUAL_FATAL(collector1, collector2); + } +} + + +UNIT_TEST_CASE(CollectorManagerUnittest, TestGetCollector); + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp new file mode 100644 index 0000000000..76946515fd --- /dev/null +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -0,0 +1,78 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "HostMonitorInputRunner.h" +#include "Logger.h" +#include "ProcessQueueItem.h" +#include "ProcessQueueManager.h" +#include "QueueKey.h" +#include "QueueKeyManager.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class HostMonitorInputRunnerUnittest : public testing::Test { +public: + void TestUpdateAndRemoveCollector() const; + void TestScheduleOnce() const; +}; + +void HostMonitorInputRunnerUnittest::TestUpdateAndRemoveCollector() const { + auto runner = HostMonitorInputRunner::GetInstance(); + runner->Init(); + runner->UpdateCollector("test", {"mock"}, QueueKey{}); + APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid("test", "mock")); + APSARA_TEST_TRUE_FATAL(runner->HasRegisteredPlugins()); + runner->RemoveCollector("test"); + APSARA_TEST_FALSE_FATAL(runner->IsCollectTaskValid("test", "mock")); + APSARA_TEST_FALSE_FATAL(runner->HasRegisteredPlugins()); + runner->Stop(); +} + +void HostMonitorInputRunnerUnittest::TestScheduleOnce() const { + auto runner = HostMonitorInputRunner::GetInstance(); + runner->mTimer = std::make_shared(); + runner->Init(); + runner->mThreadPool->Start(); + std::string configName = "test"; + auto queueKey = QueueKeyManager::GetInstance()->GetKey(configName); + auto ctx = PipelineContext(); + ctx.SetConfigName(configName); + ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(queueKey, 0, ctx); + + HostMonitorTimerEvent event(std::chrono::steady_clock::now(), 15, configName, "mock", queueKey); + runner->ScheduleOnce(&event); + std::this_thread::sleep_for(std::chrono::seconds(1)); + auto item = std::unique_ptr(new ProcessQueueItem(std::make_shared(), 0)); + ProcessQueueManager::GetInstance()->EnablePop(configName); + APSARA_TEST_TRUE_FATAL(ProcessQueueManager::GetInstance()->PopItem(0, item, configName)); + APSARA_TEST_EQUAL_FATAL("test", configName); + APSARA_TEST_TRUE_FATAL(item->mEventGroup.GetEvents().size() == 1); + + // verify schdule next + APSARA_TEST_EQUAL_FATAL(runner->mTimer->mQueue.size(), 1); + runner->mThreadPool->Stop(); + runner->Stop(); +} + +UNIT_TEST_CASE(HostMonitorInputRunnerUnittest, TestUpdateAndRemoveCollector); +UNIT_TEST_CASE(HostMonitorInputRunnerUnittest, TestScheduleOnce); + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/host_monitor/ProcessCollectorUnittest.cpp b/core/unittest/host_monitor/ProcessCollectorUnittest.cpp new file mode 100644 index 0000000000..8e4c8c327c --- /dev/null +++ b/core/unittest/host_monitor/ProcessCollectorUnittest.cpp @@ -0,0 +1,58 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ProcessCollector.h" +#include "host_monitor/Constants.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class ProcessCollectorUnittest : public testing::Test { +public: + void TestReadProcessStat() const; + void TestSortProcessByCpu() const; +}; + +void ProcessCollectorUnittest::TestReadProcessStat() const { + PROCESS_DIR = "."; + auto collector = ProcessCollector(); + auto ptr = collector.ReadProcessStat(1); + APSARA_TEST_NOT_EQUAL(nullptr, ptr); + APSARA_TEST_EQUAL(1, ptr->pid); + APSARA_TEST_EQUAL("cat", ptr->name); +} + +void ProcessCollectorUnittest::TestSortProcessByCpu() const { + PROCESS_DIR = "/proc"; + auto collector = ProcessCollector(); + auto processes = vector(); + collector.SortProcessByCpu(processes, 5); // fist time will be ignored + collector.SortProcessByCpu(processes, 5); + APSARA_TEST_EQUAL(5, processes.size()); + auto prev = processes[0]; + for (auto i = 1; i < processes.size(); i++) { + auto process = processes[i]; + APSARA_TEST_TRUE(process->cpuInfo.percent <= prev->cpuInfo.percent); + prev = process; + } +} + +UNIT_TEST_CASE(ProcessCollectorUnittest, TestReadProcessStat); +UNIT_TEST_CASE(ProcessCollectorUnittest, TestSortProcessByCpu); + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/host_monitor/SystemInformationToolsUnittest.cpp b/core/unittest/host_monitor/SystemInformationToolsUnittest.cpp new file mode 100644 index 0000000000..5da4e004d8 --- /dev/null +++ b/core/unittest/host_monitor/SystemInformationToolsUnittest.cpp @@ -0,0 +1,38 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "SystemInformationTools.h" +#include "host_monitor/Constants.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class SystemInformationToolsUnittest : public testing::Test { +public: + void TestGetSystemBootSeconds() const; +}; + +void SystemInformationToolsUnittest::TestGetSystemBootSeconds() const { + PROCESS_DIR = "."; + APSARA_TEST_EQUAL(1731142542, GetSystemBootSeconds()); +} + + +UNIT_TEST_CASE(SystemInformationToolsUnittest, TestGetSystemBootSeconds); + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/host_monitor/stat b/core/unittest/host_monitor/stat new file mode 100644 index 0000000000..2499f9d324 --- /dev/null +++ b/core/unittest/host_monitor/stat @@ -0,0 +1 @@ +btime 1731142542 \ No newline at end of file diff --git a/core/unittest/input/CMakeLists.txt b/core/unittest/input/CMakeLists.txt index 49129244cf..3e18f939f5 100644 --- a/core/unittest/input/CMakeLists.txt +++ b/core/unittest/input/CMakeLists.txt @@ -36,6 +36,9 @@ target_link_libraries(input_ebpf_network_security_unittest unittest_base) add_executable(input_ebpf_network_observer_unittest InputNetworkObserverUnittest.cpp) target_link_libraries(input_ebpf_network_observer_unittest unittest_base) +add_executable(input_host_meat_unittest InputHostMetaUnittest.cpp) +target_link_libraries(input_host_meat_unittest unittest_base) + include(GoogleTest) gtest_discover_tests(input_file_unittest) gtest_discover_tests(input_container_stdio_unittest) @@ -44,3 +47,4 @@ gtest_discover_tests(input_ebpf_file_security_unittest) gtest_discover_tests(input_ebpf_process_security_unittest) gtest_discover_tests(input_ebpf_network_security_unittest) gtest_discover_tests(input_ebpf_network_observer_unittest) +gtest_discover_tests(input_host_meat_unittest) diff --git a/core/unittest/input/InputHostMetaUnittest.cpp b/core/unittest/input/InputHostMetaUnittest.cpp new file mode 100644 index 0000000000..9c284f945a --- /dev/null +++ b/core/unittest/input/InputHostMetaUnittest.cpp @@ -0,0 +1,131 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "PluginRegistry.h" +#include "common/JsonUtil.h" +#include "ebpf/config.h" +#include "pipeline/Pipeline.h" +#include "plugin/input/InputHostMeta.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class InputHostMetaUnittest : public testing::Test { +public: + void TestName(); + void TestSupportAck(); + void OnSuccessfulInit(); + void OnFailedInit(); + void OnSuccessfulStart(); + void OnSuccessfulStop(); + // void OnPipelineUpdate(); + +protected: + void SetUp() override { + p.mName = "test_config"; + ctx.SetConfigName("test_config"); + ctx.SetPipeline(p); + PluginRegistry::GetInstance()->LoadPlugins(); + } + +private: + Pipeline p; + PipelineContext ctx; +}; + +void InputHostMetaUnittest::TestName() { + InputHostMeta input; + std::string name = input.Name(); + APSARA_TEST_EQUAL(name, "input_host_meta"); +} + +void InputHostMetaUnittest::TestSupportAck() { + InputHostMeta input; + bool supportAck = input.SupportAck(); + APSARA_TEST_FALSE(supportAck); +} + +void InputHostMetaUnittest::OnSuccessfulInit() { + unique_ptr input; + Json::Value configJson, optionalGoPipeline; + string configStr, errorMsg; + + // valid optional param + configStr = R"( + { + "Type": "input_host_meta" + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + input.reset(new InputHostMeta()); + input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); + APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); + APSARA_TEST_EQUAL(input->sName, "input_host_meta"); +} + +void InputHostMetaUnittest::OnFailedInit() { +} + +void InputHostMetaUnittest::OnSuccessfulStart() { + unique_ptr input; + Json::Value configJson, optionalGoPipeline; + string configStr, errorMsg; + + configStr = R"( + { + "Type": "input_host_meta" + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + input.reset(new InputHostMeta()); + input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); + APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); + APSARA_TEST_TRUE(input->Start()); +} + +void InputHostMetaUnittest::OnSuccessfulStop() { + unique_ptr input; + Json::Value configJson, optionalGoPipeline; + string configStr, errorMsg; + + configStr = R"( + { + "Type": "input_host_meta" + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + input.reset(new InputHostMeta()); + input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); + APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); + APSARA_TEST_TRUE(input->Start()); + APSARA_TEST_TRUE(input->Stop(false)); +} + +UNIT_TEST_CASE(InputHostMetaUnittest, TestName) +UNIT_TEST_CASE(InputHostMetaUnittest, TestSupportAck) +UNIT_TEST_CASE(InputHostMetaUnittest, OnSuccessfulInit) +UNIT_TEST_CASE(InputHostMetaUnittest, OnFailedInit) +UNIT_TEST_CASE(InputHostMetaUnittest, OnSuccessfulStart) +UNIT_TEST_CASE(InputHostMetaUnittest, OnSuccessfulStop) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/processor/CMakeLists.txt b/core/unittest/processor/CMakeLists.txt index 98cb60a55d..ad62741998 100644 --- a/core/unittest/processor/CMakeLists.txt +++ b/core/unittest/processor/CMakeLists.txt @@ -63,6 +63,9 @@ target_link_libraries(boost_regex_benchmark ${UT_BASE_TARGET}) add_executable(processor_prom_parse_metric_native_unittest ProcessorPromParseMetricNativeUnittest.cpp) target_link_libraries(processor_prom_parse_metric_native_unittest unittest_base) +add_executable(processor_host_meta_native_unittest ProcessorHostMetaNativeUnittest.cpp) +target_link_libraries(processor_host_meta_native_unittest unittest_base) + include(GoogleTest) gtest_discover_tests(processor_split_log_string_native_unittest) gtest_discover_tests(processor_split_multiline_log_string_native_unittest) @@ -78,3 +81,4 @@ gtest_discover_tests(processor_desensitize_native_unittest) gtest_discover_tests(processor_merge_multiline_log_native_unittest) gtest_discover_tests(processor_parse_container_log_native_unittest) gtest_discover_tests(processor_prom_parse_metric_native_unittest) +gtest_discover_tests(processor_host_meta_native_unittest) diff --git a/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp b/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp new file mode 100644 index 0000000000..cc60ecda1b --- /dev/null +++ b/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp @@ -0,0 +1,120 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "LogEvent.h" +#include "constants/EntityConstants.h" +#include "pipeline/Pipeline.h" +#include "plugin/processor/inner/ProcessorHostMetaNative.h" +#include "unittest/Unittest.h" + +namespace logtail { + +class ProcessorHostMetaNativeUnittest : public ::testing::Test { +public: + void TestInit(); + void TestProcess(); + void TestGetProcessEntityID(); + +protected: + void SetUp() override { mContext.SetConfigName("project##config_0"); } + +private: + PipelineContext mContext; +}; + +void ProcessorHostMetaNativeUnittest::TestInit() { + // make config + Json::Value config; + Pipeline pipeline; + mContext.SetPipeline(pipeline); + + { + setenv(DEFAULT_ENV_KEY_HOST_TYPE.c_str(), DEFAULT_ENV_VALUE_ECS.c_str(), 1); + ProcessorHostMetaNative processor; + processor.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processor.Init(config)); + APSARA_TEST_EQUAL_FATAL(processor.mDomain, DEFAULT_CONTENT_VALUE_DOMAIN_ACS); + APSARA_TEST_EQUAL_FATAL(processor.mEntityType, + DEFAULT_CONTENT_VALUE_DOMAIN_ACS + "." + DEFAULT_ENV_VALUE_ECS + "." + + DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS); + } + { + setenv(DEFAULT_ENV_KEY_HOST_TYPE.c_str(), DEFAULT_CONTENT_VALUE_DOMAIN_INFRA.c_str(), 1); + ProcessorHostMetaNative processor; + processor.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processor.Init(config)); + APSARA_TEST_EQUAL_FATAL(processor.mDomain, DEFAULT_CONTENT_VALUE_DOMAIN_INFRA); + APSARA_TEST_EQUAL_FATAL(processor.mEntityType, + DEFAULT_CONTENT_VALUE_DOMAIN_INFRA + "." + DEFAULT_ENV_VALUE_HOST + "." + + DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS); + } +} + +void ProcessorHostMetaNativeUnittest::TestProcess() { + // make config + Json::Value config; + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.SetMetadataNoCopy(EventGroupMetaKey::HOST_MONITOR_COLLECT_TIME, "123456"); + auto event = eventGroup.AddLogEvent(); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PID, "123"); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME, "123456"); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PPID, "123"); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_USER, "root"); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_COMM, "comm"); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CWD, "cwd"); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_BINARY, "binary"); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS, "arguments"); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE, "language"); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID, "container_id"); + + ProcessorHostMetaNative processor; + processor.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processor.Init(config)); + processor.Process(eventGroup); + APSARA_TEST_EQUAL_FATAL(eventGroup.GetEvents().size(), 1); + auto newEvent = eventGroup.GetEvents().front().Cast(); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_DOMAIN), processor.mDomain); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_ENTITY_TYPE), processor.mEntityType); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME), "123456"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS), "30"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_METHOD), DEFAULT_CONTENT_VALUE_METHOD_UPDATE); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("pid"), "123"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("ppid"), "123"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("user"), "root"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("comm"), "comm"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("create_time"), "123456"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("cwd"), "cwd"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("binary"), "binary"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("arguments"), "arguments"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("language"), "language"); + APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("containerID"), "container_id"); +} + +void ProcessorHostMetaNativeUnittest::TestGetProcessEntityID() { + ProcessorHostMetaNative processor; + processor.Init(Json::Value()); + processor.mHostEntityID = "123"; + APSARA_TEST_EQUAL(processor.GetProcessEntityID("123", "123"), "f5bb0c8de146c67b44babbf4e6584cc0"); +} + +UNIT_TEST_CASE(ProcessorHostMetaNativeUnittest, TestInit) +UNIT_TEST_CASE(ProcessorHostMetaNativeUnittest, TestProcess) +UNIT_TEST_CASE(ProcessorHostMetaNativeUnittest, TestGetProcessEntityID) + +} // namespace logtail + +UNIT_TEST_MAIN From 8f9f8ee0760557c790376cfced1981553a944ac2 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 29 Nov 2024 00:01:27 +0800 Subject: [PATCH 02/13] fix --- core/application/Application.cpp | 6 +- core/common/FileSystemUtil.cpp | 4 +- core/common/FileSystemUtil.h | 8 +- core/common/MachineInfoUtil.cpp | 93 +++++++++++ core/common/MachineInfoUtil.h | 8 +- core/common/StringTools.cpp | 2 +- core/common/common.cmake | 2 +- core/common/timer/HostMonitorTimerEvent.h | 60 ------- core/common/timer/Timer.cpp | 6 + core/common/timer/Timer.h | 10 ++ core/host_monitor/Constants.cpp | 6 +- core/host_monitor/Constants.h | 10 +- core/host_monitor/HostMonitorInputRunner.cpp | 147 ++++++++++-------- core/host_monitor/HostMonitorInputRunner.h | 31 ++-- .../HostMonitorTimerEvent.cpp | 7 +- core/host_monitor/HostMonitorTimerEvent.h | 62 ++++++++ core/host_monitor/collector/BaseCollector.h | 0 .../collector/CollectorManager.cpp | 44 ------ .../host_monitor/collector/CollectorManager.h | 44 ------ core/host_monitor/collector/MockCollector.cpp | 33 ---- core/host_monitor/collector/MockCollector.h | 36 ----- .../collector/ProcessCollector.cpp | 12 +- .../host_monitor/collector/ProcessCollector.h | 9 +- core/models/PipelineEventGroup.h | 2 +- core/pipeline/PipelineManager.cpp | 1 + core/plugin/input/InputHostMeta.cpp | 6 +- core/plugin/input/InputHostMeta.h | 7 +- .../inner/ProcessorHostMetaNative.cpp | 6 +- core/unittest/host_monitor/CMakeLists.txt | 4 - .../host_monitor/CollectorManagerUnittest.cpp | 42 ----- .../HostMonitorInputRunnerUnittest.cpp | 18 ++- .../ProcessorHostMetaNativeUnittest.cpp | 2 +- 32 files changed, 326 insertions(+), 402 deletions(-) delete mode 100644 core/common/timer/HostMonitorTimerEvent.h mode change 100644 => 100755 core/host_monitor/HostMonitorInputRunner.cpp rename core/{common/timer => host_monitor}/HostMonitorTimerEvent.cpp (78%) create mode 100644 core/host_monitor/HostMonitorTimerEvent.h mode change 100644 => 100755 core/host_monitor/collector/BaseCollector.h delete mode 100644 core/host_monitor/collector/CollectorManager.cpp delete mode 100644 core/host_monitor/collector/CollectorManager.h delete mode 100644 core/host_monitor/collector/MockCollector.cpp delete mode 100644 core/host_monitor/collector/MockCollector.h mode change 100644 => 100755 core/host_monitor/collector/ProcessCollector.cpp mode change 100644 => 100755 core/host_monitor/collector/ProcessCollector.h delete mode 100644 core/unittest/host_monitor/CollectorManagerUnittest.cpp diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 09901cced9..7cd18c73e3 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -14,6 +14,8 @@ #include "application/Application.h" +#include "timer/Timer.h" + #ifndef LOGTAIL_NO_TC_MALLOC #include #endif @@ -263,9 +265,9 @@ void Application::Start() { // GCOVR_EXCL_START } ProcessorRunner::GetInstance()->Init(); + Timer::GetInstance()->Init(); - time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, - lastCheckTagsTime = 0, lastQueueGCTime = 0; + time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, lastCheckTagsTime = 0, lastQueueGCTime = 0; #ifndef LOGTAIL_NO_TC_MALLOC time_t lastTcmallocReleaseMemTime = 0; #endif diff --git a/core/common/FileSystemUtil.cpp b/core/common/FileSystemUtil.cpp index 80d7e24fd0..28d51fb9f2 100644 --- a/core/common/FileSystemUtil.cpp +++ b/core/common/FileSystemUtil.cpp @@ -156,7 +156,7 @@ int GetLines(std::istream& is, return 0; } -int GetLines(const bfs::path& filename, +int GetLines(const std::filesystem::path& filename, bool enableEmptyLine, const std::function& pushBack, std::string* errorMessage) { @@ -178,7 +178,7 @@ int GetLines(const bfs::path& filename, return ret; } -int GetFileLines(const bfs::path& filename, +int GetFileLines(const std::filesystem::path& filename, std::vector& res, bool enableEmptyLine, std::string* errorMessage) { diff --git a/core/common/FileSystemUtil.h b/core/common/FileSystemUtil.h index 8e0f523600..4254c85b3f 100644 --- a/core/common/FileSystemUtil.h +++ b/core/common/FileSystemUtil.h @@ -26,14 +26,12 @@ #elif defined(_MSC_VER) #include #endif -#include +#include #include "DevInode.h" #include "ErrorUtil.h" #include "LogtailCommonFlags.h" -namespace bfs = boost::filesystem; - // Filesystem utility. namespace logtail { @@ -95,11 +93,11 @@ int GetLines(std::istream& is, bool enableEmptyLine, const std::function& pushBack, std::string* errorMessage); -int GetLines(const bfs::path& filename, +int GetLines(const std::filesystem::path& filename, bool enableEmptyLine, const std::function& pushBack, std::string* errorMessage); -int GetFileLines(const bfs::path& filename, +int GetFileLines(const std::filesystem::path& filename, std::vector& res, bool enableEmptyLine = true, std::string* errorMessage = nullptr); diff --git a/core/common/MachineInfoUtil.cpp b/core/common/MachineInfoUtil.cpp index 47e0650fa7..d545802427 100644 --- a/core/common/MachineInfoUtil.cpp +++ b/core/common/MachineInfoUtil.cpp @@ -15,6 +15,11 @@ #include "MachineInfoUtil.h" #include + +#include + +#include "AppConfig.h" +#include "common/UUIDUtil.h" #if defined(__linux__) #include #include @@ -40,8 +45,10 @@ #include "FileSystemUtil.h" #include "StringTools.h" +#include "common/FileSystemUtil.h" #include "logger/Logger.h" +DEFINE_FLAG_STRING(agent_host_id, "", ""); #if defined(_MSC_VER) typedef LONG NTSTATUS, *PNTSTATUS; @@ -496,6 +503,92 @@ size_t FetchECSMetaCallback(char* buffer, size_t size, size_t nmemb, std::string return sizes; } +std::string GetSerialNumberFromEcsAssist(const std::string& machineIdFile) { + std::string sn; + if (CheckExistance(machineIdFile)) { + if (!ReadFileContent(machineIdFile, sn)) { + return ""; + } + } + return sn; +} + +static std::string GetEcsAssistMachineIdFile() { +#if defined(WIN32) + return "C:\\ProgramData\\aliyun\\assist\\hybrid\\machine-id"; +#else + return "/usr/local/share/aliyun-assist/hybrid/machine-id"; +#endif +} + +std::string GetSerialNumberFromEcsAssist() { + return GetSerialNumberFromEcsAssist(GetEcsAssistMachineIdFile()); +} + +std::string RandomHostid() { + static std::string hostId = CalculateRandomUUID(); + return hostId; +} + +const std::string& GetLocalHostId() { + static std::string fileName = AppConfig::GetInstance()->GetLoongcollectorConfDir() + PATH_SEPARATOR + "host_id"; + static std::string hostId; + if (!hostId.empty()) { + return hostId; + } + if (CheckExistance(fileName)) { + if (!ReadFileContent(fileName, hostId)) { + hostId = ""; + } + } + if (hostId.empty()) { + hostId = RandomHostid(); + + LOG_INFO(sLogger, ("save hostId file to local file system, hostId", hostId)); + int fd = open(fileName.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0755); + if (fd == -1) { + int savedErrno = errno; + if (savedErrno != EEXIST) { + LOG_ERROR(sLogger, ("save hostId file fail", fileName)("errno", strerror(savedErrno))); + } + } else { + // 文件成功创建,现在写入hostId + ssize_t written = write(fd, hostId.c_str(), hostId.length()); + if (written == static_cast(hostId.length())) { + LOG_INFO(sLogger, ("hostId saved successfully to", fileName)); + } else { + int writeErrno = errno; + LOG_ERROR(sLogger, ("Failed to write hostId to file", fileName)("errno", strerror(writeErrno))); + } + close(fd); + } + } + return hostId; +} + +std::string FetchHostId() { + static std::string hostId; + if (!hostId.empty()) { + return hostId; + } + hostId = STRING_FLAG(agent_host_id); + if (!hostId.empty()) { + return hostId; + } + ECSMeta meta = FetchECSMeta(); + hostId = meta.instanceID; + if (!hostId.empty()) { + return hostId; + } + hostId = GetSerialNumberFromEcsAssist(); + if (!hostId.empty()) { + return hostId; + } + hostId = GetLocalHostId(); + + return hostId; +} + ECSMeta FetchECSMeta() { CURL* curl; for (size_t retryTimes = 1; retryTimes <= 5; retryTimes++) { diff --git a/core/common/MachineInfoUtil.h b/core/common/MachineInfoUtil.h index 0983cbf352..d8d6a3e963 100644 --- a/core/common/MachineInfoUtil.h +++ b/core/common/MachineInfoUtil.h @@ -15,10 +15,11 @@ */ #pragma once +#include +#include + #include #include -#include -#include namespace logtail { @@ -37,8 +38,9 @@ std::string GetHostIp(const std::string& intf = ""); void GetAllPids(std::unordered_set& pids); bool GetKernelInfo(std::string& kernelRelease, int64_t& kernelVersion); bool GetRedHatReleaseInfo(std::string& os, int64_t& osVersion, std::string bashPath = ""); -bool IsDigitsDotsHostname(const char *hostname); +bool IsDigitsDotsHostname(const char* hostname); ECSMeta FetchECSMeta(); +std::string FetchHostId(); // GetAnyAvailableIP walks through all interfaces (AF_INET) to find an available IP. // Priority: diff --git a/core/common/StringTools.cpp b/core/common/StringTools.cpp index 57b29cfde9..36ad405bfc 100644 --- a/core/common/StringTools.cpp +++ b/core/common/StringTools.cpp @@ -365,7 +365,7 @@ void RemoveFilePathTrailingSlash(std::string& filePath) { bool IsInt(const char* sz) { bool ok = (sz != nullptr && *sz != '\0'); - for (auto* it = reinterpret_cast(sz); ok && *it; ++it) { + for (auto* it = sz; ok && *it; ++it) { ok = (0 != std::isdigit(*it)); } return ok; diff --git a/core/common/common.cmake b/core/common/common.cmake index 94d715bf8d..cd5e9401c6 100644 --- a/core/common/common.cmake +++ b/core/common/common.cmake @@ -29,7 +29,7 @@ list(APPEND THIS_SOURCE_FILES_LIST ${XX_HASH_SOURCE_FILES}) # add memory in common list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/memory/SourceBuffer.h) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpRequest.cpp) -list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp ${CMAKE_SOURCE_DIR}/common/timer/HostMonitorTimerEvent.cpp) +list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/compression/Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/CompressorFactory.cpp ${CMAKE_SOURCE_DIR}/common/compression/LZ4Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/ZstdCompressor.cpp) # remove several files in common list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/BoostRegexValidator.cpp ${CMAKE_SOURCE_DIR}/common/GetUUID.cpp) diff --git a/core/common/timer/HostMonitorTimerEvent.h b/core/common/timer/HostMonitorTimerEvent.h deleted file mode 100644 index e9f3c08457..0000000000 --- a/core/common/timer/HostMonitorTimerEvent.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2024 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include "QueueKey.h" -#include "timer/TimerEvent.h" - -namespace logtail { - -class HostMonitorTimerEvent : public TimerEvent { -public: - HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime, - size_t interval, - std::string configName, - std::string collectorName, - QueueKey processQueueKey) - : TimerEvent(execTime), - mConfigName(std::move(configName)), - mCollectorName(collectorName), - mProcessQueueKey(processQueueKey), - mInputIdx(0) { - mInterval = std::chrono::seconds(interval); - } - - bool IsValid() const override; - bool Execute() override; - - const std::string GetConfigName() const { return mConfigName; } - const std::string GetCollectorName() const { return mCollectorName; } - const QueueKey GetProcessQueueKey() const { return mProcessQueueKey; } - int GetInputIndex() const { return mInputIdx; } - const std::chrono::seconds GetInterval() const { return mInterval; } - void ResetForNextExec() { SetExecTime(GetExecTime() + mInterval); } - -private: - std::string mConfigName; - std::string mCollectorName; - QueueKey mProcessQueueKey; - int mInputIdx; - std::chrono::seconds mInterval; -}; - -} // namespace logtail diff --git a/core/common/timer/Timer.cpp b/core/common/timer/Timer.cpp index 08ed7ed12c..57c7ab77f1 100644 --- a/core/common/timer/Timer.cpp +++ b/core/common/timer/Timer.cpp @@ -23,6 +23,9 @@ namespace logtail { void Timer::Init() { { lock_guard lock(mThreadRunningMux); + if (mIsThreadRunning) { + return; + } mIsThreadRunning = true; } mThreadRes = async(launch::async, &Timer::Run, this); @@ -31,6 +34,9 @@ void Timer::Init() { void Timer::Stop() { { lock_guard lock(mThreadRunningMux); + if (!mIsThreadRunning) { + return; + } mIsThreadRunning = false; } mCV.notify_one(); diff --git a/core/common/timer/Timer.h b/core/common/timer/Timer.h index 4b241e91d5..ad1ca15be4 100644 --- a/core/common/timer/Timer.h +++ b/core/common/timer/Timer.h @@ -34,11 +34,21 @@ struct TimerEventCompare { class Timer { public: + Timer(const Timer&) = delete; + Timer(Timer&&) = delete; + Timer& operator=(const Timer&) = delete; + Timer& operator=(Timer&&) = delete; + ~Timer() = default; + static Timer* GetInstance() { + static Timer sInstance; + return &sInstance; + } void Init(); void Stop(); void PushEvent(std::unique_ptr&& e); private: + Timer() = default; void Run(); mutable std::mutex mQueueMux; diff --git a/core/host_monitor/Constants.cpp b/core/host_monitor/Constants.cpp index 366b314480..4d538400e7 100644 --- a/core/host_monitor/Constants.cpp +++ b/core/host_monitor/Constants.cpp @@ -20,11 +20,11 @@ namespace logtail { #ifndef APSARA_UNIT_TEST_MAIN -const bfs::path PROCESS_DIR = "/proc"; +const std::filesystem::path PROCESS_DIR = "/proc"; #else -bfs::path PROCESS_DIR = "/proc"; +std::filesystem::path PROCESS_DIR = "/proc"; #endif -const bfs::path PROCESS_STAT = "stat"; +const std::filesystem::path PROCESS_STAT = "stat"; } // namespace logtail diff --git a/core/host_monitor/Constants.h b/core/host_monitor/Constants.h index 788065b81e..cdbf3e7dd2 100644 --- a/core/host_monitor/Constants.h +++ b/core/host_monitor/Constants.h @@ -16,18 +16,16 @@ #pragma once -#include - -namespace bfs = boost::filesystem; +#include namespace logtail { #ifndef APSARA_UNIT_TEST_MAIN -extern const bfs::path PROCESS_DIR; +extern const std::filesystem::path PROCESS_DIR; #else -extern bfs::path PROCESS_DIR; +extern std::filesystem::path PROCESS_DIR; #endif -const extern bfs::path PROCESS_STAT; +const extern std::filesystem::path PROCESS_STAT; } // namespace logtail diff --git a/core/host_monitor/HostMonitorInputRunner.cpp b/core/host_monitor/HostMonitorInputRunner.cpp old mode 100644 new mode 100755 index ab2618623e..926be0151f --- a/core/host_monitor/HostMonitorInputRunner.cpp +++ b/core/host_monitor/HostMonitorInputRunner.cpp @@ -16,80 +16,89 @@ #include "HostMonitorInputRunner.h" +#include #include #include +#include +#include +#include +#include -#include "CollectorManager.h" -#include "Lock.h" -#include "LogEvent.h" -#include "Logger.h" -#include "ProcessQueueItem.h" -#include "ProcessQueueManager.h" -#include "ThreadPool.h" -#include "timer/HostMonitorTimerEvent.h" -#include "timer/TimerEvent.h" +#include "HostMonitorTimerEvent.h" +#include "common/Lock.h" +#include "common/timer/Timer.h" +#include "host_monitor/collector/ProcessCollector.h" +#include "logger/Logger.h" +#include "runner/ProcessorRunner.h" namespace logtail { -HostMonitorInputRunner::HostMonitorInputRunner() { - mTimer = std::make_shared(); - mThreadPool = std::make_shared(3); +HostMonitorInputRunner::HostMonitorInputRunner() : mThreadPool(ThreadPool(3)) { + RegisterCollector(); } void HostMonitorInputRunner::UpdateCollector(const std::string& configName, - const std::vector& collectorNames, - QueueKey processQueueKey) { - WriteLock lock(mCollectorMapRWLock); - mCollectorMap[configName] = collectorNames; - for (const auto& collectorName : collectorNames) { + const std::vector& newCollectors, + QueueKey processQueueKey, + int inputIndex) { + std::vector oldCollectors; + { + std::unique_lock lock(mCollectorRegisterMapMutex); + auto it = mCollectorRegisterMap.find(configName); + if (it != mCollectorRegisterMap.end()) { + oldCollectors = it->second; + } + mCollectorRegisterMap[configName] = newCollectors; + } + for (const auto& collectorName : newCollectors) { LOG_INFO(sLogger, ("add new host monitor collector", configName)("collector", collectorName)); - mTimer->PushEvent(BuildTimerEvent(configName, collectorName, processQueueKey)); + auto collectConfig = std::make_unique( + configName, collectorName, processQueueKey, inputIndex, std::chrono::seconds(DEFAULT_SCHEDULE_INTERVAL)); + // only push event when the collector is new added + if (std::find(oldCollectors.begin(), oldCollectors.end(), collectorName) == oldCollectors.end()) { + Timer::GetInstance()->PushEvent(BuildTimerEvent(std::move(collectConfig))); + } } } void HostMonitorInputRunner::RemoveCollector(const std::string& configName) { - WriteLock lock(mCollectorMapRWLock); - mCollectorMap.erase(configName); + std::unique_lock lock(mCollectorRegisterMapMutex); + mCollectorRegisterMap.erase(configName); } void HostMonitorInputRunner::Init() { - std::lock_guard lock(mStartMutex); - if (mIsStarted) { + if (mIsStarted.exchange(true)) { return; } LOG_INFO(sLogger, ("HostMonitorInputRunner", "Start")); - mIsStarted = true; - #ifndef APSARA_UNIT_TEST_MAIN - mThreadPool->Start(); - mTimer->Init(); + mThreadPool.Start(); #endif } void HostMonitorInputRunner::Stop() { - std::lock_guard lock(mStartMutex); - if (!mIsStarted) { + if (!mIsStarted.exchange(false)) { return; } - - mIsStarted = false; #ifndef APSARA_UNIT_TEST_MAIN - mTimer->Stop(); - mThreadPool->Stop(); + std::future result = std::async(std::launch::async, [this]() { mThreadPool.Stop(); }); + if (result.wait_for(std::chrono::seconds(3)) == std::future_status::timeout) { + LOG_ERROR(sLogger, ("HostMonitorInputRunner stop timeout 3 seconds", "may cause thread leak")); + } #endif LOG_INFO(sLogger, ("HostMonitorInputRunner", "Stop")); } bool HostMonitorInputRunner::HasRegisteredPlugins() const { - ReadLock lock(mCollectorMapRWLock); - return !mCollectorMap.empty(); + std::shared_lock lock(mCollectorRegisterMapMutex); + return !mCollectorRegisterMap.empty(); } bool HostMonitorInputRunner::IsCollectTaskValid(const std::string& configName, const std::string& collectorName) const { - ReadLock lock(mCollectorMapRWLock); - auto collectors = mCollectorMap.find(configName); - if (collectors == mCollectorMap.end()) { + std::shared_lock lock(mCollectorRegisterMapMutex); + auto collectors = mCollectorRegisterMap.find(configName); + if (collectors == mCollectorRegisterMap.end()) { return false; } for (const auto& collectorName : collectors->second) { @@ -100,46 +109,50 @@ bool HostMonitorInputRunner::IsCollectTaskValid(const std::string& configName, c return false; } -void HostMonitorInputRunner::ScheduleOnce(HostMonitorTimerEvent* event) { - // TODO: reuse event - HostMonitorTimerEvent eventCopy = *event; - mThreadPool->Add([this, eventCopy]() mutable { - auto configName = eventCopy.GetConfigName(); - auto collectorName = eventCopy.GetCollectorName(); - auto processQueueKey = eventCopy.GetProcessQueueKey(); +void HostMonitorInputRunner::ScheduleOnce(std::unique_ptr collectConfig) { + mThreadPool.Add([this, &collectConfig]() { PipelineEventGroup group(std::make_shared()); - auto collector = CollectorManager::GetInstance()->GetCollector(collectorName); - collector->Collect(group); - - std::unique_ptr item - = std::make_unique(std::move(group), eventCopy.GetInputIndex()); - if (ProcessQueueManager::GetInstance()->IsValidToPush(processQueueKey)) { - ProcessQueueManager::GetInstance()->PushQueue(processQueueKey, std::move(item)); - } else { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - // try again - if (ProcessQueueManager::GetInstance()->IsValidToPush(processQueueKey)) { - ProcessQueueManager::GetInstance()->PushQueue(processQueueKey, std::move(item)); - } else { - LOG_WARNING(sLogger, ("process queue is full", "discard data")("config", configName)); - } + auto collector = GetCollector(collectConfig->mCollectorName); + if (!collector) { + collector->Collect(group); } - LOG_DEBUG(sLogger, ("schedule host monitor collector again", configName)("collector", collectorName)); + bool result = ProcessorRunner::GetInstance()->PushQueue( + collectConfig->mProcessQueueKey, collectConfig->mInputIndex, std::move(group), 3); + if (!result) { + LOG_WARNING(sLogger, + ("push queue failed", "discard data")("config", collectConfig->mConfigName)( + "collector", collectConfig->mCollectorName)); + } + LOG_DEBUG(sLogger, + ("schedule host monitor collector again", collectConfig->mConfigName)("collector", + collectConfig->mCollectorName)); - eventCopy.ResetForNextExec(); - mTimer->PushEvent(std::make_unique(eventCopy)); + auto event = BuildTimerEvent(std::move(collectConfig)); + event->ResetForNextExec(); + Timer::GetInstance()->PushEvent(std::move(event)); }); } -std::unique_ptr HostMonitorInputRunner::BuildTimerEvent(const std::string& configName, - const std::string& collectorName, - QueueKey processQueueKey) { +std::unique_ptr +HostMonitorInputRunner::BuildTimerEvent(std::unique_ptr collectConfig) { auto now = std::chrono::steady_clock::now(); - auto event = std::make_unique( - now, DEFAULT_SCHEDULE_INTERVAL, configName, collectorName, processQueueKey); + auto event = std::make_unique(now, std::move(collectConfig)); return event; } +std::shared_ptr HostMonitorInputRunner::GetCollector(const std::string& collectorName) { + auto it = mCollectorInstanceMap.find(collectorName); + if (it == mCollectorInstanceMap.end()) { + return nullptr; + } + return it->second; +} + +template +void HostMonitorInputRunner::RegisterCollector() { + auto collector = std::make_shared(); + mCollectorInstanceMap[collector->GetName()] = collector; +} } // namespace logtail diff --git a/core/host_monitor/HostMonitorInputRunner.h b/core/host_monitor/HostMonitorInputRunner.h index 5aceb92c30..f83ffc3af9 100644 --- a/core/host_monitor/HostMonitorInputRunner.h +++ b/core/host_monitor/HostMonitorInputRunner.h @@ -16,16 +16,19 @@ #pragma once +#include #include +#include #include +#include #include +#include "BaseCollector.h" #include "InputRunner.h" #include "Lock.h" #include "QueueKey.h" #include "ThreadPool.h" -#include "timer/HostMonitorTimerEvent.h" -#include "timer/Timer.h" +#include "host_monitor/HostMonitorTimerEvent.h" namespace logtail { @@ -45,7 +48,8 @@ class HostMonitorInputRunner : public InputRunner { void UpdateCollector(const std::string& configName, const std::vector& collectorNames, - QueueKey processQueueKey); + QueueKey processQueueKey, + int inputIndex); void RemoveCollector(const std::string& configName); void Init() override; @@ -53,21 +57,24 @@ class HostMonitorInputRunner : public InputRunner { bool HasRegisteredPlugins() const override; bool IsCollectTaskValid(const std::string& configName, const std::string& collectorName) const; - void ScheduleOnce(HostMonitorTimerEvent* event); + void ScheduleOnce(std::unique_ptr collectConfig); private: HostMonitorInputRunner(); - std::unique_ptr - BuildTimerEvent(const std::string& configName, const std::string& collectorName, QueueKey processQueueKey); + std::unique_ptr + BuildTimerEvent(std::unique_ptr collectConfig); - bool mIsStarted = false; - std::mutex mStartMutex; + template + void RegisterCollector(); + std::shared_ptr GetCollector(const std::string& collectorName); - std::shared_ptr mTimer; - std::shared_ptr mThreadPool; + std::atomic_bool mIsStarted = false; - mutable ReadWriteLock mCollectorMapRWLock; - std::unordered_map> mCollectorMap; + ThreadPool mThreadPool; + + mutable std::shared_mutex mCollectorRegisterMapMutex; + std::unordered_map> mCollectorRegisterMap; + std::unordered_map> mCollectorInstanceMap; #ifdef APSARA_UNIT_TEST_MAIN friend class HostMonitorInputRunnerUnittest; diff --git a/core/common/timer/HostMonitorTimerEvent.cpp b/core/host_monitor/HostMonitorTimerEvent.cpp similarity index 78% rename from core/common/timer/HostMonitorTimerEvent.cpp rename to core/host_monitor/HostMonitorTimerEvent.cpp index 2c4fcbd355..3fb5079309 100644 --- a/core/common/timer/HostMonitorTimerEvent.cpp +++ b/core/host_monitor/HostMonitorTimerEvent.cpp @@ -16,16 +16,19 @@ #include "HostMonitorTimerEvent.h" +#include + #include "HostMonitorInputRunner.h" namespace logtail { bool HostMonitorTimerEvent::IsValid() const { - return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(mConfigName, mCollectorName); + return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(mCollectConfig->mConfigName, + mCollectConfig->mCollectorName); } bool HostMonitorTimerEvent::Execute() { - HostMonitorInputRunner::GetInstance()->ScheduleOnce(this); + HostMonitorInputRunner::GetInstance()->ScheduleOnce(std::move(mCollectConfig)); return true; } diff --git a/core/host_monitor/HostMonitorTimerEvent.h b/core/host_monitor/HostMonitorTimerEvent.h new file mode 100644 index 0000000000..07a3dceb40 --- /dev/null +++ b/core/host_monitor/HostMonitorTimerEvent.h @@ -0,0 +1,62 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "QueueKey.h" +#include "timer/TimerEvent.h" + +namespace logtail { + + +class HostMonitorTimerEvent : public TimerEvent { +public: + struct CollectConfig { + CollectConfig(std::string configName, + std::string collectorName, + QueueKey processQueueKey, + int inputIndex, + std::chrono::seconds interval) + : mConfigName(configName), + mCollectorName(collectorName), + mProcessQueueKey(processQueueKey), + mInputIndex(inputIndex), + mInterval(interval) {} + + std::string mConfigName; + std::string mCollectorName; + QueueKey mProcessQueueKey; + size_t mInputIndex; + std::chrono::seconds mInterval; + }; + + HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime, std::unique_ptr collectConfig) + : TimerEvent(execTime), mCollectConfig(std::move(collectConfig)) {} + + bool IsValid() const override; + bool Execute() override; + void ResetForNextExec() { SetExecTime(GetExecTime() + mCollectConfig->mInterval); } + +private: + std::unique_ptr mCollectConfig; +}; + +} // namespace logtail diff --git a/core/host_monitor/collector/BaseCollector.h b/core/host_monitor/collector/BaseCollector.h old mode 100644 new mode 100755 diff --git a/core/host_monitor/collector/CollectorManager.cpp b/core/host_monitor/collector/CollectorManager.cpp deleted file mode 100644 index 9a9c865b85..0000000000 --- a/core/host_monitor/collector/CollectorManager.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2024 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "CollectorManager.h" - -#include "BaseCollector.h" -#include "MockCollector.h" -#include "host_monitor/collector/ProcessCollector.h" - -namespace logtail { - -CollectorManager::CollectorManager() { - RegisterCollector(); - RegisterCollector(); -} - -std::shared_ptr CollectorManager::GetCollector(const std::string& collectorName) { - auto it = mCollectorMap.find(collectorName); - if (it == mCollectorMap.end()) { - return nullptr; - } - return it->second; -} - -template -void CollectorManager::RegisterCollector() { - auto collector = std::make_shared(); - mCollectorMap[collector->GetName()] = collector; -} - -} // namespace logtail diff --git a/core/host_monitor/collector/CollectorManager.h b/core/host_monitor/collector/CollectorManager.h deleted file mode 100644 index 2882bc5ef2..0000000000 --- a/core/host_monitor/collector/CollectorManager.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2024 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include "host_monitor/collector/BaseCollector.h" -namespace logtail { - -class CollectorManager { -public: - static CollectorManager* GetInstance() { - static CollectorManager sInstance; - return &sInstance; - } - - std::shared_ptr GetCollector(const std::string& collectorName); - -private: - CollectorManager(); - ~CollectorManager() = default; - - template - void RegisterCollector(); - - std::unordered_map> mCollectorMap; -}; - -} // namespace logtail diff --git a/core/host_monitor/collector/MockCollector.cpp b/core/host_monitor/collector/MockCollector.cpp deleted file mode 100644 index c9046b5d43..0000000000 --- a/core/host_monitor/collector/MockCollector.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2024 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "MockCollector.h" - -#include - -#include "PipelineEventGroup.h" -#include "constants/EntityConstants.h" - -namespace logtail { - -void MockCollector::Collect(PipelineEventGroup& group) { - auto event = group.AddLogEvent(); - event->SetTimestamp(time(nullptr)); - std::string content = "mock content"; - event->SetContent("mock", content); -} - -} // namespace logtail diff --git a/core/host_monitor/collector/MockCollector.h b/core/host_monitor/collector/MockCollector.h deleted file mode 100644 index 48f8292ae8..0000000000 --- a/core/host_monitor/collector/MockCollector.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2024 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include "constants/EntityConstants.h" -#include "host_monitor/collector/BaseCollector.h" - -namespace logtail { - -class MockCollector : public BaseCollector { -public: - MockCollector() { mName = "mock"; }; - ~MockCollector() override = default; - - void Collect(PipelineEventGroup& group) override; - -private: -}; - -} // namespace logtail diff --git a/core/host_monitor/collector/ProcessCollector.cpp b/core/host_monitor/collector/ProcessCollector.cpp old mode 100644 new mode 100755 index 5479f7909b..c999c8444c --- a/core/host_monitor/collector/ProcessCollector.cpp +++ b/core/host_monitor/collector/ProcessCollector.cpp @@ -42,9 +42,7 @@ namespace logtail { const size_t ProcessTopN = 20; void ProcessCollector::Collect(PipelineEventGroup& group) { - std::lock_guard lock(mCollectLock); - - group.SetMetadata(EventGroupMetaKey::HOST_MONITOR_COLLECT_TIME, std::to_string(time(nullptr))); + group.SetMetadata(EventGroupMetaKey::COLLECT_TIME, std::to_string(time(nullptr))); std::vector processes; SortProcessByCpu(processes, ProcessTopN); for (auto process : processes) { @@ -205,13 +203,15 @@ ProcessStatPtr ProcessCollector::ParseProcessStat(pid_t pid, std::string& line) return ptr; } -bool ProcessCollector::WalkAllProcess(const bfs::path& root, const std::function& callback) { - if (!bfs::exists(root) || !bfs::is_directory(root)) { +bool ProcessCollector::WalkAllProcess(const std::filesystem::path& root, + const std::function& callback) { + if (!std::filesystem::exists(root) || !std::filesystem::is_directory(root)) { LOG_ERROR(sLogger, ("ProcessCollector", "root path is not a directory or not exist")("root", root)); return false; } - for (const auto& dirEntry : bfs::directory_iterator{root, bfs::directory_options::skip_permission_denied}) { + for (const auto& dirEntry : + std::filesystem::directory_iterator{root, std::filesystem::directory_options::skip_permission_denied}) { std::string filename = dirEntry.path().filename().string(); if (IsInt(filename)) { callback(filename); diff --git a/core/host_monitor/collector/ProcessCollector.h b/core/host_monitor/collector/ProcessCollector.h old mode 100644 new mode 100755 index 836f106812..a7fa8c5aa4 --- a/core/host_monitor/collector/ProcessCollector.h +++ b/core/host_monitor/collector/ProcessCollector.h @@ -22,23 +22,19 @@ #include #include #include +#include #include -#include -#include #include #include #include "Flags.h" #include "Logger.h" -#include "MachineInfoUtil.h" -#include "StringTools.h" #include "constants/EntityConstants.h" #include "host_monitor/Constants.h" #include "host_monitor/collector/BaseCollector.h" DECLARE_FLAG_INT32(process_collect_silent_count); -namespace bfs = boost::filesystem; using namespace std::chrono; namespace logtail { @@ -167,11 +163,10 @@ class ProcessCollector : public BaseCollector { ProcessStatPtr ReadProcessStat(pid_t pid); ProcessStatPtr ParseProcessStat(pid_t pid, std::string& line); - bool WalkAllProcess(const bfs::path& root, const std::function& callback); + bool WalkAllProcess(const std::filesystem::path& root, const std::function& callback); ProcessStatPtr GetPreProcessStat(pid_t pid) { return mPrevProcessStat[pid]; } - std::mutex mCollectLock; steady_clock::time_point mProcessSortTime; std::vector mSortProcessStats; std::unordered_map mPrevProcessStat; diff --git a/core/models/PipelineEventGroup.h b/core/models/PipelineEventGroup.h index be4508827e..885f343ba9 100644 --- a/core/models/PipelineEventGroup.h +++ b/core/models/PipelineEventGroup.h @@ -58,7 +58,7 @@ enum class EventGroupMetaKey { PROMETHEUS_SCRAPE_TIMESTAMP_MILLISEC, PROMETHEUS_UP_STATE, - HOST_MONITOR_COLLECT_TIME, + COLLECT_TIME, SOURCE_ID }; diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 183262da9c..be6ce21c1b 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -199,6 +199,7 @@ void PipelineManager::StopAllPipelines() { LogtailPlugin::GetInstance()->StopAllPipelines(true); + Timer::GetInstance()->Stop(); ProcessorRunner::GetInstance()->Stop(); FlushAllBatch(); diff --git a/core/plugin/input/InputHostMeta.cpp b/core/plugin/input/InputHostMeta.cpp index b8dd6c93fd..c62316117e 100644 --- a/core/plugin/input/InputHostMeta.cpp +++ b/core/plugin/input/InputHostMeta.cpp @@ -38,13 +38,15 @@ bool InputHostMeta::Start() { LOG_INFO(sLogger, ("input host meta start", mContext->GetConfigName())); HostMonitorInputRunner::GetInstance()->Init(); HostMonitorInputRunner::GetInstance()->UpdateCollector( - mContext->GetConfigName(), {"process"}, mContext->GetProcessQueueKey()); + mContext->GetConfigName(), {"process"}, mContext->GetProcessQueueKey(), mIndex); return true; } bool InputHostMeta::Stop(bool isPipelineRemoving) { LOG_INFO(sLogger, ("input host meta stop", mContext->GetConfigName())); - HostMonitorInputRunner::GetInstance()->RemoveCollector(mContext->GetConfigName()); + if (isPipelineRemoving) { + HostMonitorInputRunner::GetInstance()->RemoveCollector(mContext->GetConfigName()); + } return true; } diff --git a/core/plugin/input/InputHostMeta.h b/core/plugin/input/InputHostMeta.h index 94587705ba..b31a901599 100644 --- a/core/plugin/input/InputHostMeta.h +++ b/core/plugin/input/InputHostMeta.h @@ -16,7 +16,6 @@ #pragma once -#include "json/value.h" #include "pipeline/plugin/interface/Input.h" namespace logtail { @@ -29,15 +28,11 @@ class InputHostMeta : public Input { bool Init(const Json::Value& config, Json::Value& optionalGoPipeline) override; bool Start() override; bool Stop(bool isPipelineRemoving) override; - bool SupportAck() const override { return false; } + bool SupportAck() const override { return true; } private: bool CreateInnerProcessors(const Json::Value& config); - std::string mDomain; - std::string mEntityType; - std::string mHostEntityID; - #ifdef APSARA_UNIT_TEST_MAIN friend class InputHostMetaUnittest; #endif diff --git a/core/plugin/processor/inner/ProcessorHostMetaNative.cpp b/core/plugin/processor/inner/ProcessorHostMetaNative.cpp index 1b8b845e40..c64016a4d0 100644 --- a/core/plugin/processor/inner/ProcessorHostMetaNative.cpp +++ b/core/plugin/processor/inner/ProcessorHostMetaNative.cpp @@ -39,9 +39,8 @@ bool ProcessorHostMetaNative::Init(const Json::Value& config) { if (hostType == DEFAULT_ENV_VALUE_ECS) { oss << DEFAULT_CONTENT_VALUE_DOMAIN_ACS << "." << DEFAULT_ENV_VALUE_ECS << "." << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; - ECSMeta ecsMeta = FetchECSMeta(); mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_ACS; - mHostEntityID = ecsMeta.instanceID; + mHostEntityID = FetchHostId(); } else { oss << DEFAULT_CONTENT_VALUE_DOMAIN_INFRA << "." << DEFAULT_ENV_VALUE_HOST << "." << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; @@ -86,8 +85,7 @@ void ProcessorHostMetaNative::ProcessEvent(PipelineEventGroup& group, targetEvent->SetContent(DEFAULT_CONTENT_KEY_DOMAIN, mDomain); targetEvent->SetContent(DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME, sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME)); - targetEvent->SetContent(DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME, - group.GetMetadata(EventGroupMetaKey::HOST_MONITOR_COLLECT_TIME)); + targetEvent->SetContent(DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME, group.GetMetadata(EventGroupMetaKey::COLLECT_TIME)); targetEvent->SetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS, "30"); // TODO: support delete event targetEvent->SetContent(DEFAULT_CONTENT_KEY_METHOD, DEFAULT_CONTENT_VALUE_METHOD_UPDATE); diff --git a/core/unittest/host_monitor/CMakeLists.txt b/core/unittest/host_monitor/CMakeLists.txt index 7a412f8e81..fd761c8c74 100644 --- a/core/unittest/host_monitor/CMakeLists.txt +++ b/core/unittest/host_monitor/CMakeLists.txt @@ -15,9 +15,6 @@ cmake_minimum_required(VERSION 3.22) project(host_monitor_unittest) -add_executable(collector_manager_unittest CollectorManagerUnittest.cpp) -target_link_libraries(collector_manager_unittest ${UT_BASE_TARGET}) - add_executable(process_collector_unittest ProcessCollectorUnittest.cpp) target_link_libraries(process_collector_unittest ${UT_BASE_TARGET}) @@ -32,7 +29,6 @@ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/1/ DESTINATION ${CMAKE_CURRENT_BINARY_DIR} file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/stat DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) include(GoogleTest) -gtest_discover_tests(collector_manager_unittest) gtest_discover_tests(process_collector_unittest) gtest_discover_tests(host_monitor_input_runner_unittest) gtest_discover_tests(system_information_tools_unittest) diff --git a/core/unittest/host_monitor/CollectorManagerUnittest.cpp b/core/unittest/host_monitor/CollectorManagerUnittest.cpp deleted file mode 100644 index 595be90ded..0000000000 --- a/core/unittest/host_monitor/CollectorManagerUnittest.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "CollectorManager.h" -#include "unittest/Unittest.h" - -using namespace std; - -namespace logtail { - -class CollectorManagerUnittest : public testing::Test { -public: - void TestGetCollector() const; -}; - -void CollectorManagerUnittest::TestGetCollector() const { - { - auto collector1 = CollectorManager::GetInstance()->GetCollector("mock"); - APSARA_TEST_NOT_EQUAL_FATAL(nullptr, collector1); - APSARA_TEST_EQUAL_FATAL("mock", collector1->GetName()); - auto collector2 = CollectorManager::GetInstance()->GetCollector("mock"); - APSARA_TEST_EQUAL_FATAL(collector1, collector2); - } -} - - -UNIT_TEST_CASE(CollectorManagerUnittest, TestGetCollector); - -} // namespace logtail - -UNIT_TEST_MAIN diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp index 76946515fd..b2345bc113 100644 --- a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include "HostMonitorInputRunner.h" -#include "Logger.h" +#include "HostMonitorTimerEvent.h" #include "ProcessQueueItem.h" #include "ProcessQueueManager.h" #include "QueueKey.h" #include "QueueKeyManager.h" +#include "common/timer/Timer.h" #include "unittest/Unittest.h" using namespace std; @@ -35,7 +37,7 @@ class HostMonitorInputRunnerUnittest : public testing::Test { void HostMonitorInputRunnerUnittest::TestUpdateAndRemoveCollector() const { auto runner = HostMonitorInputRunner::GetInstance(); runner->Init(); - runner->UpdateCollector("test", {"mock"}, QueueKey{}); + runner->UpdateCollector("test", {"mock"}, QueueKey{}, 0); APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid("test", "mock")); APSARA_TEST_TRUE_FATAL(runner->HasRegisteredPlugins()); runner->RemoveCollector("test"); @@ -46,17 +48,17 @@ void HostMonitorInputRunnerUnittest::TestUpdateAndRemoveCollector() const { void HostMonitorInputRunnerUnittest::TestScheduleOnce() const { auto runner = HostMonitorInputRunner::GetInstance(); - runner->mTimer = std::make_shared(); runner->Init(); - runner->mThreadPool->Start(); + runner->mThreadPool.Start(); std::string configName = "test"; auto queueKey = QueueKeyManager::GetInstance()->GetKey(configName); auto ctx = PipelineContext(); ctx.SetConfigName(configName); ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(queueKey, 0, ctx); - HostMonitorTimerEvent event(std::chrono::steady_clock::now(), 15, configName, "mock", queueKey); - runner->ScheduleOnce(&event); + auto collectConfig = std::make_unique( + configName, "process", queueKey, 0, std::chrono::seconds(1)); + runner->ScheduleOnce(std::move(collectConfig)); std::this_thread::sleep_for(std::chrono::seconds(1)); auto item = std::unique_ptr(new ProcessQueueItem(std::make_shared(), 0)); ProcessQueueManager::GetInstance()->EnablePop(configName); @@ -65,8 +67,8 @@ void HostMonitorInputRunnerUnittest::TestScheduleOnce() const { APSARA_TEST_TRUE_FATAL(item->mEventGroup.GetEvents().size() == 1); // verify schdule next - APSARA_TEST_EQUAL_FATAL(runner->mTimer->mQueue.size(), 1); - runner->mThreadPool->Stop(); + APSARA_TEST_EQUAL_FATAL(Timer::GetInstance()->mQueue.size(), 1); + runner->mThreadPool.Stop(); runner->Stop(); } diff --git a/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp b/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp index cc60ecda1b..1d00798a32 100644 --- a/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp @@ -68,7 +68,7 @@ void ProcessorHostMetaNativeUnittest::TestProcess() { Json::Value config; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::HOST_MONITOR_COLLECT_TIME, "123456"); + eventGroup.SetMetadataNoCopy(EventGroupMetaKey::COLLECT_TIME, "123456"); auto event = eventGroup.AddLogEvent(); event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PID, "123"); event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME, "123456"); From 5435d0316702794363a2e925110c31e6548ca4c8 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 27 Dec 2024 00:34:11 +0800 Subject: [PATCH 03/13] fix --- core/common/MachineInfoUtil.cpp | 15 +-- core/common/ThreadPool.h | 6 +- core/constants/EntityConstants.cpp | 20 +-- core/host_monitor/HostMonitorInputRunner.cpp | 41 ++++--- core/host_monitor/HostMonitorInputRunner.h | 15 ++- core/host_monitor/HostMonitorTimerEvent.cpp | 6 +- core/host_monitor/HostMonitorTimerEvent.h | 8 +- core/host_monitor/collector/BaseCollector.h | 5 +- ...llector.cpp => ProcessEntityCollector.cpp} | 78 ++++++++++-- ...ssCollector.h => ProcessEntityCollector.h} | 30 +++-- core/pipeline/plugin/PluginRegistry.cpp | 2 - core/plugin/input/InputHostMeta.cpp | 23 +--- core/plugin/input/InputHostMeta.h | 3 - .../inner/ProcessorHostMetaNative.cpp | 115 ------------------ .../processor/inner/ProcessorHostMetaNative.h | 48 -------- core/prometheus/PrometheusInputRunner.cpp | 3 +- core/prometheus/schedulers/BaseScheduler.cpp | 4 +- core/prometheus/schedulers/BaseScheduler.h | 4 +- .../schedulers/TargetSubscriberScheduler.cpp | 5 +- .../HostMonitorInputRunnerUnittest.cpp | 5 +- .../host_monitor/ProcessCollectorUnittest.cpp | 4 +- core/unittest/input/CMakeLists.txt | 6 +- .../prometheus/ScrapeSchedulerUnittest.cpp | 18 ++- 23 files changed, 161 insertions(+), 303 deletions(-) mode change 100644 => 100755 core/host_monitor/HostMonitorTimerEvent.cpp rename core/host_monitor/collector/{ProcessCollector.cpp => ProcessEntityCollector.cpp} (68%) rename core/host_monitor/collector/{ProcessCollector.h => ProcessEntityCollector.h} (86%) delete mode 100644 core/plugin/processor/inner/ProcessorHostMetaNative.cpp delete mode 100644 core/plugin/processor/inner/ProcessorHostMetaNative.h diff --git a/core/common/MachineInfoUtil.cpp b/core/common/MachineInfoUtil.cpp index d545802427..3e1b67558e 100644 --- a/core/common/MachineInfoUtil.cpp +++ b/core/common/MachineInfoUtil.cpp @@ -530,12 +530,9 @@ std::string RandomHostid() { return hostId; } -const std::string& GetLocalHostId() { +const std::string GetLocalHostId() { static std::string fileName = AppConfig::GetInstance()->GetLoongcollectorConfDir() + PATH_SEPARATOR + "host_id"; - static std::string hostId; - if (!hostId.empty()) { - return hostId; - } + std::string hostId; if (CheckExistance(fileName)) { if (!ReadFileContent(fileName, hostId)) { hostId = ""; @@ -571,10 +568,6 @@ std::string FetchHostId() { if (!hostId.empty()) { return hostId; } - hostId = STRING_FLAG(agent_host_id); - if (!hostId.empty()) { - return hostId; - } ECSMeta meta = FetchECSMeta(); hostId = meta.instanceID; if (!hostId.empty()) { @@ -584,6 +577,10 @@ std::string FetchHostId() { if (!hostId.empty()) { return hostId; } + hostId = STRING_FLAG(agent_host_id); + if (!hostId.empty()) { + return hostId; + } hostId = GetLocalHostId(); return hostId; diff --git a/core/common/ThreadPool.h b/core/common/ThreadPool.h index 09ccd733ee..eaacae9d54 100644 --- a/core/common/ThreadPool.h +++ b/core/common/ThreadPool.h @@ -15,13 +15,13 @@ */ #pragma once -#include -#include #include #include #include -#include +#include #include +#include +#include namespace logtail { diff --git a/core/constants/EntityConstants.cpp b/core/constants/EntityConstants.cpp index 17df9c2b5c..95db9051aa 100644 --- a/core/constants/EntityConstants.cpp +++ b/core/constants/EntityConstants.cpp @@ -35,14 +35,14 @@ const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE = "expire"; // for process entity const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS = "process"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_PID = "process_pid"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID = "process_ppid"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_USER = "process_user"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM = "process_comm"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME = "process_create_time"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD = "process_cwd"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY = "process_binary"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS = "process_arguments"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE = "process_language"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID = "process_container_id"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_PID = "pid"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID = "ppid"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_USER = "user"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM = "comm"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME = "create_time"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD = "cwd"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY = "binary"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS = "arguments"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE = "language"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID = "container_id"; } // namespace logtail diff --git a/core/host_monitor/HostMonitorInputRunner.cpp b/core/host_monitor/HostMonitorInputRunner.cpp index 926be0151f..348ab13d94 100755 --- a/core/host_monitor/HostMonitorInputRunner.cpp +++ b/core/host_monitor/HostMonitorInputRunner.cpp @@ -25,9 +25,10 @@ #include #include "HostMonitorTimerEvent.h" +#include "ProcessEntityCollector.h" #include "common/Lock.h" #include "common/timer/Timer.h" -#include "host_monitor/collector/ProcessCollector.h" +#include "host_monitor/collector/ProcessEntityCollector.h" #include "logger/Logger.h" #include "runner/ProcessorRunner.h" @@ -35,7 +36,7 @@ namespace logtail { HostMonitorInputRunner::HostMonitorInputRunner() : mThreadPool(ThreadPool(3)) { - RegisterCollector(); + RegisterCollector(); } void HostMonitorInputRunner::UpdateCollector(const std::string& configName, @@ -53,11 +54,11 @@ void HostMonitorInputRunner::UpdateCollector(const std::string& configName, } for (const auto& collectorName : newCollectors) { LOG_INFO(sLogger, ("add new host monitor collector", configName)("collector", collectorName)); - auto collectConfig = std::make_unique( + HostMonitorTimerEvent::CollectConfig collectConfig( configName, collectorName, processQueueKey, inputIndex, std::chrono::seconds(DEFAULT_SCHEDULE_INTERVAL)); // only push event when the collector is new added if (std::find(oldCollectors.begin(), oldCollectors.end(), collectorName) == oldCollectors.end()) { - Timer::GetInstance()->PushEvent(BuildTimerEvent(std::move(collectConfig))); + Timer::GetInstance()->PushEvent(BuildTimerEvent(collectConfig)); } } } @@ -109,35 +110,39 @@ bool HostMonitorInputRunner::IsCollectTaskValid(const std::string& configName, c return false; } -void HostMonitorInputRunner::ScheduleOnce(std::unique_ptr collectConfig) { - mThreadPool.Add([this, &collectConfig]() { +void HostMonitorInputRunner::ScheduleOnce(HostMonitorTimerEvent::CollectConfig& config) { + auto collectFn = [this, config]() mutable { PipelineEventGroup group(std::make_shared()); - auto collector = GetCollector(collectConfig->mCollectorName); + auto collector = GetCollector(config.mCollectorName); if (!collector) { collector->Collect(group); + } else { + LOG_ERROR(sLogger, + ("collector not found, will not collect", + "discard data")("config", config.mConfigName)("collector", config.mCollectorName)); + return; } bool result = ProcessorRunner::GetInstance()->PushQueue( - collectConfig->mProcessQueueKey, collectConfig->mInputIndex, std::move(group), 3); + config.mProcessQueueKey, config.mInputIndex, std::move(group), 3); if (!result) { LOG_WARNING(sLogger, - ("push queue failed", "discard data")("config", collectConfig->mConfigName)( - "collector", collectConfig->mCollectorName)); + ("push queue failed", "discard data")("config", config.mConfigName)("collector", + config.mCollectorName)); } LOG_DEBUG(sLogger, - ("schedule host monitor collector again", collectConfig->mConfigName)("collector", - collectConfig->mCollectorName)); - - auto event = BuildTimerEvent(std::move(collectConfig)); + ("schedule host monitor collector again", config.mConfigName)("collector", config.mCollectorName)); + auto event = BuildTimerEvent(config); event->ResetForNextExec(); Timer::GetInstance()->PushEvent(std::move(event)); - }); + }; + mThreadPool.Add(collectFn); } std::unique_ptr -HostMonitorInputRunner::BuildTimerEvent(std::unique_ptr collectConfig) { +HostMonitorInputRunner::BuildTimerEvent(HostMonitorTimerEvent::CollectConfig& collectConfig) { auto now = std::chrono::steady_clock::now(); - auto event = std::make_unique(now, std::move(collectConfig)); + auto event = std::make_unique(now, collectConfig); return event; } @@ -152,7 +157,7 @@ std::shared_ptr HostMonitorInputRunner::GetCollector(const std::s template void HostMonitorInputRunner::RegisterCollector() { auto collector = std::make_shared(); - mCollectorInstanceMap[collector->GetName()] = collector; + mCollectorInstanceMap[collector->Name()] = collector; } } // namespace logtail diff --git a/core/host_monitor/HostMonitorInputRunner.h b/core/host_monitor/HostMonitorInputRunner.h index f83ffc3af9..c8cfa94609 100644 --- a/core/host_monitor/HostMonitorInputRunner.h +++ b/core/host_monitor/HostMonitorInputRunner.h @@ -23,12 +23,12 @@ #include #include -#include "BaseCollector.h" -#include "InputRunner.h" -#include "Lock.h" -#include "QueueKey.h" -#include "ThreadPool.h" +#include "common/ThreadPool.h" #include "host_monitor/HostMonitorTimerEvent.h" +#include "host_monitor/collector/BaseCollector.h" +#include "host_monitor/collector/ProcessEntityCollector.h" +#include "pipeline/queue/QueueKey.h" +#include "runner/InputRunner.h" namespace logtail { @@ -57,12 +57,11 @@ class HostMonitorInputRunner : public InputRunner { bool HasRegisteredPlugins() const override; bool IsCollectTaskValid(const std::string& configName, const std::string& collectorName) const; - void ScheduleOnce(std::unique_ptr collectConfig); + void ScheduleOnce(HostMonitorTimerEvent::CollectConfig& collectConfig); private: HostMonitorInputRunner(); - std::unique_ptr - BuildTimerEvent(std::unique_ptr collectConfig); + std::unique_ptr BuildTimerEvent(HostMonitorTimerEvent::CollectConfig& collectConfig); template void RegisterCollector(); diff --git a/core/host_monitor/HostMonitorTimerEvent.cpp b/core/host_monitor/HostMonitorTimerEvent.cpp old mode 100644 new mode 100755 index 3fb5079309..32c5dae81a --- a/core/host_monitor/HostMonitorTimerEvent.cpp +++ b/core/host_monitor/HostMonitorTimerEvent.cpp @@ -23,12 +23,12 @@ namespace logtail { bool HostMonitorTimerEvent::IsValid() const { - return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(mCollectConfig->mConfigName, - mCollectConfig->mCollectorName); + return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(mCollectConfig.mConfigName, + mCollectConfig.mCollectorName); } bool HostMonitorTimerEvent::Execute() { - HostMonitorInputRunner::GetInstance()->ScheduleOnce(std::move(mCollectConfig)); + HostMonitorInputRunner::GetInstance()->ScheduleOnce(mCollectConfig); return true; } diff --git a/core/host_monitor/HostMonitorTimerEvent.h b/core/host_monitor/HostMonitorTimerEvent.h index 07a3dceb40..1279982f79 100644 --- a/core/host_monitor/HostMonitorTimerEvent.h +++ b/core/host_monitor/HostMonitorTimerEvent.h @@ -48,15 +48,15 @@ class HostMonitorTimerEvent : public TimerEvent { std::chrono::seconds mInterval; }; - HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime, std::unique_ptr collectConfig) - : TimerEvent(execTime), mCollectConfig(std::move(collectConfig)) {} + HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime, CollectConfig collectConfig) + : TimerEvent(execTime), mCollectConfig(collectConfig) {} bool IsValid() const override; bool Execute() override; - void ResetForNextExec() { SetExecTime(GetExecTime() + mCollectConfig->mInterval); } + void ResetForNextExec() { SetExecTime(GetExecTime() + mCollectConfig.mInterval); } private: - std::unique_ptr mCollectConfig; + CollectConfig mCollectConfig; }; } // namespace logtail diff --git a/core/host_monitor/collector/BaseCollector.h b/core/host_monitor/collector/BaseCollector.h index 73115e1a53..8db6bf8003 100755 --- a/core/host_monitor/collector/BaseCollector.h +++ b/core/host_monitor/collector/BaseCollector.h @@ -27,11 +27,10 @@ class BaseCollector { bool IsValid() const { return mValidState; } virtual void Collect(PipelineEventGroup& group) = 0; - - const std::string GetName() const { return mName; } + virtual const std::string& Name() const = 0; protected: - std::string mName; + const std::string mName; bool mValidState = false; }; diff --git a/core/host_monitor/collector/ProcessCollector.cpp b/core/host_monitor/collector/ProcessEntityCollector.cpp similarity index 68% rename from core/host_monitor/collector/ProcessCollector.cpp rename to core/host_monitor/collector/ProcessEntityCollector.cpp index c999c8444c..70682c83c9 100755 --- a/core/host_monitor/collector/ProcessCollector.cpp +++ b/core/host_monitor/collector/ProcessEntityCollector.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "ProcessCollector.h" +#include "ProcessEntityCollector.h" #include #include @@ -24,12 +24,12 @@ #include #include #include -#include #include #include #include #include +#include "Common.h" #include "FileSystemUtil.h" #include "Logger.h" #include "PipelineEventGroup.h" @@ -41,23 +41,67 @@ namespace logtail { const size_t ProcessTopN = 20; -void ProcessCollector::Collect(PipelineEventGroup& group) { +const std::string ProcessEntityCollector::sName = "process_entity"; + +ProcessEntityCollector::ProcessEntityCollector() : mProcessSilentCount(INT32_FLAG(process_collect_silent_count)) { + // try to read process dir + if (access(PROCESS_DIR.c_str(), R_OK) != 0) { + LOG_ERROR(sLogger, ("process collector init failed", "process dir not exist or ")("dir", PROCESS_DIR)); + mValidState = false; + } else { + mValidState = true; + } + auto hostType = ToString(getenv(DEFAULT_ENV_KEY_HOST_TYPE.c_str())); + std::ostringstream oss; + if (hostType == DEFAULT_ENV_VALUE_ECS) { + oss << DEFAULT_CONTENT_VALUE_DOMAIN_ACS << "." << DEFAULT_ENV_VALUE_ECS << "." + << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; + mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_ACS; + } else { + oss << DEFAULT_CONTENT_VALUE_DOMAIN_INFRA << "." << DEFAULT_ENV_VALUE_HOST << "." + << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; + mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_INFRA; + } + mHostEntityID = FetchHostId(); + mEntityType = oss.str(); +}; + +void ProcessEntityCollector::Collect(PipelineEventGroup& group) { group.SetMetadata(EventGroupMetaKey::COLLECT_TIME, std::to_string(time(nullptr))); std::vector processes; - SortProcessByCpu(processes, ProcessTopN); + GetSortedProcess(processes, ProcessTopN); for (auto process : processes) { auto event = group.AddLogEvent(); time_t logtime = time(nullptr); event->SetTimestamp(logtime); + std::string processCreateTime + = std::to_string(duration_cast(process->startTime.time_since_epoch()).count()); + + // common fields + event->SetContent(DEFAULT_CONTENT_KEY_ENTITY_TYPE, mEntityType); + event->SetContent(DEFAULT_CONTENT_KEY_ENTITY_ID, + GetProcessEntityID(std::to_string(process->pid), processCreateTime)); + event->SetContent(DEFAULT_CONTENT_KEY_DOMAIN, mDomain); + event->SetContent(DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME, processCreateTime); + event->SetContent(DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME, std::to_string(logtime)); + event->SetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS, "30"); + + // custom fields event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PID, std::to_string(process->pid)); event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PPID, std::to_string(process->parentPid)); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME, - std::to_string(duration_cast(process->startTime.time_since_epoch()).count())); + // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_USER, ""); TODO: get user name + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_COMM, process->name); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME, processCreateTime); + // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CWD, ""); TODO: get cwd + // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_BINARY, ""); TODO: get binary + // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS, ""); TODO: get arguments + // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE, ""); TODO: get language + // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID, ""); TODO: get container id } } -void ProcessCollector::SortProcessByCpu(std::vector& processStats, size_t topN) { +void ProcessEntityCollector::GetSortedProcess(std::vector& processStats, size_t topN) { steady_clock::time_point now = steady_clock::now(); auto compare = [](const std::pair& a, const std::pair& b) { return a.second < b.second; @@ -103,7 +147,7 @@ void ProcessCollector::SortProcessByCpu(std::vector& processStat mSortProcessStats = processStats; } -ProcessStatPtr ProcessCollector::GetProcessStat(pid_t pid, bool& isFirstCollect) { +ProcessStatPtr ProcessEntityCollector::GetProcessStat(pid_t pid, bool& isFirstCollect) { const auto now = steady_clock::now(); // TODO: more accurate cache @@ -140,7 +184,7 @@ ProcessStatPtr ProcessCollector::GetProcessStat(pid_t pid, bool& isFirstCollect) return ptr; } -ProcessStatPtr ProcessCollector::ReadProcessStat(pid_t pid) { +ProcessStatPtr ProcessEntityCollector::ReadProcessStat(pid_t pid) { LOG_DEBUG(sLogger, ("read process stat", pid)); auto processStat = PROCESS_DIR / std::to_string(pid) / PROCESS_STAT; @@ -157,7 +201,7 @@ ProcessStatPtr ProcessCollector::ReadProcessStat(pid_t pid) { // 1 (cat) R 0 1 1 34816 1 4194560 1110 0 0 0 1 1 0 0 20 0 1 0 18938584 4505600 171 18446744073709551615 4194304 4238788 // 140727020025920 0 0 0 0 0 0 0 0 0 17 3 0 0 0 0 0 6336016 6337300 21442560 140727020027760 140727020027777 // 140727020027777 140727020027887 0 -ProcessStatPtr ProcessCollector::ParseProcessStat(pid_t pid, std::string& line) { +ProcessStatPtr ProcessEntityCollector::ParseProcessStat(pid_t pid, std::string& line) { ProcessStatPtr ptr = std::make_shared(); ptr->pid = pid; auto nameStartPos = line.find_first_of('('); @@ -203,10 +247,10 @@ ProcessStatPtr ProcessCollector::ParseProcessStat(pid_t pid, std::string& line) return ptr; } -bool ProcessCollector::WalkAllProcess(const std::filesystem::path& root, - const std::function& callback) { +bool ProcessEntityCollector::WalkAllProcess(const std::filesystem::path& root, + const std::function& callback) { if (!std::filesystem::exists(root) || !std::filesystem::is_directory(root)) { - LOG_ERROR(sLogger, ("ProcessCollector", "root path is not a directory or not exist")("root", root)); + LOG_ERROR(sLogger, ("ProcessEntityCollector", "root path is not a directory or not exist")("root", root)); return false; } @@ -220,4 +264,12 @@ bool ProcessCollector::WalkAllProcess(const std::filesystem::path& root, return true; } +const std::string ProcessEntityCollector::GetProcessEntityID(StringView pid, StringView createTime) { + std::ostringstream oss; + oss << mHostEntityID << pid << createTime; + auto bigID = sdk::CalcMD5(oss.str()); + std::transform(bigID.begin(), bigID.end(), bigID.begin(), ::tolower); + return bigID; +} + } // namespace logtail diff --git a/core/host_monitor/collector/ProcessCollector.h b/core/host_monitor/collector/ProcessEntityCollector.h similarity index 86% rename from core/host_monitor/collector/ProcessCollector.h rename to core/host_monitor/collector/ProcessEntityCollector.h index a7fa8c5aa4..3e84ab34b1 100755 --- a/core/host_monitor/collector/ProcessCollector.h +++ b/core/host_monitor/collector/ProcessEntityCollector.h @@ -29,6 +29,8 @@ #include "Flags.h" #include "Logger.h" +#include "MachineInfoUtil.h" +#include "StringTools.h" #include "constants/EntityConstants.h" #include "host_monitor/Constants.h" #include "host_monitor/collector/BaseCollector.h" @@ -39,7 +41,6 @@ using namespace std::chrono; namespace logtail { - struct ProcessCpuInfo { uint64_t user = 0; uint64_t sys = 0; @@ -140,30 +141,24 @@ constexpr int operator-(EnumProcessStat a, EnumProcessStat b) { return (int)a - (int)b; } -class ProcessCollector : public BaseCollector { +class ProcessEntityCollector : public BaseCollector { public: - ProcessCollector() : mProcessSilentCount(INT32_FLAG(process_collect_silent_count)) { - mName = "process"; - - // try to read process dir - if (access(PROCESS_DIR.c_str(), R_OK) != 0) { - LOG_ERROR(sLogger, ("process collector init failed", "process dir not exist or ")("dir", PROCESS_DIR)); - mValidState = false; - } else { - mValidState = true; - } - }; - ~ProcessCollector() override = default; + ProcessEntityCollector(); + ~ProcessEntityCollector() override = default; void Collect(PipelineEventGroup& group) override; + static const std::string sName; + const std::string& Name() const override { return sName; } + private: - void SortProcessByCpu(std::vector& processStats, size_t topN); + void GetSortedProcess(std::vector& processStats, size_t topN); ProcessStatPtr GetProcessStat(pid_t pid, bool& isFirstCollect); ProcessStatPtr ReadProcessStat(pid_t pid); ProcessStatPtr ParseProcessStat(pid_t pid, std::string& line); bool WalkAllProcess(const std::filesystem::path& root, const std::function& callback); + const std::string GetProcessEntityID(StringView pid, StringView createTime); ProcessStatPtr GetPreProcessStat(pid_t pid) { return mPrevProcessStat[pid]; } @@ -172,9 +167,12 @@ class ProcessCollector : public BaseCollector { std::unordered_map mPrevProcessStat; const int mProcessSilentCount; + std::string mHostEntityID; + std::string mEntityType; + std::string mDomain; #ifdef APSARA_UNIT_TEST_MAIN - friend class ProcessCollectorUnittest; + friend class ProcessEntityCollectorUnittest; #endif }; diff --git a/core/pipeline/plugin/PluginRegistry.cpp b/core/pipeline/plugin/PluginRegistry.cpp index ee6d314fc5..ba2f728233 100644 --- a/core/pipeline/plugin/PluginRegistry.cpp +++ b/core/pipeline/plugin/PluginRegistry.cpp @@ -32,7 +32,6 @@ #include "plugin/input/InputFile.h" #include "plugin/input/InputHostMeta.h" #include "plugin/input/InputPrometheus.h" -#include "plugin/processor/inner/ProcessorHostMetaNative.h" #if defined(__linux__) && !defined(__ANDROID__) #include "plugin/input/InputFileSecurity.h" #include "plugin/input/InputInternalMetrics.h" @@ -154,7 +153,6 @@ void PluginRegistry::LoadStaticPlugins() { RegisterProcessorCreator(new StaticProcessorCreator()); RegisterProcessorCreator(new StaticProcessorCreator()); RegisterProcessorCreator(new StaticProcessorCreator()); - RegisterProcessorCreator(new StaticProcessorCreator()); #if defined(__linux__) && !defined(__ANDROID__) && !defined(__EXCLUDE_SPL__) if (BOOL_FLAG(enable_processor_spl)) { RegisterProcessorCreator(new StaticProcessorCreator()); diff --git a/core/plugin/input/InputHostMeta.cpp b/core/plugin/input/InputHostMeta.cpp index c62316117e..f6e151d926 100644 --- a/core/plugin/input/InputHostMeta.cpp +++ b/core/plugin/input/InputHostMeta.cpp @@ -14,24 +14,17 @@ #include "InputHostMeta.h" -#include -#include - #include "HostMonitorInputRunner.h" #include "Logger.h" -#include "PluginRegistry.h" -#include "ProcessorInstance.h" #include "constants/EntityConstants.h" #include "json/value.h" -#include "pipeline/Pipeline.h" -#include "plugin/processor/inner/ProcessorHostMetaNative.h" namespace logtail { const std::string InputHostMeta::sName = "input_host_meta"; bool InputHostMeta::Init(const Json::Value& config, Json::Value& optionalGoPipeline) { - return CreateInnerProcessors(config); + return true; } bool InputHostMeta::Start() { @@ -50,18 +43,4 @@ bool InputHostMeta::Stop(bool isPipelineRemoving) { return true; } -bool InputHostMeta::CreateInnerProcessors(const Json::Value& config) { - std::unique_ptr processor; - { - processor = PluginRegistry::GetInstance()->CreateProcessor(ProcessorHostMetaNative::sName, - mContext->GetPipeline().GenNextPluginMeta(false)); - Json::Value detail; - if (!processor->Init(detail, *mContext)) { - return false; - } - mInnerProcessors.emplace_back(std::move(processor)); - } - return true; -} - } // namespace logtail diff --git a/core/plugin/input/InputHostMeta.h b/core/plugin/input/InputHostMeta.h index b31a901599..cdae8b6f2e 100644 --- a/core/plugin/input/InputHostMeta.h +++ b/core/plugin/input/InputHostMeta.h @@ -30,9 +30,6 @@ class InputHostMeta : public Input { bool Stop(bool isPipelineRemoving) override; bool SupportAck() const override { return true; } -private: - bool CreateInnerProcessors(const Json::Value& config); - #ifdef APSARA_UNIT_TEST_MAIN friend class InputHostMetaUnittest; #endif diff --git a/core/plugin/processor/inner/ProcessorHostMetaNative.cpp b/core/plugin/processor/inner/ProcessorHostMetaNative.cpp deleted file mode 100644 index c64016a4d0..0000000000 --- a/core/plugin/processor/inner/ProcessorHostMetaNative.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2024 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ProcessorHostMetaNative.h" - -#include -#include -#include -#include - -#include "Common.h" -#include "LogEvent.h" -#include "Logger.h" -#include "MachineInfoUtil.h" -#include "PipelineEventGroup.h" -#include "StringTools.h" -#include "constants/EntityConstants.h" - -namespace logtail { - -const std::string ProcessorHostMetaNative::sName = "processor_host_meta_native"; - -bool ProcessorHostMetaNative::Init(const Json::Value& config) { - auto hostType = ToString(getenv(DEFAULT_ENV_KEY_HOST_TYPE.c_str())); - std::ostringstream oss; - if (hostType == DEFAULT_ENV_VALUE_ECS) { - oss << DEFAULT_CONTENT_VALUE_DOMAIN_ACS << "." << DEFAULT_ENV_VALUE_ECS << "." - << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; - mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_ACS; - mHostEntityID = FetchHostId(); - } else { - oss << DEFAULT_CONTENT_VALUE_DOMAIN_INFRA << "." << DEFAULT_ENV_VALUE_HOST << "." - << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; - mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_INFRA; - mHostEntityID = GetHostIp(); - } - mEntityType = oss.str(); - return true; -} - -void ProcessorHostMetaNative::Process(PipelineEventGroup& group) { - EventsContainer& events = group.MutableEvents(); - EventsContainer newEvents; - - for (auto& event : events) { - ProcessEvent(group, std::move(event), newEvents); - } - events.swap(newEvents); -} - -bool ProcessorHostMetaNative::IsSupportedEvent(const PipelineEventPtr& event) const { - return event.Is(); -} - -void ProcessorHostMetaNative::ProcessEvent(PipelineEventGroup& group, - PipelineEventPtr&& e, - EventsContainer& newEvents) { - if (!IsSupportedEvent(e)) { - newEvents.emplace_back(std::move(e)); - return; - } - - auto& sourceEvent = e.Cast(); - std::unique_ptr targetEvent = group.CreateLogEvent(true); - targetEvent->SetTimestamp(sourceEvent.GetTimestamp()); - - // TODO: support host entity - targetEvent->SetContent(DEFAULT_CONTENT_KEY_ENTITY_TYPE, mEntityType); - targetEvent->SetContent(DEFAULT_CONTENT_KEY_ENTITY_ID, - GetProcessEntityID(sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_PID), - sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME))); - targetEvent->SetContent(DEFAULT_CONTENT_KEY_DOMAIN, mDomain); - targetEvent->SetContent(DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME, - sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME)); - targetEvent->SetContent(DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME, group.GetMetadata(EventGroupMetaKey::COLLECT_TIME)); - targetEvent->SetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS, "30"); - // TODO: support delete event - targetEvent->SetContent(DEFAULT_CONTENT_KEY_METHOD, DEFAULT_CONTENT_VALUE_METHOD_UPDATE); - - targetEvent->SetContent("pid", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_PID)); - targetEvent->SetContent("ppid", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_PPID)); - targetEvent->SetContent("user", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_USER)); - targetEvent->SetContent("comm", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_COMM)); - targetEvent->SetContent("create_time", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME)); - targetEvent->SetContent("cwd", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CWD)); - targetEvent->SetContent("binary", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_BINARY)); - targetEvent->SetContent("arguments", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS)); - targetEvent->SetContent("language", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE)); - targetEvent->SetContent("containerID", sourceEvent.GetContent(DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID)); - newEvents.emplace_back(std::move(targetEvent), true, nullptr); -} - - -const std::string ProcessorHostMetaNative::GetProcessEntityID(StringView pid, StringView createTime) { - std::ostringstream oss; - oss << mHostEntityID << pid << createTime; - auto bigID = sdk::CalcMD5(oss.str()); - std::transform(bigID.begin(), bigID.end(), bigID.begin(), ::tolower); - return bigID; -} - -} // namespace logtail diff --git a/core/plugin/processor/inner/ProcessorHostMetaNative.h b/core/plugin/processor/inner/ProcessorHostMetaNative.h deleted file mode 100644 index 3b32e20b06..0000000000 --- a/core/plugin/processor/inner/ProcessorHostMetaNative.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2024 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "PipelineEventPtr.h" -#include "Processor.h" - -namespace logtail { -class ProcessorHostMetaNative : public Processor { -public: - static const std::string sName; - - const std::string& Name() const override { return sName; } - bool Init(const Json::Value& config) override; - void Process(PipelineEventGroup& group) override; - -protected: - bool IsSupportedEvent(const PipelineEventPtr& event) const override; - -private: - void ProcessEvent(PipelineEventGroup& group, PipelineEventPtr&& e, EventsContainer& newEvents); - - const std::string GetProcessEntityID(StringView pid, StringView createTime); - - std::string mDomain; - std::string mEntityType; - std::string mHostEntityID; - -#ifdef APSARA_UNIT_TEST_MAIN - friend class ProcessorHostMetaNativeUnittest; -#endif -}; - -} // namespace logtail diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index ca49e35091..a4759c2583 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -50,7 +50,6 @@ PrometheusInputRunner::PrometheusInputRunner() mEventPool(true), mUnRegisterMs(0) { mClient = std::make_unique(); - mTimer = std::make_shared(); // self monitor MetricLabels labels; @@ -84,7 +83,7 @@ void PrometheusInputRunner::UpdateScrapeInput(std::shared_ptrInitSelfMonitor(defaultLabels); targetSubscriber->mUnRegisterMs = mUnRegisterMs.load(); - targetSubscriber->SetComponent(mTimer, &mEventPool); + targetSubscriber->SetComponent(&mEventPool); auto randSleepMilliSec = GetRandSleepMilliSec( targetSubscriber->GetId(), prometheus::RefeshIntervalSeconds, GetCurrentTimeInMilliSeconds()); auto firstExecTime = std::chrono::steady_clock::now() + std::chrono::milliseconds(randSleepMilliSec); diff --git a/core/prometheus/schedulers/BaseScheduler.cpp b/core/prometheus/schedulers/BaseScheduler.cpp index af564f1622..969d25aa6d 100644 --- a/core/prometheus/schedulers/BaseScheduler.cpp +++ b/core/prometheus/schedulers/BaseScheduler.cpp @@ -34,8 +34,8 @@ bool BaseScheduler::IsCancelled() { return !mValidState; } -void BaseScheduler::SetComponent(shared_ptr timer, EventPool* eventPool) { - mTimer = std::move(timer); +void BaseScheduler::SetComponent(EventPool* eventPool) { + mTimer = Timer::GetInstance(); mEventPool = eventPool; } } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/BaseScheduler.h b/core/prometheus/schedulers/BaseScheduler.h index 26739cdcd0..cc1740156e 100644 --- a/core/prometheus/schedulers/BaseScheduler.h +++ b/core/prometheus/schedulers/BaseScheduler.h @@ -24,7 +24,7 @@ class BaseScheduler { void DelayExecTime(uint64_t delaySeconds); virtual void Cancel(); - void SetComponent(std::shared_ptr timer, EventPool* eventPool); + void SetComponent(EventPool* eventPool); protected: bool IsCancelled(); @@ -39,7 +39,7 @@ class BaseScheduler { std::shared_ptr> mFuture; std::shared_ptr> mIsContextValidFuture; - std::shared_ptr mTimer; + Timer* mTimer; EventPool* mEventPool = nullptr; }; } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index 42381f1229..719cf50b97 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -223,7 +223,7 @@ TargetSubscriberScheduler::BuildScrapeSchedulerSet(std::vector& targetGr auto scrapeScheduler = std::make_shared(mScrapeConfigPtr, host, port, resultLabel, mQueueKey, mInputIndex); - scrapeScheduler->SetComponent(mTimer, mEventPool); + scrapeScheduler->SetComponent(mEventPool); auto randSleepMilliSec = GetRandSleepMilliSec( scrapeScheduler->GetId(), mScrapeConfigPtr->mScrapeIntervalSeconds, GetCurrentTimeInMilliSeconds()); @@ -333,7 +333,8 @@ void TargetSubscriberScheduler::InitSelfMonitor(const MetricLabels& defaultLabel mSelfMonitor = std::make_shared(); mSelfMonitor->InitMetricManager(sSubscriberMetricKeys, mDefaultLabels); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE, std::move(mDefaultLabels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE, std::move(mDefaultLabels)); mPromSubscriberTargets = mMetricsRecordRef.CreateIntGauge(METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS); mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_PLUGIN_TOTAL_DELAY_MS); } diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp index b2345bc113..f5b326830d 100644 --- a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -56,9 +56,8 @@ void HostMonitorInputRunnerUnittest::TestScheduleOnce() const { ctx.SetConfigName(configName); ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(queueKey, 0, ctx); - auto collectConfig = std::make_unique( - configName, "process", queueKey, 0, std::chrono::seconds(1)); - runner->ScheduleOnce(std::move(collectConfig)); + HostMonitorTimerEvent::CollectConfig collectConfig(configName, "process", queueKey, 0, std::chrono::seconds(1)); + runner->ScheduleOnce(collectConfig); std::this_thread::sleep_for(std::chrono::seconds(1)); auto item = std::unique_ptr(new ProcessQueueItem(std::make_shared(), 0)); ProcessQueueManager::GetInstance()->EnablePop(configName); diff --git a/core/unittest/host_monitor/ProcessCollectorUnittest.cpp b/core/unittest/host_monitor/ProcessCollectorUnittest.cpp index 8e4c8c327c..1c13432df0 100644 --- a/core/unittest/host_monitor/ProcessCollectorUnittest.cpp +++ b/core/unittest/host_monitor/ProcessCollectorUnittest.cpp @@ -39,8 +39,8 @@ void ProcessCollectorUnittest::TestSortProcessByCpu() const { PROCESS_DIR = "/proc"; auto collector = ProcessCollector(); auto processes = vector(); - collector.SortProcessByCpu(processes, 5); // fist time will be ignored - collector.SortProcessByCpu(processes, 5); + collector.GetSortedProcess(processes, 5); // fist time will be ignored + collector.GetSortedProcess(processes, 5); APSARA_TEST_EQUAL(5, processes.size()); auto prev = processes[0]; for (auto i = 1; i < processes.size(); i++) { diff --git a/core/unittest/input/CMakeLists.txt b/core/unittest/input/CMakeLists.txt index 3e18f939f5..728365a2d3 100644 --- a/core/unittest/input/CMakeLists.txt +++ b/core/unittest/input/CMakeLists.txt @@ -36,8 +36,8 @@ target_link_libraries(input_ebpf_network_security_unittest unittest_base) add_executable(input_ebpf_network_observer_unittest InputNetworkObserverUnittest.cpp) target_link_libraries(input_ebpf_network_observer_unittest unittest_base) -add_executable(input_host_meat_unittest InputHostMetaUnittest.cpp) -target_link_libraries(input_host_meat_unittest unittest_base) +add_executable(input_host_meta_unittest InputHostMetaUnittest.cpp) +target_link_libraries(input_host_meta_unittest unittest_base) include(GoogleTest) gtest_discover_tests(input_file_unittest) @@ -47,4 +47,4 @@ gtest_discover_tests(input_ebpf_file_security_unittest) gtest_discover_tests(input_ebpf_process_security_unittest) gtest_discover_tests(input_ebpf_network_security_unittest) gtest_discover_tests(input_ebpf_network_observer_unittest) -gtest_discover_tests(input_host_meat_unittest) +gtest_discover_tests(input_host_meta_unittest) diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index 2c5138768b..9bfbdd55bd 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -197,12 +197,11 @@ void ScrapeSchedulerUnittest::TestScheduler() { Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); ScrapeScheduler event(mScrapeConfig, "localhost", 8080, labels, 0, 0); - auto timer = make_shared(); EventPool eventPool{true}; - event.SetComponent(timer, &eventPool); + event.SetComponent(&eventPool); event.ScheduleNext(); - APSARA_TEST_TRUE(timer->mQueue.size() == 1); + APSARA_TEST_TRUE(Timer::GetInstance()->mQueue.size() == 1); event.Cancel(); @@ -216,22 +215,21 @@ void ScrapeSchedulerUnittest::TestQueueIsFull() { ScrapeScheduler event(mScrapeConfig, "localhost", 8080, labels, 0, 0); auto defaultLabels = MetricLabels(); event.InitSelfMonitor(defaultLabels); - auto timer = make_shared(); EventPool eventPool{true}; - event.SetComponent(timer, &eventPool); + event.SetComponent(&eventPool); auto now = std::chrono::steady_clock::now(); event.SetFirstExecTime(now); event.ScheduleNext(); - APSARA_TEST_TRUE(timer->mQueue.size() == 1); + APSARA_TEST_TRUE(Timer::GetInstance()->mQueue.size() == 1); - const auto& e = timer->mQueue.top(); + const auto& e = Timer::GetInstance()->mQueue.top(); APSARA_TEST_EQUAL(now, e->GetExecTime()); APSARA_TEST_FALSE(e->IsValid()); - timer->mQueue.pop(); + Timer::GetInstance()->mQueue.pop(); // queue is full, so it should schedule next after 1 second - APSARA_TEST_EQUAL(1UL, timer->mQueue.size()); - const auto& next = timer->mQueue.top(); + APSARA_TEST_EQUAL(1UL, Timer::GetInstance()->mQueue.size()); + const auto& next = Timer::GetInstance()->mQueue.top(); APSARA_TEST_EQUAL(now + std::chrono::seconds(1), next->GetExecTime()); } From f7b71ab302543e786101822d5f8768d371b0d3e9 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 27 Dec 2024 18:03:03 +0800 Subject: [PATCH 04/13] fix --- core/common/MachineInfoUtil.cpp | 89 ------------- core/common/MachineInfoUtil.h | 1 - core/constants/EntityConstants.cpp | 5 +- core/constants/EntityConstants.h | 4 +- core/host_monitor/HostMonitorInputRunner.cpp | 99 +++++++-------- core/host_monitor/HostMonitorInputRunner.h | 17 ++- core/host_monitor/HostMonitorTimerEvent.cpp | 8 +- core/host_monitor/HostMonitorTimerEvent.h | 13 +- core/host_monitor/SystemInformationTools.cpp | 2 +- .../collector/ProcessEntityCollector.cpp | 22 ++-- .../collector/ProcessEntityCollector.h | 6 +- core/plugin/input/InputHostMeta.cpp | 5 +- core/prometheus/PrometheusInputRunner.cpp | 2 +- core/unittest/host_monitor/CMakeLists.txt | 6 +- .../HostMonitorInputRunnerUnittest.cpp | 10 +- ...cpp => ProcessEntityCollectorUnittest.cpp} | 28 ++-- core/unittest/input/InputHostMetaUnittest.cpp | 3 +- core/unittest/processor/CMakeLists.txt | 4 - .../ProcessorHostMetaNativeUnittest.cpp | 120 ------------------ .../prometheus/ScrapeSchedulerUnittest.cpp | 3 +- 20 files changed, 114 insertions(+), 333 deletions(-) rename core/unittest/host_monitor/{ProcessCollectorUnittest.cpp => ProcessEntityCollectorUnittest.cpp} (60%) delete mode 100644 core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp diff --git a/core/common/MachineInfoUtil.cpp b/core/common/MachineInfoUtil.cpp index 3e1b67558e..450d7b611d 100644 --- a/core/common/MachineInfoUtil.cpp +++ b/core/common/MachineInfoUtil.cpp @@ -16,10 +16,6 @@ #include -#include - -#include "AppConfig.h" -#include "common/UUIDUtil.h" #if defined(__linux__) #include #include @@ -43,12 +39,10 @@ #include -#include "FileSystemUtil.h" #include "StringTools.h" #include "common/FileSystemUtil.h" #include "logger/Logger.h" -DEFINE_FLAG_STRING(agent_host_id, "", ""); #if defined(_MSC_VER) typedef LONG NTSTATUS, *PNTSTATUS; @@ -503,89 +497,6 @@ size_t FetchECSMetaCallback(char* buffer, size_t size, size_t nmemb, std::string return sizes; } -std::string GetSerialNumberFromEcsAssist(const std::string& machineIdFile) { - std::string sn; - if (CheckExistance(machineIdFile)) { - if (!ReadFileContent(machineIdFile, sn)) { - return ""; - } - } - return sn; -} - -static std::string GetEcsAssistMachineIdFile() { -#if defined(WIN32) - return "C:\\ProgramData\\aliyun\\assist\\hybrid\\machine-id"; -#else - return "/usr/local/share/aliyun-assist/hybrid/machine-id"; -#endif -} - -std::string GetSerialNumberFromEcsAssist() { - return GetSerialNumberFromEcsAssist(GetEcsAssistMachineIdFile()); -} - -std::string RandomHostid() { - static std::string hostId = CalculateRandomUUID(); - return hostId; -} - -const std::string GetLocalHostId() { - static std::string fileName = AppConfig::GetInstance()->GetLoongcollectorConfDir() + PATH_SEPARATOR + "host_id"; - std::string hostId; - if (CheckExistance(fileName)) { - if (!ReadFileContent(fileName, hostId)) { - hostId = ""; - } - } - if (hostId.empty()) { - hostId = RandomHostid(); - - LOG_INFO(sLogger, ("save hostId file to local file system, hostId", hostId)); - int fd = open(fileName.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0755); - if (fd == -1) { - int savedErrno = errno; - if (savedErrno != EEXIST) { - LOG_ERROR(sLogger, ("save hostId file fail", fileName)("errno", strerror(savedErrno))); - } - } else { - // 文件成功创建,现在写入hostId - ssize_t written = write(fd, hostId.c_str(), hostId.length()); - if (written == static_cast(hostId.length())) { - LOG_INFO(sLogger, ("hostId saved successfully to", fileName)); - } else { - int writeErrno = errno; - LOG_ERROR(sLogger, ("Failed to write hostId to file", fileName)("errno", strerror(writeErrno))); - } - close(fd); - } - } - return hostId; -} - -std::string FetchHostId() { - static std::string hostId; - if (!hostId.empty()) { - return hostId; - } - ECSMeta meta = FetchECSMeta(); - hostId = meta.instanceID; - if (!hostId.empty()) { - return hostId; - } - hostId = GetSerialNumberFromEcsAssist(); - if (!hostId.empty()) { - return hostId; - } - hostId = STRING_FLAG(agent_host_id); - if (!hostId.empty()) { - return hostId; - } - hostId = GetLocalHostId(); - - return hostId; -} - ECSMeta FetchECSMeta() { CURL* curl; for (size_t retryTimes = 1; retryTimes <= 5; retryTimes++) { diff --git a/core/common/MachineInfoUtil.h b/core/common/MachineInfoUtil.h index d8d6a3e963..10cbaaf86b 100644 --- a/core/common/MachineInfoUtil.h +++ b/core/common/MachineInfoUtil.h @@ -40,7 +40,6 @@ bool GetKernelInfo(std::string& kernelRelease, int64_t& kernelVersion); bool GetRedHatReleaseInfo(std::string& os, int64_t& osVersion, std::string bashPath = ""); bool IsDigitsDotsHostname(const char* hostname); ECSMeta FetchECSMeta(); -std::string FetchHostId(); // GetAnyAvailableIP walks through all interfaces (AF_INET) to find an available IP. // Priority: diff --git a/core/constants/EntityConstants.cpp b/core/constants/EntityConstants.cpp index 95db9051aa..afbe698d63 100644 --- a/core/constants/EntityConstants.cpp +++ b/core/constants/EntityConstants.cpp @@ -18,9 +18,8 @@ namespace logtail { -const std::string DEFAULT_ENV_KEY_HOST_TYPE = "HOST_TYPE"; -const std::string DEFAULT_ENV_VALUE_ECS = "ecs"; -const std::string DEFAULT_ENV_VALUE_HOST = "host"; +const std::string DEFAULT_HOST_TYPE_ECS = "ecs"; +const std::string DEFAULT_HOST_TYPE_HOST = "host"; const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE = "__entity_type__"; const std::string DEFAULT_CONTENT_KEY_ENTITY_ID = "__entity_id__"; const std::string DEFAULT_CONTENT_KEY_DOMAIN = "__domain__"; diff --git a/core/constants/EntityConstants.h b/core/constants/EntityConstants.h index 0da3a13920..cb544db989 100644 --- a/core/constants/EntityConstants.h +++ b/core/constants/EntityConstants.h @@ -19,8 +19,8 @@ namespace logtail { extern const std::string DEFAULT_ENV_KEY_HOST_TYPE; -extern const std::string DEFAULT_ENV_VALUE_ECS; -extern const std::string DEFAULT_ENV_VALUE_HOST; +extern const std::string DEFAULT_HOST_TYPE_ECS; +extern const std::string DEFAULT_HOST_TYPE_HOST; extern const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE; extern const std::string DEFAULT_CONTENT_KEY_ENTITY_ID; extern const std::string DEFAULT_CONTENT_KEY_DOMAIN; diff --git a/core/host_monitor/HostMonitorInputRunner.cpp b/core/host_monitor/HostMonitorInputRunner.cpp index 348ab13d94..3ac973df04 100755 --- a/core/host_monitor/HostMonitorInputRunner.cpp +++ b/core/host_monitor/HostMonitorInputRunner.cpp @@ -16,7 +16,7 @@ #include "HostMonitorInputRunner.h" -#include +#include #include #include #include @@ -26,7 +26,6 @@ #include "HostMonitorTimerEvent.h" #include "ProcessEntityCollector.h" -#include "common/Lock.h" #include "common/timer/Timer.h" #include "host_monitor/collector/ProcessEntityCollector.h" #include "logger/Logger.h" @@ -39,33 +38,33 @@ HostMonitorInputRunner::HostMonitorInputRunner() : mThreadPool(ThreadPool(3)) { RegisterCollector(); } -void HostMonitorInputRunner::UpdateCollector(const std::string& configName, - const std::vector& newCollectors, +void HostMonitorInputRunner::UpdateCollector(const std::vector& newCollectors, QueueKey processQueueKey, int inputIndex) { - std::vector oldCollectors; - { - std::unique_lock lock(mCollectorRegisterMapMutex); - auto it = mCollectorRegisterMap.find(configName); - if (it != mCollectorRegisterMap.end()) { - oldCollectors = it->second; - } - mCollectorRegisterMap[configName] = newCollectors; - } - for (const auto& collectorName : newCollectors) { - LOG_INFO(sLogger, ("add new host monitor collector", configName)("collector", collectorName)); - HostMonitorTimerEvent::CollectConfig collectConfig( - configName, collectorName, processQueueKey, inputIndex, std::chrono::seconds(DEFAULT_SCHEDULE_INTERVAL)); - // only push event when the collector is new added - if (std::find(oldCollectors.begin(), oldCollectors.end(), collectorName) == oldCollectors.end()) { - Timer::GetInstance()->PushEvent(BuildTimerEvent(collectConfig)); + std::unique_lock lock(mRegisteredCollectorMapMutex); + for (auto& collector : newCollectors) { + auto oldCollector = mRegisteredCollectorMap.find(collector); + if (oldCollector == mRegisteredCollectorMap.end()) { + mRegisteredCollectorMap[collector] = true; + HostMonitorTimerEvent::CollectConfig collectConfig( + collector, processQueueKey, inputIndex, std::chrono::seconds(DEFAULT_SCHEDULE_INTERVAL)); + auto now = std::chrono::steady_clock::now(); + auto event = std::make_unique(now + collectConfig.mInterval, collectConfig); + Timer::GetInstance()->PushEvent(std::move(event)); + } else { + // config removed and added again, timer event is still in the queue + if (!oldCollector->second) { + oldCollector->second = true; + } } } } -void HostMonitorInputRunner::RemoveCollector(const std::string& configName) { - std::unique_lock lock(mCollectorRegisterMapMutex); - mCollectorRegisterMap.erase(configName); +void HostMonitorInputRunner::RemoveCollector() { + std::unique_lock lock(mRegisteredCollectorMapMutex); + for (auto& collector : mRegisteredCollectorMap) { + collector.second = false; + } } void HostMonitorInputRunner::Init() { @@ -92,59 +91,51 @@ void HostMonitorInputRunner::Stop() { } bool HostMonitorInputRunner::HasRegisteredPlugins() const { - std::shared_lock lock(mCollectorRegisterMapMutex); - return !mCollectorRegisterMap.empty(); -} - -bool HostMonitorInputRunner::IsCollectTaskValid(const std::string& configName, const std::string& collectorName) const { - std::shared_lock lock(mCollectorRegisterMapMutex); - auto collectors = mCollectorRegisterMap.find(configName); - if (collectors == mCollectorRegisterMap.end()) { - return false; - } - for (const auto& collectorName : collectors->second) { - if (collectorName == collectorName) { + std::shared_lock lock(mRegisteredCollectorMapMutex); + for (auto& collector : mRegisteredCollectorMap) { + if (collector.second) { return true; } } return false; } +bool HostMonitorInputRunner::IsCollectTaskValid(const std::string& configName, const std::string& collectorName) { + std::unique_lock lock(mRegisteredCollectorMapMutex); + auto it = mRegisteredCollectorMap.find(collectorName); + if (it != mRegisteredCollectorMap.end() && it->second) { + return true; + } + return false; +} + void HostMonitorInputRunner::ScheduleOnce(HostMonitorTimerEvent::CollectConfig& config) { auto collectFn = [this, config]() mutable { PipelineEventGroup group(std::make_shared()); auto collector = GetCollector(config.mCollectorName); - if (!collector) { + if (collector) { collector->Collect(group); } else { LOG_ERROR(sLogger, - ("collector not found, will not collect", + ("collector not found, will not collect again", "discard data")("config", config.mConfigName)("collector", config.mCollectorName)); return; } - bool result = ProcessorRunner::GetInstance()->PushQueue( - config.mProcessQueueKey, config.mInputIndex, std::move(group), 3); - if (!result) { - LOG_WARNING(sLogger, - ("push queue failed", "discard data")("config", config.mConfigName)("collector", - config.mCollectorName)); + if (group.GetEvents().size() > 0) { + bool result = ProcessorRunner::GetInstance()->PushQueue( + config.mProcessQueueKey, config.mInputIndex, std::move(group)); + if (!result) { + // there is no process for host monitor, so should not block in process queue + LOG_ERROR(sLogger, + ("host monitor push process queue failed", + "discard data")("config", config.mConfigName)("collector", config.mCollectorName)); + } } - LOG_DEBUG(sLogger, - ("schedule host monitor collector again", config.mConfigName)("collector", config.mCollectorName)); - auto event = BuildTimerEvent(config); - event->ResetForNextExec(); - Timer::GetInstance()->PushEvent(std::move(event)); }; mThreadPool.Add(collectFn); } -std::unique_ptr -HostMonitorInputRunner::BuildTimerEvent(HostMonitorTimerEvent::CollectConfig& collectConfig) { - auto now = std::chrono::steady_clock::now(); - auto event = std::make_unique(now, collectConfig); - return event; -} std::shared_ptr HostMonitorInputRunner::GetCollector(const std::string& collectorName) { auto it = mCollectorInstanceMap.find(collectorName); diff --git a/core/host_monitor/HostMonitorInputRunner.h b/core/host_monitor/HostMonitorInputRunner.h index c8cfa94609..461316acd0 100644 --- a/core/host_monitor/HostMonitorInputRunner.h +++ b/core/host_monitor/HostMonitorInputRunner.h @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include "common/ThreadPool.h" @@ -46,22 +48,19 @@ class HostMonitorInputRunner : public InputRunner { return &sInstance; } - void UpdateCollector(const std::string& configName, - const std::vector& collectorNames, - QueueKey processQueueKey, - int inputIndex); - void RemoveCollector(const std::string& configName); + // Only support singleton mode + void UpdateCollector(const std::vector& collectorNames, QueueKey processQueueKey, int inputIndex); + void RemoveCollector(); void Init() override; void Stop() override; bool HasRegisteredPlugins() const override; - bool IsCollectTaskValid(const std::string& configName, const std::string& collectorName) const; + bool IsCollectTaskValid(const std::string& configName, const std::string& collectorName); void ScheduleOnce(HostMonitorTimerEvent::CollectConfig& collectConfig); private: HostMonitorInputRunner(); - std::unique_ptr BuildTimerEvent(HostMonitorTimerEvent::CollectConfig& collectConfig); template void RegisterCollector(); @@ -71,8 +70,8 @@ class HostMonitorInputRunner : public InputRunner { ThreadPool mThreadPool; - mutable std::shared_mutex mCollectorRegisterMapMutex; - std::unordered_map> mCollectorRegisterMap; + mutable std::shared_mutex mRegisteredCollectorMapMutex; + std::unordered_map mRegisteredCollectorMap; std::unordered_map> mCollectorInstanceMap; #ifdef APSARA_UNIT_TEST_MAIN diff --git a/core/host_monitor/HostMonitorTimerEvent.cpp b/core/host_monitor/HostMonitorTimerEvent.cpp index 32c5dae81a..150801afd4 100755 --- a/core/host_monitor/HostMonitorTimerEvent.cpp +++ b/core/host_monitor/HostMonitorTimerEvent.cpp @@ -16,9 +16,8 @@ #include "HostMonitorTimerEvent.h" -#include - -#include "HostMonitorInputRunner.h" +#include "common/timer/Timer.h" +#include "host_monitor/HostMonitorInputRunner.h" namespace logtail { @@ -28,7 +27,10 @@ bool HostMonitorTimerEvent::IsValid() const { } bool HostMonitorTimerEvent::Execute() { + LOG_DEBUG(sLogger, ("schedule host monitor collector", mCollectConfig.mConfigName)); HostMonitorInputRunner::GetInstance()->ScheduleOnce(mCollectConfig); + auto event = std::make_unique(GetExecTime() + mCollectConfig.mInterval, mCollectConfig); + Timer::GetInstance()->PushEvent(std::move(event)); return true; } diff --git a/core/host_monitor/HostMonitorTimerEvent.h b/core/host_monitor/HostMonitorTimerEvent.h index 1279982f79..0ae67cfd37 100644 --- a/core/host_monitor/HostMonitorTimerEvent.h +++ b/core/host_monitor/HostMonitorTimerEvent.h @@ -30,13 +30,11 @@ namespace logtail { class HostMonitorTimerEvent : public TimerEvent { public: struct CollectConfig { - CollectConfig(std::string configName, - std::string collectorName, + CollectConfig(std::string collectorName, QueueKey processQueueKey, int inputIndex, std::chrono::seconds interval) - : mConfigName(configName), - mCollectorName(collectorName), + : mCollectorName(collectorName), mProcessQueueKey(processQueueKey), mInputIndex(inputIndex), mInterval(interval) {} @@ -48,14 +46,13 @@ class HostMonitorTimerEvent : public TimerEvent { std::chrono::seconds mInterval; }; - HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime, CollectConfig collectConfig) - : TimerEvent(execTime), mCollectConfig(collectConfig) {} - bool IsValid() const override; bool Execute() override; - void ResetForNextExec() { SetExecTime(GetExecTime() + mCollectConfig.mInterval); } private: + HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime, CollectConfig collectConfig) + : TimerEvent(execTime), mCollectConfig(collectConfig) {} + CollectConfig mCollectConfig; }; diff --git a/core/host_monitor/SystemInformationTools.cpp b/core/host_monitor/SystemInformationTools.cpp index edbad31cab..c36b7c92e8 100644 --- a/core/host_monitor/SystemInformationTools.cpp +++ b/core/host_monitor/SystemInformationTools.cpp @@ -30,8 +30,8 @@ int64_t GetSystemBootSeconds() { std::vector cpuLines = {}; std::string errorMessage; int ret = GetFileLines(PROCESS_DIR / PROCESS_STAT, cpuLines, true, &errorMessage); - LOG_WARNING(sLogger, ("failed to get cpu lines", errorMessage)("ret", ret)("cpuLines", cpuLines.size())); if (ret != 0 || cpuLines.empty()) { + LOG_WARNING(sLogger, ("failed to get cpu lines", errorMessage)("ret", ret)("cpuLines", cpuLines.size())); return duration_cast(system_clock::now().time_since_epoch()).count(); } diff --git a/core/host_monitor/collector/ProcessEntityCollector.cpp b/core/host_monitor/collector/ProcessEntityCollector.cpp index 70682c83c9..3eb1e4eaca 100755 --- a/core/host_monitor/collector/ProcessEntityCollector.cpp +++ b/core/host_monitor/collector/ProcessEntityCollector.cpp @@ -46,23 +46,25 @@ const std::string ProcessEntityCollector::sName = "process_entity"; ProcessEntityCollector::ProcessEntityCollector() : mProcessSilentCount(INT32_FLAG(process_collect_silent_count)) { // try to read process dir if (access(PROCESS_DIR.c_str(), R_OK) != 0) { - LOG_ERROR(sLogger, ("process collector init failed", "process dir not exist or ")("dir", PROCESS_DIR)); + LOG_ERROR(sLogger, + ("process collector init failed", "process dir not exist or no permission")("dir", PROCESS_DIR)); mValidState = false; } else { mValidState = true; } - auto hostType = ToString(getenv(DEFAULT_ENV_KEY_HOST_TYPE.c_str())); + // TODO: fix after host id ready + mHostEntityID = ""; // FetchHostId(); + mDomain = ""; std::ostringstream oss; - if (hostType == DEFAULT_ENV_VALUE_ECS) { - oss << DEFAULT_CONTENT_VALUE_DOMAIN_ACS << "." << DEFAULT_ENV_VALUE_ECS << "." + if (mDomain == DEFAULT_HOST_TYPE_ECS) { + oss << DEFAULT_CONTENT_VALUE_DOMAIN_ACS << "." << DEFAULT_HOST_TYPE_ECS << "." << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_ACS; } else { - oss << DEFAULT_CONTENT_VALUE_DOMAIN_INFRA << "." << DEFAULT_ENV_VALUE_HOST << "." + oss << DEFAULT_CONTENT_VALUE_DOMAIN_INFRA << "." << DEFAULT_HOST_TYPE_HOST << "." << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_INFRA; } - mHostEntityID = FetchHostId(); mEntityType = oss.str(); }; @@ -90,9 +92,9 @@ void ProcessEntityCollector::Collect(PipelineEventGroup& group) { // custom fields event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PID, std::to_string(process->pid)); event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PPID, std::to_string(process->parentPid)); - // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_USER, ""); TODO: get user name event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_COMM, process->name); event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME, processCreateTime); + // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_USER, ""); TODO: get user name // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CWD, ""); TODO: get cwd // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_BINARY, ""); TODO: get binary // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS, ""); TODO: get arguments @@ -151,13 +153,13 @@ ProcessStatPtr ProcessEntityCollector::GetProcessStat(pid_t pid, bool& isFirstCo const auto now = steady_clock::now(); // TODO: more accurate cache - auto prev = GetPreProcessStat(pid); + auto prev = mPrevProcessStat[pid]; isFirstCollect = prev == nullptr || prev->lastTime.time_since_epoch().count() == 0; // proc/[pid]/stat的统计粒度通常为10ms，两次采样之间需要足够大才能平滑。 if (prev && now < prev->lastTime + seconds{1}) { return prev; } - auto ptr = ReadProcessStat(pid); + auto ptr = ReadNewProcessStat(pid); if (!ptr) { return nullptr; } @@ -184,7 +186,7 @@ ProcessStatPtr ProcessEntityCollector::GetProcessStat(pid_t pid, bool& isFirstCo return ptr; } -ProcessStatPtr ProcessEntityCollector::ReadProcessStat(pid_t pid) { +ProcessStatPtr ProcessEntityCollector::ReadNewProcessStat(pid_t pid) { LOG_DEBUG(sLogger, ("read process stat", pid)); auto processStat = PROCESS_DIR / std::to_string(pid) / PROCESS_STAT; diff --git a/core/host_monitor/collector/ProcessEntityCollector.h b/core/host_monitor/collector/ProcessEntityCollector.h index 3e84ab34b1..d189a694ed 100755 --- a/core/host_monitor/collector/ProcessEntityCollector.h +++ b/core/host_monitor/collector/ProcessEntityCollector.h @@ -154,13 +154,11 @@ class ProcessEntityCollector : public BaseCollector { private: void GetSortedProcess(std::vector& processStats, size_t topN); ProcessStatPtr GetProcessStat(pid_t pid, bool& isFirstCollect); - ProcessStatPtr ReadProcessStat(pid_t pid); + ProcessStatPtr ReadNewProcessStat(pid_t pid); ProcessStatPtr ParseProcessStat(pid_t pid, std::string& line); - bool WalkAllProcess(const std::filesystem::path& root, const std::function& callback); - const std::string GetProcessEntityID(StringView pid, StringView createTime); - ProcessStatPtr GetPreProcessStat(pid_t pid) { return mPrevProcessStat[pid]; } + const std::string GetProcessEntityID(StringView pid, StringView createTime); steady_clock::time_point mProcessSortTime; std::vector mSortProcessStats; diff --git a/core/plugin/input/InputHostMeta.cpp b/core/plugin/input/InputHostMeta.cpp index f6e151d926..88a4443d6d 100644 --- a/core/plugin/input/InputHostMeta.cpp +++ b/core/plugin/input/InputHostMeta.cpp @@ -30,15 +30,14 @@ bool InputHostMeta::Init(const Json::Value& config, Json::Value& optionalGoPipel bool InputHostMeta::Start() { LOG_INFO(sLogger, ("input host meta start", mContext->GetConfigName())); HostMonitorInputRunner::GetInstance()->Init(); - HostMonitorInputRunner::GetInstance()->UpdateCollector( - mContext->GetConfigName(), {"process"}, mContext->GetProcessQueueKey(), mIndex); + HostMonitorInputRunner::GetInstance()->UpdateCollector({"process"}, mContext->GetProcessQueueKey(), mIndex); return true; } bool InputHostMeta::Stop(bool isPipelineRemoving) { LOG_INFO(sLogger, ("input host meta stop", mContext->GetConfigName())); if (isPipelineRemoving) { - HostMonitorInputRunner::GetInstance()->RemoveCollector(mContext->GetConfigName()); + HostMonitorInputRunner::GetInstance()->RemoveCollector(); } return true; } diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index ec4a2db50b..eeb4260413 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -83,7 +83,7 @@ void PrometheusInputRunner::UpdateScrapeInput(std::shared_ptrInitSelfMonitor(defaultLabels); targetSubscriber->mUnRegisterMs = mUnRegisterMs.load(); - targetSubscriber->SetComponent(mTimer, &mEventPool); + targetSubscriber->SetComponent(&mEventPool); auto currSystemTime = chrono::system_clock::now(); auto randSleepMilliSec = GetRandSleepMilliSec(targetSubscriber->GetId(), diff --git a/core/unittest/host_monitor/CMakeLists.txt b/core/unittest/host_monitor/CMakeLists.txt index fd761c8c74..9d27ca9d41 100644 --- a/core/unittest/host_monitor/CMakeLists.txt +++ b/core/unittest/host_monitor/CMakeLists.txt @@ -15,8 +15,8 @@ cmake_minimum_required(VERSION 3.22) project(host_monitor_unittest) -add_executable(process_collector_unittest ProcessCollectorUnittest.cpp) -target_link_libraries(process_collector_unittest ${UT_BASE_TARGET}) +add_executable(process_entity_collector_unittest ProcessEntityCollectorUnittest.cpp) +target_link_libraries(process_entity_collector_unittest ${UT_BASE_TARGET}) add_executable(host_monitor_input_runner_unittest HostMonitorInputRunnerUnittest.cpp) target_link_libraries(host_monitor_input_runner_unittest ${UT_BASE_TARGET}) @@ -29,6 +29,6 @@ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/1/ DESTINATION ${CMAKE_CURRENT_BINARY_DIR} file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/stat DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) include(GoogleTest) -gtest_discover_tests(process_collector_unittest) +gtest_discover_tests(process_entity_collector_unittest) gtest_discover_tests(host_monitor_input_runner_unittest) gtest_discover_tests(system_information_tools_unittest) diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp index f5b326830d..6087696990 100644 --- a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -17,6 +17,7 @@ #include "HostMonitorInputRunner.h" #include "HostMonitorTimerEvent.h" +#include "Logger.h" #include "ProcessQueueItem.h" #include "ProcessQueueManager.h" #include "QueueKey.h" @@ -37,10 +38,10 @@ class HostMonitorInputRunnerUnittest : public testing::Test { void HostMonitorInputRunnerUnittest::TestUpdateAndRemoveCollector() const { auto runner = HostMonitorInputRunner::GetInstance(); runner->Init(); - runner->UpdateCollector("test", {"mock"}, QueueKey{}, 0); + runner->UpdateCollector({"mock"}, QueueKey{}, 0); APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid("test", "mock")); APSARA_TEST_TRUE_FATAL(runner->HasRegisteredPlugins()); - runner->RemoveCollector("test"); + runner->RemoveCollector(); APSARA_TEST_FALSE_FATAL(runner->IsCollectTaskValid("test", "mock")); APSARA_TEST_FALSE_FATAL(runner->HasRegisteredPlugins()); runner->Stop(); @@ -56,14 +57,15 @@ void HostMonitorInputRunnerUnittest::TestScheduleOnce() const { ctx.SetConfigName(configName); ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(queueKey, 0, ctx); - HostMonitorTimerEvent::CollectConfig collectConfig(configName, "process", queueKey, 0, std::chrono::seconds(1)); + HostMonitorTimerEvent::CollectConfig collectConfig("process_entity", queueKey, 0, std::chrono::seconds(60)); + runner->ScheduleOnce(collectConfig); + std::this_thread::sleep_for(std::chrono::seconds(1)); runner->ScheduleOnce(collectConfig); std::this_thread::sleep_for(std::chrono::seconds(1)); auto item = std::unique_ptr(new ProcessQueueItem(std::make_shared(), 0)); ProcessQueueManager::GetInstance()->EnablePop(configName); APSARA_TEST_TRUE_FATAL(ProcessQueueManager::GetInstance()->PopItem(0, item, configName)); APSARA_TEST_EQUAL_FATAL("test", configName); - APSARA_TEST_TRUE_FATAL(item->mEventGroup.GetEvents().size() == 1); // verify schdule next APSARA_TEST_EQUAL_FATAL(Timer::GetInstance()->mQueue.size(), 1); diff --git a/core/unittest/host_monitor/ProcessCollectorUnittest.cpp b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp similarity index 60% rename from core/unittest/host_monitor/ProcessCollectorUnittest.cpp rename to core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp index 1c13432df0..3bcadcb618 100644 --- a/core/unittest/host_monitor/ProcessCollectorUnittest.cpp +++ b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp @@ -12,32 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ProcessCollector.h" #include "host_monitor/Constants.h" +#include "host_monitor/collector/ProcessEntityCollector.h" #include "unittest/Unittest.h" using namespace std; namespace logtail { -class ProcessCollectorUnittest : public testing::Test { +class ProcessEntityCollectorUnittest : public testing::Test { public: - void TestReadProcessStat() const; + void TestGetNewProcessStat() const; void TestSortProcessByCpu() const; + void TestGetProcessEntityID() const; }; -void ProcessCollectorUnittest::TestReadProcessStat() const { +void ProcessEntityCollectorUnittest::TestGetNewProcessStat() const { PROCESS_DIR = "."; - auto collector = ProcessCollector(); - auto ptr = collector.ReadProcessStat(1); + auto collector = ProcessEntityCollector(); + auto ptr = collector.GetNewProcessStat(1); APSARA_TEST_NOT_EQUAL(nullptr, ptr); APSARA_TEST_EQUAL(1, ptr->pid); APSARA_TEST_EQUAL("cat", ptr->name); } -void ProcessCollectorUnittest::TestSortProcessByCpu() const { +void ProcessEntityCollectorUnittest::TestSortProcessByCpu() const { PROCESS_DIR = "/proc"; - auto collector = ProcessCollector(); + auto collector = ProcessEntityCollector(); auto processes = vector(); collector.GetSortedProcess(processes, 5); // fist time will be ignored collector.GetSortedProcess(processes, 5); @@ -50,8 +51,15 @@ void ProcessCollectorUnittest::TestSortProcessByCpu() const { } } -UNIT_TEST_CASE(ProcessCollectorUnittest, TestReadProcessStat); -UNIT_TEST_CASE(ProcessCollectorUnittest, TestSortProcessByCpu); +void ProcessEntityCollectorUnittest::TestGetProcessEntityID() const { + ProcessEntityCollector collect; + collect.mHostEntityID = "123"; + APSARA_TEST_EQUAL(collect.GetProcessEntityID("123", "123"), "f5bb0c8de146c67b44babbf4e6584cc0"); +} + +UNIT_TEST_CASE(ProcessEntityCollectorUnittest, TestGetNewProcessStat); +UNIT_TEST_CASE(ProcessEntityCollectorUnittest, TestSortProcessByCpu); +UNIT_TEST_CASE(ProcessEntityCollectorUnittest, TestGetProcessEntityID); } // namespace logtail diff --git a/core/unittest/input/InputHostMetaUnittest.cpp b/core/unittest/input/InputHostMetaUnittest.cpp index 9c284f945a..d1e1b8b08d 100644 --- a/core/unittest/input/InputHostMetaUnittest.cpp +++ b/core/unittest/input/InputHostMetaUnittest.cpp @@ -16,7 +16,6 @@ #include "PluginRegistry.h" #include "common/JsonUtil.h" -#include "ebpf/config.h" #include "pipeline/Pipeline.h" #include "plugin/input/InputHostMeta.h" #include "unittest/Unittest.h" @@ -57,7 +56,7 @@ void InputHostMetaUnittest::TestName() { void InputHostMetaUnittest::TestSupportAck() { InputHostMeta input; bool supportAck = input.SupportAck(); - APSARA_TEST_FALSE(supportAck); + APSARA_TEST_TRUE(supportAck); } void InputHostMetaUnittest::OnSuccessfulInit() { diff --git a/core/unittest/processor/CMakeLists.txt b/core/unittest/processor/CMakeLists.txt index 3b4d3e0fcf..39b5721186 100644 --- a/core/unittest/processor/CMakeLists.txt +++ b/core/unittest/processor/CMakeLists.txt @@ -57,9 +57,6 @@ target_link_libraries(processor_parse_container_log_native_unittest ${UT_BASE_TA add_executable(processor_prom_parse_metric_native_unittest ProcessorPromParseMetricNativeUnittest.cpp) target_link_libraries(processor_prom_parse_metric_native_unittest unittest_base) -add_executable(processor_host_meta_native_unittest ProcessorHostMetaNativeUnittest.cpp) -target_link_libraries(processor_host_meta_native_unittest unittest_base) - include(GoogleTest) gtest_discover_tests(processor_split_log_string_native_unittest) gtest_discover_tests(processor_split_multiline_log_string_native_unittest) @@ -75,7 +72,6 @@ gtest_discover_tests(processor_desensitize_native_unittest) gtest_discover_tests(processor_merge_multiline_log_native_unittest) gtest_discover_tests(processor_parse_container_log_native_unittest) gtest_discover_tests(processor_prom_parse_metric_native_unittest) -gtest_discover_tests(processor_host_meta_native_unittest) add_executable(boost_regex_benchmark BoostRegexBenchmark.cpp) target_link_libraries(boost_regex_benchmark ${UT_BASE_TARGET}) diff --git a/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp b/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp deleted file mode 100644 index 1d00798a32..0000000000 --- a/core/unittest/processor/ProcessorHostMetaNativeUnittest.cpp +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2023 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "LogEvent.h" -#include "constants/EntityConstants.h" -#include "pipeline/Pipeline.h" -#include "plugin/processor/inner/ProcessorHostMetaNative.h" -#include "unittest/Unittest.h" - -namespace logtail { - -class ProcessorHostMetaNativeUnittest : public ::testing::Test { -public: - void TestInit(); - void TestProcess(); - void TestGetProcessEntityID(); - -protected: - void SetUp() override { mContext.SetConfigName("project##config_0"); } - -private: - PipelineContext mContext; -}; - -void ProcessorHostMetaNativeUnittest::TestInit() { - // make config - Json::Value config; - Pipeline pipeline; - mContext.SetPipeline(pipeline); - - { - setenv(DEFAULT_ENV_KEY_HOST_TYPE.c_str(), DEFAULT_ENV_VALUE_ECS.c_str(), 1); - ProcessorHostMetaNative processor; - processor.SetContext(mContext); - APSARA_TEST_TRUE_FATAL(processor.Init(config)); - APSARA_TEST_EQUAL_FATAL(processor.mDomain, DEFAULT_CONTENT_VALUE_DOMAIN_ACS); - APSARA_TEST_EQUAL_FATAL(processor.mEntityType, - DEFAULT_CONTENT_VALUE_DOMAIN_ACS + "." + DEFAULT_ENV_VALUE_ECS + "." - + DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS); - } - { - setenv(DEFAULT_ENV_KEY_HOST_TYPE.c_str(), DEFAULT_CONTENT_VALUE_DOMAIN_INFRA.c_str(), 1); - ProcessorHostMetaNative processor; - processor.SetContext(mContext); - APSARA_TEST_TRUE_FATAL(processor.Init(config)); - APSARA_TEST_EQUAL_FATAL(processor.mDomain, DEFAULT_CONTENT_VALUE_DOMAIN_INFRA); - APSARA_TEST_EQUAL_FATAL(processor.mEntityType, - DEFAULT_CONTENT_VALUE_DOMAIN_INFRA + "." + DEFAULT_ENV_VALUE_HOST + "." - + DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS); - } -} - -void ProcessorHostMetaNativeUnittest::TestProcess() { - // make config - Json::Value config; - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::COLLECT_TIME, "123456"); - auto event = eventGroup.AddLogEvent(); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PID, "123"); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME, "123456"); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PPID, "123"); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_USER, "root"); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_COMM, "comm"); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CWD, "cwd"); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_BINARY, "binary"); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS, "arguments"); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE, "language"); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID, "container_id"); - - ProcessorHostMetaNative processor; - processor.SetContext(mContext); - APSARA_TEST_TRUE_FATAL(processor.Init(config)); - processor.Process(eventGroup); - APSARA_TEST_EQUAL_FATAL(eventGroup.GetEvents().size(), 1); - auto newEvent = eventGroup.GetEvents().front().Cast(); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_DOMAIN), processor.mDomain); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_ENTITY_TYPE), processor.mEntityType); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME), "123456"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS), "30"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent(DEFAULT_CONTENT_KEY_METHOD), DEFAULT_CONTENT_VALUE_METHOD_UPDATE); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("pid"), "123"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("ppid"), "123"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("user"), "root"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("comm"), "comm"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("create_time"), "123456"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("cwd"), "cwd"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("binary"), "binary"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("arguments"), "arguments"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("language"), "language"); - APSARA_TEST_EQUAL_FATAL(newEvent.GetContent("containerID"), "container_id"); -} - -void ProcessorHostMetaNativeUnittest::TestGetProcessEntityID() { - ProcessorHostMetaNative processor; - processor.Init(Json::Value()); - processor.mHostEntityID = "123"; - APSARA_TEST_EQUAL(processor.GetProcessEntityID("123", "123"), "f5bb0c8de146c67b44babbf4e6584cc0"); -} - -UNIT_TEST_CASE(ProcessorHostMetaNativeUnittest, TestInit) -UNIT_TEST_CASE(ProcessorHostMetaNativeUnittest, TestProcess) -UNIT_TEST_CASE(ProcessorHostMetaNativeUnittest, TestGetProcessEntityID) - -} // namespace logtail - -UNIT_TEST_MAIN diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index d7415ce991..9b2f4f30e6 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -250,9 +250,8 @@ void ScrapeSchedulerUnittest::TestExactlyScrape() { ScrapeScheduler event(mScrapeConfig, "localhost", 8080, labels, 0, 0); auto defaultLabels = MetricLabels(); event.InitSelfMonitor(defaultLabels); - auto timer = make_shared(); EventPool eventPool{true}; - event.SetComponent(timer, &eventPool); + event.SetComponent(&eventPool); auto execTime = std::chrono::steady_clock::now(); auto scrapeTime = std::chrono::system_clock::now(); event.SetFirstExecTime(execTime, scrapeTime); From cc1f455dd0690b7596c726febf07b84846546651 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 27 Dec 2024 18:13:02 +0800 Subject: [PATCH 05/13] fix --- core/host_monitor/HostMonitorTimerEvent.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/host_monitor/HostMonitorTimerEvent.h b/core/host_monitor/HostMonitorTimerEvent.h index 0ae67cfd37..db2794ea10 100644 --- a/core/host_monitor/HostMonitorTimerEvent.h +++ b/core/host_monitor/HostMonitorTimerEvent.h @@ -46,13 +46,13 @@ class HostMonitorTimerEvent : public TimerEvent { std::chrono::seconds mInterval; }; + HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime, CollectConfig collectConfig) + : TimerEvent(execTime), mCollectConfig(collectConfig) {} + bool IsValid() const override; bool Execute() override; private: - HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime, CollectConfig collectConfig) - : TimerEvent(execTime), mCollectConfig(collectConfig) {} - CollectConfig mCollectConfig; }; From 0693bb10a29eb9223838463e41b0f82b38851281 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 27 Dec 2024 22:42:55 +0800 Subject: [PATCH 06/13] fix --- core/common/timer/Timer.cpp | 7 ++++ core/common/timer/Timer.h | 3 +- core/host_monitor/HostMonitorInputRunner.cpp | 6 +++ core/plugin/input/InputHostMeta.cpp | 2 +- core/prometheus/PrometheusInputRunner.cpp | 2 - core/prometheus/PrometheusInputRunner.h | 1 - core/prometheus/schedulers/BaseScheduler.cpp | 1 - core/prometheus/schedulers/BaseScheduler.h | 4 +- .../prometheus/schedulers/ScrapeScheduler.cpp | 6 +-- .../schedulers/TargetSubscriberScheduler.cpp | 40 +++++++++---------- .../HostMonitorInputRunnerUnittest.cpp | 8 +++- .../ProcessEntityCollectorUnittest.cpp | 2 +- .../prometheus/ScrapeSchedulerUnittest.cpp | 2 + 13 files changed, 47 insertions(+), 37 deletions(-) diff --git a/core/common/timer/Timer.cpp b/core/common/timer/Timer.cpp index 7533b1da24..e33aa0aacc 100644 --- a/core/common/timer/Timer.cpp +++ b/core/common/timer/Timer.cpp @@ -51,6 +51,13 @@ void Timer::Stop() { } } +void Timer::Clear() { + lock_guard lock(mQueueMux); + while (!mQueue.empty()) { + mQueue.pop(); + } +} + void Timer::PushEvent(unique_ptr&& e) { lock_guard lock(mQueueMux); if (mQueue.empty() || e->GetExecTime() < mQueue.top()->GetExecTime()) { diff --git a/core/common/timer/Timer.h b/core/common/timer/Timer.h index ad1ca15be4..efb1ec4f5e 100644 --- a/core/common/timer/Timer.h +++ b/core/common/timer/Timer.h @@ -46,6 +46,7 @@ class Timer { void Init(); void Stop(); void PushEvent(std::unique_ptr&& e); + void Clear(); private: Timer() = default; @@ -57,7 +58,7 @@ class Timer { std::future mThreadRes; mutable std::mutex mThreadRunningMux; - bool mIsThreadRunning = true; + bool mIsThreadRunning = false; mutable std::condition_variable mCV; #ifdef APSARA_UNIT_TEST_MAIN diff --git a/core/host_monitor/HostMonitorInputRunner.cpp b/core/host_monitor/HostMonitorInputRunner.cpp index 3ac973df04..a852c3d67f 100755 --- a/core/host_monitor/HostMonitorInputRunner.cpp +++ b/core/host_monitor/HostMonitorInputRunner.cpp @@ -45,12 +45,17 @@ void HostMonitorInputRunner::UpdateCollector(const std::vector& new for (auto& collector : newCollectors) { auto oldCollector = mRegisteredCollectorMap.find(collector); if (oldCollector == mRegisteredCollectorMap.end()) { + if (mCollectorInstanceMap.find(collector) == mCollectorInstanceMap.end()) { + LOG_ERROR(sLogger, ("host monitor", "collector not found")("collector", collector)); + continue; + } mRegisteredCollectorMap[collector] = true; HostMonitorTimerEvent::CollectConfig collectConfig( collector, processQueueKey, inputIndex, std::chrono::seconds(DEFAULT_SCHEDULE_INTERVAL)); auto now = std::chrono::steady_clock::now(); auto event = std::make_unique(now + collectConfig.mInterval, collectConfig); Timer::GetInstance()->PushEvent(std::move(event)); + LOG_INFO(sLogger, ("host monitor", "add new collector")("collector", collector)); } else { // config removed and added again, timer event is still in the queue if (!oldCollector->second) { @@ -122,6 +127,7 @@ void HostMonitorInputRunner::ScheduleOnce(HostMonitorTimerEvent::CollectConfig& return; } + LOG_DEBUG(sLogger, ("host monitor collect", "collector")(config.mCollectorName, group.GetEvents().size())); if (group.GetEvents().size() > 0) { bool result = ProcessorRunner::GetInstance()->PushQueue( config.mProcessQueueKey, config.mInputIndex, std::move(group)); diff --git a/core/plugin/input/InputHostMeta.cpp b/core/plugin/input/InputHostMeta.cpp index 88a4443d6d..0a5c06117a 100644 --- a/core/plugin/input/InputHostMeta.cpp +++ b/core/plugin/input/InputHostMeta.cpp @@ -30,7 +30,7 @@ bool InputHostMeta::Init(const Json::Value& config, Json::Value& optionalGoPipel bool InputHostMeta::Start() { LOG_INFO(sLogger, ("input host meta start", mContext->GetConfigName())); HostMonitorInputRunner::GetInstance()->Init(); - HostMonitorInputRunner::GetInstance()->UpdateCollector({"process"}, mContext->GetProcessQueueKey(), mIndex); + HostMonitorInputRunner::GetInstance()->UpdateCollector({"process_entity"}, mContext->GetProcessQueueKey(), mIndex); return true; } diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index eeb4260413..931bedf374 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -137,7 +137,6 @@ void PrometheusInputRunner::Init() { mIsStarted = true; #ifndef APSARA_UNIT_TEST_MAIN - mTimer->Init(); AsynCurlRunner::GetInstance()->Init(); #endif @@ -203,7 +202,6 @@ void PrometheusInputRunner::Stop() { } #ifndef APSARA_UNIT_TEST_MAIN - mTimer->Stop(); LOG_INFO(sLogger, ("PrometheusInputRunner", "stop asyn curl runner")); AsynCurlRunner::GetInstance()->Stop(); #endif diff --git a/core/prometheus/PrometheusInputRunner.h b/core/prometheus/PrometheusInputRunner.h index 996caf163e..cca25857f8 100644 --- a/core/prometheus/PrometheusInputRunner.h +++ b/core/prometheus/PrometheusInputRunner.h @@ -75,7 +75,6 @@ class PrometheusInputRunner : public InputRunner { std::string mPodName; std::unique_ptr mClient; - std::shared_ptr mTimer; EventPool mEventPool; mutable ReadWriteLock mSubscriberMapRWLock; diff --git a/core/prometheus/schedulers/BaseScheduler.cpp b/core/prometheus/schedulers/BaseScheduler.cpp index 72532851a9..58d286b36b 100644 --- a/core/prometheus/schedulers/BaseScheduler.cpp +++ b/core/prometheus/schedulers/BaseScheduler.cpp @@ -40,7 +40,6 @@ bool BaseScheduler::IsCancelled() { } void BaseScheduler::SetComponent(EventPool* eventPool) { - mTimer = Timer::GetInstance(); mEventPool = eventPool; } } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/BaseScheduler.h b/core/prometheus/schedulers/BaseScheduler.h index e246b63609..985d2bbe55 100644 --- a/core/prometheus/schedulers/BaseScheduler.h +++ b/core/prometheus/schedulers/BaseScheduler.h @@ -20,7 +20,8 @@ class BaseScheduler { std::chrono::steady_clock::time_point GetNextExecTime(); - void SetFirstExecTime(std::chrono::steady_clock::time_point firstExecTime,std::chrono::system_clock::time_point firstScrapeTime); + void SetFirstExecTime(std::chrono::steady_clock::time_point firstExecTime, + std::chrono::system_clock::time_point firstScrapeTime); void DelayExecTime(uint64_t delaySeconds); virtual void Cancel(); @@ -44,7 +45,6 @@ class BaseScheduler { std::shared_ptr> mFuture; std::shared_ptr> mIsContextValidFuture; - Timer* mTimer; EventPool* mEventPool = nullptr; }; } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index 0adfae2209..1e97e6a78e 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -190,7 +190,7 @@ void ScrapeScheduler::ScheduleNext() { } auto event = BuildScrapeTimerEvent(GetNextExecTime()); - mTimer->PushEvent(std::move(event)); + Timer::GetInstance()->PushEvent(std::move(event)); } void ScrapeScheduler::ScrapeOnce(std::chrono::steady_clock::time_point execTime) { @@ -201,9 +201,7 @@ void ScrapeScheduler::ScrapeOnce(std::chrono::steady_clock::time_point execTime) }); mFuture = future; auto event = BuildScrapeTimerEvent(execTime); - if (mTimer) { - mTimer->PushEvent(std::move(event)); - } + Timer::GetInstance()->PushEvent(std::move(event)); } std::unique_ptr ScrapeScheduler::BuildScrapeTimerEvent(std::chrono::steady_clock::time_point execTime) { diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index ffad53f844..451cd4d9c1 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -106,25 +106,23 @@ void TargetSubscriberScheduler::UpdateScrapeScheduler( for (const auto& [k, v] : newScrapeSchedulerMap) { if (mScrapeSchedulerMap.find(k) == mScrapeSchedulerMap.end()) { mScrapeSchedulerMap[k] = v; - if (mTimer) { - auto tmpCurrentMilliSeconds = GetCurrentTimeInMilliSeconds(); - auto tmpRandSleepMilliSec = GetRandSleepMilliSec( - v->GetId(), mScrapeConfigPtr->mScrapeIntervalSeconds, tmpCurrentMilliSeconds); - - // zero-cost upgrade - if (mUnRegisterMs > 0 - && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - - (uint64_t)mScrapeConfigPtr->mScrapeIntervalSeconds * 1000 - > mUnRegisterMs) - && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - - (uint64_t)mScrapeConfigPtr->mScrapeIntervalSeconds * 1000 * 2 - < mUnRegisterMs)) { - // scrape once just now - LOG_INFO(sLogger, ("scrape zero cost", ToString(tmpCurrentMilliSeconds))); - v->ScrapeOnce(std::chrono::steady_clock::now()); - } - v->ScheduleNext(); + auto tmpCurrentMilliSeconds = GetCurrentTimeInMilliSeconds(); + auto tmpRandSleepMilliSec = GetRandSleepMilliSec( + v->GetId(), mScrapeConfigPtr->mScrapeIntervalSeconds, tmpCurrentMilliSeconds); + + // zero-cost upgrade + if (mUnRegisterMs > 0 + && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec + - (uint64_t)mScrapeConfigPtr->mScrapeIntervalSeconds * 1000 + > mUnRegisterMs) + && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec + - (uint64_t)mScrapeConfigPtr->mScrapeIntervalSeconds * 1000 * 2 + < mUnRegisterMs)) { + // scrape once just now + LOG_INFO(sLogger, ("scrape zero cost", ToString(tmpCurrentMilliSeconds))); + v->ScrapeOnce(std::chrono::steady_clock::now()); } + v->ScheduleNext(); } } } @@ -261,7 +259,7 @@ void TargetSubscriberScheduler::ScheduleNext() { } auto event = BuildSubscriberTimerEvent(GetNextExecTime()); - mTimer->PushEvent(std::move(event)); + Timer::GetInstance()->PushEvent(std::move(event)); } void TargetSubscriberScheduler::Cancel() { @@ -281,9 +279,7 @@ void TargetSubscriberScheduler::SubscribeOnce(std::chrono::steady_clock::time_po }); mFuture = future; auto event = BuildSubscriberTimerEvent(execTime); - if (mTimer) { - mTimer->PushEvent(std::move(event)); - } + Timer::GetInstance()->PushEvent(std::move(event)); } std::unique_ptr diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp index 6087696990..f017279b60 100644 --- a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -33,6 +33,12 @@ class HostMonitorInputRunnerUnittest : public testing::Test { public: void TestUpdateAndRemoveCollector() const; void TestScheduleOnce() const; + +private: + void TearDown() override { + HostMonitorInputRunner::GetInstance()->Stop(); + Timer::GetInstance()->Clear(); + } }; void HostMonitorInputRunnerUnittest::TestUpdateAndRemoveCollector() const { @@ -67,8 +73,6 @@ void HostMonitorInputRunnerUnittest::TestScheduleOnce() const { APSARA_TEST_TRUE_FATAL(ProcessQueueManager::GetInstance()->PopItem(0, item, configName)); APSARA_TEST_EQUAL_FATAL("test", configName); - // verify schdule next - APSARA_TEST_EQUAL_FATAL(Timer::GetInstance()->mQueue.size(), 1); runner->mThreadPool.Stop(); runner->Stop(); } diff --git a/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp index 3bcadcb618..d9234228b7 100644 --- a/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp +++ b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp @@ -30,7 +30,7 @@ class ProcessEntityCollectorUnittest : public testing::Test { void ProcessEntityCollectorUnittest::TestGetNewProcessStat() const { PROCESS_DIR = "."; auto collector = ProcessEntityCollector(); - auto ptr = collector.GetNewProcessStat(1); + auto ptr = collector.ReadNewProcessStat(1); APSARA_TEST_NOT_EQUAL(nullptr, ptr); APSARA_TEST_EQUAL(1, ptr->pid); APSARA_TEST_EQUAL("cat", ptr->name); diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index 9b2f4f30e6..2be9643ba7 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -54,6 +54,8 @@ class ScrapeSchedulerUnittest : public testing::Test { mScrapeConfig->mRequestHeaders = {{"Authorization", "Bearer xxxxx"}}; } + void TearDown() override { Timer::GetInstance()->Clear(); } + private: std::shared_ptr mScrapeConfig; }; From 77f1384a77d865a8906126b7f87daa217079d7ae Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 27 Dec 2024 23:21:00 +0800 Subject: [PATCH 07/13] fix --- .../host_monitor/HostMonitorInputRunnerUnittest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp index f017279b60..a4508e64b1 100644 --- a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -44,11 +44,11 @@ class HostMonitorInputRunnerUnittest : public testing::Test { void HostMonitorInputRunnerUnittest::TestUpdateAndRemoveCollector() const { auto runner = HostMonitorInputRunner::GetInstance(); runner->Init(); - runner->UpdateCollector({"mock"}, QueueKey{}, 0); - APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid("test", "mock")); + runner->UpdateCollector({"process_entity"}, QueueKey{}, 0); + APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid("test", "process_entity")); APSARA_TEST_TRUE_FATAL(runner->HasRegisteredPlugins()); runner->RemoveCollector(); - APSARA_TEST_FALSE_FATAL(runner->IsCollectTaskValid("test", "mock")); + APSARA_TEST_FALSE_FATAL(runner->IsCollectTaskValid("test", "process_entity")); APSARA_TEST_FALSE_FATAL(runner->HasRegisteredPlugins()); runner->Stop(); } From 719fb39516f2f26ee98fcb6051c9120b83b65fce Mon Sep 17 00:00:00 2001 From: abingcbc Date: Sat, 28 Dec 2024 15:36:44 +0800 Subject: [PATCH 08/13] fix --- .../host_monitor/ProcessEntityCollectorUnittest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp index d9234228b7..13e52a2a70 100644 --- a/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp +++ b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp @@ -40,9 +40,9 @@ void ProcessEntityCollectorUnittest::TestSortProcessByCpu() const { PROCESS_DIR = "/proc"; auto collector = ProcessEntityCollector(); auto processes = vector(); - collector.GetSortedProcess(processes, 5); // fist time will be ignored - collector.GetSortedProcess(processes, 5); - APSARA_TEST_EQUAL(5, processes.size()); + collector.GetSortedProcess(processes, 3); // fist time will be ignored + collector.GetSortedProcess(processes, 3); + APSARA_TEST_EQUAL(3, processes.size()); auto prev = processes[0]; for (auto i = 1; i < processes.size(); i++) { auto process = processes[i]; From 84332c9ea9e5f74f74676acb56a6eec43927ec13 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Sun, 5 Jan 2025 22:03:02 +0800 Subject: [PATCH 09/13] fix --- core/common/timer/Timer.cpp | 16 +-- core/common/timer/Timer.h | 2 + core/constants/EntityConstants.cpp | 2 +- core/constants/EntityConstants.h | 2 +- core/host_monitor/HostMonitorInputRunner.cpp | 107 +++++++++--------- core/host_monitor/HostMonitorInputRunner.h | 23 ++-- core/host_monitor/HostMonitorTimerEvent.cpp | 8 +- core/host_monitor/HostMonitorTimerEvent.h | 1 - core/host_monitor/collector/BaseCollector.h | 5 - .../collector/ProcessEntityCollector.cpp | 48 ++++---- core/models/PipelineEventGroup.h | 2 +- core/plugin/input/InputHostMeta.cpp | 4 +- .../HostMonitorInputRunnerUnittest.cpp | 25 ++-- core/unittest/host_monitor/MockCollector.h | 42 +++++++ 14 files changed, 175 insertions(+), 112 deletions(-) create mode 100644 core/unittest/host_monitor/MockCollector.h diff --git a/core/common/timer/Timer.cpp b/core/common/timer/Timer.cpp index e33aa0aacc..b792885d5b 100644 --- a/core/common/timer/Timer.cpp +++ b/core/common/timer/Timer.cpp @@ -51,13 +51,6 @@ void Timer::Stop() { } } -void Timer::Clear() { - lock_guard lock(mQueueMux); - while (!mQueue.empty()) { - mQueue.pop(); - } -} - void Timer::PushEvent(unique_ptr&& e) { lock_guard lock(mQueueMux); if (mQueue.empty() || e->GetExecTime() < mQueue.top()->GetExecTime()) { @@ -101,4 +94,13 @@ void Timer::Run() { } } +#ifdef APSARA_UNIT_TEST_MAIN +void Timer::Clear() { + lock_guard lock(mQueueMux); + while (!mQueue.empty()) { + mQueue.pop(); + } +} +#endif + } // namespace logtail diff --git a/core/common/timer/Timer.h b/core/common/timer/Timer.h index efb1ec4f5e..9f6f59b283 100644 --- a/core/common/timer/Timer.h +++ b/core/common/timer/Timer.h @@ -46,7 +46,9 @@ class Timer { void Init(); void Stop(); void PushEvent(std::unique_ptr&& e); +#ifdef APSARA_UNIT_TEST_MAIN void Clear(); +#endif private: Timer() = default; diff --git a/core/constants/EntityConstants.cpp b/core/constants/EntityConstants.cpp index afbe698d63..16bd9a20cf 100644 --- a/core/constants/EntityConstants.cpp +++ b/core/constants/EntityConstants.cpp @@ -38,7 +38,7 @@ const std::string DEFAULT_CONTENT_KEY_PROCESS_PID = "pid"; const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID = "ppid"; const std::string DEFAULT_CONTENT_KEY_PROCESS_USER = "user"; const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM = "comm"; -const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME = "create_time"; +const std::string DEFAULT_CONTENT_KEY_PROCESS_KTIME = "ktime"; const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD = "cwd"; const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY = "binary"; const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS = "arguments"; diff --git a/core/constants/EntityConstants.h b/core/constants/EntityConstants.h index cb544db989..acb6c4cccc 100644 --- a/core/constants/EntityConstants.h +++ b/core/constants/EntityConstants.h @@ -39,7 +39,7 @@ extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PID; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_USER; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM; -extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME; +extern const std::string DEFAULT_CONTENT_KEY_PROCESS_KTIME; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS; diff --git a/core/host_monitor/HostMonitorInputRunner.cpp b/core/host_monitor/HostMonitorInputRunner.cpp index a852c3d67f..16fb5f4154 100755 --- a/core/host_monitor/HostMonitorInputRunner.cpp +++ b/core/host_monitor/HostMonitorInputRunner.cpp @@ -16,7 +16,6 @@ #include "HostMonitorInputRunner.h" -#include #include #include #include @@ -43,32 +42,27 @@ void HostMonitorInputRunner::UpdateCollector(const std::vector& new int inputIndex) { std::unique_lock lock(mRegisteredCollectorMapMutex); for (auto& collector : newCollectors) { - auto oldCollector = mRegisteredCollectorMap.find(collector); - if (oldCollector == mRegisteredCollectorMap.end()) { - if (mCollectorInstanceMap.find(collector) == mCollectorInstanceMap.end()) { - LOG_ERROR(sLogger, ("host monitor", "collector not found")("collector", collector)); - continue; - } - mRegisteredCollectorMap[collector] = true; - HostMonitorTimerEvent::CollectConfig collectConfig( - collector, processQueueKey, inputIndex, std::chrono::seconds(DEFAULT_SCHEDULE_INTERVAL)); - auto now = std::chrono::steady_clock::now(); - auto event = std::make_unique(now + collectConfig.mInterval, collectConfig); - Timer::GetInstance()->PushEvent(std::move(event)); - LOG_INFO(sLogger, ("host monitor", "add new collector")("collector", collector)); - } else { - // config removed and added again, timer event is still in the queue - if (!oldCollector->second) { - oldCollector->second = true; - } + auto iter = mRegisteredCollectorMap.find(collector); + if (iter == mRegisteredCollectorMap.end()) { + LOG_ERROR(sLogger, ("host monitor", "collector not support")("collector", collector)); + continue; } + // register new collector + iter->second.Enable(); + // add timer event + HostMonitorTimerEvent::CollectConfig collectConfig( + collector, processQueueKey, inputIndex, std::chrono::seconds(DEFAULT_SCHEDULE_INTERVAL)); + auto now = std::chrono::steady_clock::now(); + auto event = std::make_unique(now, collectConfig); + Timer::GetInstance()->PushEvent(std::move(event)); + LOG_INFO(sLogger, ("host monitor", "add new collector")("collector", collector)); } } void HostMonitorInputRunner::RemoveCollector() { std::unique_lock lock(mRegisteredCollectorMapMutex); for (auto& collector : mRegisteredCollectorMap) { - collector.second = false; + collector.second.Disable(); } } @@ -86,6 +80,7 @@ void HostMonitorInputRunner::Stop() { if (!mIsStarted.exchange(false)) { return; } + RemoveCollector(); #ifndef APSARA_UNIT_TEST_MAIN std::future result = std::async(std::launch::async, [this]() { mThreadPool.Stop(); }); if (result.wait_for(std::chrono::seconds(3)) == std::future_status::timeout) { @@ -98,63 +93,73 @@ void HostMonitorInputRunner::Stop() { bool HostMonitorInputRunner::HasRegisteredPlugins() const { std::shared_lock lock(mRegisteredCollectorMapMutex); for (auto& collector : mRegisteredCollectorMap) { - if (collector.second) { + if (collector.second.IsEnabled()) { return true; } } return false; } -bool HostMonitorInputRunner::IsCollectTaskValid(const std::string& configName, const std::string& collectorName) { - std::unique_lock lock(mRegisteredCollectorMapMutex); +bool HostMonitorInputRunner::IsCollectTaskValid(const std::string& collectorName) { + std::shared_lock lock(mRegisteredCollectorMapMutex); auto it = mRegisteredCollectorMap.find(collectorName); - if (it != mRegisteredCollectorMap.end() && it->second) { - return true; + if (it == mRegisteredCollectorMap.end()) { + return false; } - return false; + return it->second.IsEnabled(); } -void HostMonitorInputRunner::ScheduleOnce(HostMonitorTimerEvent::CollectConfig& config) { - auto collectFn = [this, config]() mutable { +void HostMonitorInputRunner::ScheduleOnce(std::chrono::steady_clock::time_point execTime, + HostMonitorTimerEvent::CollectConfig& config) { + auto collectFn = [this, config, execTime]() mutable { PipelineEventGroup group(std::make_shared()); - auto collector = GetCollector(config.mCollectorName); - if (collector) { - collector->Collect(group); - } else { - LOG_ERROR(sLogger, - ("collector not found, will not collect again", - "discard data")("config", config.mConfigName)("collector", config.mCollectorName)); + group.SetMetadata(EventGroupMetaKey::HOST_MONITOR_COLLECT_INTERVAL, std::to_string(config.mInterval.count())); + + std::unique_lock lock(mRegisteredCollectorMapMutex); + auto collector = mRegisteredCollectorMap.find(config.mCollectorName); + if (collector == mRegisteredCollectorMap.end()) { + LOG_ERROR( + sLogger, + ("collector not found, will not collect again", "discard data")("collector", config.mCollectorName)); return; } - - LOG_DEBUG(sLogger, ("host monitor collect", "collector")(config.mCollectorName, group.GetEvents().size())); + if (!collector->second.IsEnabled()) { + LOG_DEBUG(sLogger, + ("collector not enabled, may be caused by config update", "discard data")("collector", + config.mCollectorName)); + return; + } + collector->second.GetCollector()->Collect(group); + LOG_DEBUG( + sLogger, + ("host monitor", "collect data")("collector", config.mCollectorName)("size", group.GetEvents().size())); if (group.GetEvents().size() > 0) { + LOG_DEBUG(sLogger, ("host monitor", "push process queue")("collector", config.mProcessQueueKey)); bool result = ProcessorRunner::GetInstance()->PushQueue( config.mProcessQueueKey, config.mInputIndex, std::move(group)); if (!result) { // there is no process for host monitor, so should not block in process queue - LOG_ERROR(sLogger, - ("host monitor push process queue failed", - "discard data")("config", config.mConfigName)("collector", config.mCollectorName)); + LOG_ERROR( + sLogger, + ("host monitor push process queue failed", "discard data")("collector", config.mCollectorName)); } } + + // add next collect event + std::chrono::steady_clock::time_point nextExecTime = execTime + config.mInterval; + while (nextExecTime < std::chrono::steady_clock::now()) { + nextExecTime += config.mInterval; + } + auto event = std::make_unique(nextExecTime, config); + Timer::GetInstance()->PushEvent(std::move(event)); }; mThreadPool.Add(collectFn); } - -std::shared_ptr HostMonitorInputRunner::GetCollector(const std::string& collectorName) { - auto it = mCollectorInstanceMap.find(collectorName); - if (it == mCollectorInstanceMap.end()) { - return nullptr; - } - return it->second; -} - template void HostMonitorInputRunner::RegisterCollector() { - auto collector = std::make_shared(); - mCollectorInstanceMap[collector->Name()] = collector; + auto collector = std::make_unique(); + mRegisteredCollectorMap.emplace(T::sName, CollectorInstance(std::move(collector))); } } // namespace logtail diff --git a/core/host_monitor/HostMonitorInputRunner.h b/core/host_monitor/HostMonitorInputRunner.h index 461316acd0..a9f5fe6d1f 100644 --- a/core/host_monitor/HostMonitorInputRunner.h +++ b/core/host_monitor/HostMonitorInputRunner.h @@ -28,7 +28,6 @@ #include "common/ThreadPool.h" #include "host_monitor/HostMonitorTimerEvent.h" #include "host_monitor/collector/BaseCollector.h" -#include "host_monitor/collector/ProcessEntityCollector.h" #include "pipeline/queue/QueueKey.h" #include "runner/InputRunner.h" @@ -36,6 +35,18 @@ namespace logtail { const int DEFAULT_SCHEDULE_INTERVAL = 10; +struct CollectorInstance { + CollectorInstance(std::unique_ptr&& collector) : mCollector(std::move(collector)) {} + + std::unique_ptr mCollector; + bool mIsEnabled = false; + + BaseCollector* GetCollector() const { return mCollector.get(); } + bool IsEnabled() const { return mIsEnabled; } + void Enable() { mIsEnabled = true; } + void Disable() { mIsEnabled = false; } +}; + class HostMonitorInputRunner : public InputRunner { public: HostMonitorInputRunner(const HostMonitorInputRunner&) = delete; @@ -56,23 +67,21 @@ class HostMonitorInputRunner : public InputRunner { void Stop() override; bool HasRegisteredPlugins() const override; - bool IsCollectTaskValid(const std::string& configName, const std::string& collectorName); - void ScheduleOnce(HostMonitorTimerEvent::CollectConfig& collectConfig); + bool IsCollectTaskValid(const std::string& collectorName); + void ScheduleOnce(std::chrono::steady_clock::time_point execTime, + HostMonitorTimerEvent::CollectConfig& collectConfig); private: HostMonitorInputRunner(); template void RegisterCollector(); - std::shared_ptr GetCollector(const std::string& collectorName); std::atomic_bool mIsStarted = false; - ThreadPool mThreadPool; mutable std::shared_mutex mRegisteredCollectorMapMutex; - std::unordered_map mRegisteredCollectorMap; - std::unordered_map> mCollectorInstanceMap; + std::unordered_map mRegisteredCollectorMap; #ifdef APSARA_UNIT_TEST_MAIN friend class HostMonitorInputRunnerUnittest; diff --git a/core/host_monitor/HostMonitorTimerEvent.cpp b/core/host_monitor/HostMonitorTimerEvent.cpp index 150801afd4..62c46259df 100755 --- a/core/host_monitor/HostMonitorTimerEvent.cpp +++ b/core/host_monitor/HostMonitorTimerEvent.cpp @@ -22,15 +22,11 @@ namespace logtail { bool HostMonitorTimerEvent::IsValid() const { - return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(mCollectConfig.mConfigName, - mCollectConfig.mCollectorName); + return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(mCollectConfig.mCollectorName); } bool HostMonitorTimerEvent::Execute() { - LOG_DEBUG(sLogger, ("schedule host monitor collector", mCollectConfig.mConfigName)); - HostMonitorInputRunner::GetInstance()->ScheduleOnce(mCollectConfig); - auto event = std::make_unique(GetExecTime() + mCollectConfig.mInterval, mCollectConfig); - Timer::GetInstance()->PushEvent(std::move(event)); + HostMonitorInputRunner::GetInstance()->ScheduleOnce(GetExecTime(), mCollectConfig); return true; } diff --git a/core/host_monitor/HostMonitorTimerEvent.h b/core/host_monitor/HostMonitorTimerEvent.h index db2794ea10..a4d9465c53 100644 --- a/core/host_monitor/HostMonitorTimerEvent.h +++ b/core/host_monitor/HostMonitorTimerEvent.h @@ -39,7 +39,6 @@ class HostMonitorTimerEvent : public TimerEvent { mInputIndex(inputIndex), mInterval(interval) {} - std::string mConfigName; std::string mCollectorName; QueueKey mProcessQueueKey; size_t mInputIndex; diff --git a/core/host_monitor/collector/BaseCollector.h b/core/host_monitor/collector/BaseCollector.h index 8db6bf8003..8716307708 100755 --- a/core/host_monitor/collector/BaseCollector.h +++ b/core/host_monitor/collector/BaseCollector.h @@ -25,13 +25,8 @@ class BaseCollector { BaseCollector() = default; virtual ~BaseCollector() = default; - bool IsValid() const { return mValidState; } virtual void Collect(PipelineEventGroup& group) = 0; virtual const std::string& Name() const = 0; - -protected: - const std::string mName; - bool mValidState = false; }; } // namespace logtail diff --git a/core/host_monitor/collector/ProcessEntityCollector.cpp b/core/host_monitor/collector/ProcessEntityCollector.cpp index 3eb1e4eaca..a519722ee6 100755 --- a/core/host_monitor/collector/ProcessEntityCollector.cpp +++ b/core/host_monitor/collector/ProcessEntityCollector.cpp @@ -29,6 +29,7 @@ #include #include +#include "BaseCollector.h" #include "Common.h" #include "FileSystemUtil.h" #include "Logger.h" @@ -55,21 +56,10 @@ ProcessEntityCollector::ProcessEntityCollector() : mProcessSilentCount(INT32_FLA // TODO: fix after host id ready mHostEntityID = ""; // FetchHostId(); mDomain = ""; - std::ostringstream oss; - if (mDomain == DEFAULT_HOST_TYPE_ECS) { - oss << DEFAULT_CONTENT_VALUE_DOMAIN_ACS << "." << DEFAULT_HOST_TYPE_ECS << "." - << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; - mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_ACS; - } else { - oss << DEFAULT_CONTENT_VALUE_DOMAIN_INFRA << "." << DEFAULT_HOST_TYPE_HOST << "." - << DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; - mDomain = DEFAULT_CONTENT_VALUE_DOMAIN_INFRA; - } - mEntityType = oss.str(); + mEntityType = ""; }; void ProcessEntityCollector::Collect(PipelineEventGroup& group) { - group.SetMetadata(EventGroupMetaKey::COLLECT_TIME, std::to_string(time(nullptr))); std::vector processes; GetSortedProcess(processes, ProcessTopN); for (auto process : processes) { @@ -87,13 +77,23 @@ void ProcessEntityCollector::Collect(PipelineEventGroup& group) { event->SetContent(DEFAULT_CONTENT_KEY_DOMAIN, mDomain); event->SetContent(DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME, processCreateTime); event->SetContent(DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME, std::to_string(logtime)); - event->SetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS, "30"); + auto interval = group.GetMetadata(EventGroupMetaKey::HOST_MONITOR_COLLECT_INTERVAL); + if (!interval.empty()) { + try { + auto keepAliveSeconds = std::stoi(interval.data()) * 2; + event->SetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS, std::to_string(keepAliveSeconds)); + } catch (const std::invalid_argument& e) { + LOG_ERROR(sLogger, ("process entity collector", "invalid interval")("interval", interval)); + } catch (const std::out_of_range& e) { + LOG_ERROR(sLogger, ("process entity collector", "interval out of range")("interval", interval)); + } + } // custom fields event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PID, std::to_string(process->pid)); event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PPID, std::to_string(process->parentPid)); event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_COMM, process->name); - event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME, processCreateTime); + event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_KTIME, processCreateTime); // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_USER, ""); TODO: get user name // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CWD, ""); TODO: get cwd // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_BINARY, ""); TODO: get binary @@ -153,11 +153,16 @@ ProcessStatPtr ProcessEntityCollector::GetProcessStat(pid_t pid, bool& isFirstCo const auto now = steady_clock::now(); // TODO: more accurate cache - auto prev = mPrevProcessStat[pid]; - isFirstCollect = prev == nullptr || prev->lastTime.time_since_epoch().count() == 0; + auto prev = mPrevProcessStat.find(pid); + if (prev == mPrevProcessStat.end() || prev->second == nullptr + || prev->second->lastTime.time_since_epoch().count() == 0) { + isFirstCollect = true; + } else { + isFirstCollect = false; + } // proc/[pid]/stat的统计粒度通常为10ms，两次采样之间需要足够大才能平滑。 - if (prev && now < prev->lastTime + seconds{1}) { - return prev; + if (prev != mPrevProcessStat.end() && prev->second && now < prev->second->lastTime + seconds{1}) { + return prev->second; } auto ptr = ReadNewProcessStat(pid); if (!ptr) { @@ -170,18 +175,17 @@ ProcessStatPtr ProcessEntityCollector::GetProcessStat(pid_t pid, bool& isFirstCo ptr->cpuInfo.user = ptr->utime.count(); ptr->cpuInfo.sys = ptr->stime.count(); ptr->cpuInfo.total = ptr->cpuInfo.user + ptr->cpuInfo.sys; - if (isFirstCollect || ptr->cpuInfo.total <= prev->cpuInfo.total) { + if (isFirstCollect || ptr->cpuInfo.total <= prev->second->cpuInfo.total) { // first time called ptr->cpuInfo.percent = 0.0; } else { - auto totalDiff = static_cast(ptr->cpuInfo.total - prev->cpuInfo.total); + auto totalDiff = static_cast(ptr->cpuInfo.total - prev->second->cpuInfo.total); auto timeDiff = static_cast(ptr->lastTime.time_since_epoch().count() - - prev->lastTime.time_since_epoch().count()); + - prev->second->lastTime.time_since_epoch().count()); ptr->cpuInfo.percent = totalDiff / timeDiff * 100; } } - prev = ptr; mPrevProcessStat[pid] = ptr; return ptr; } diff --git a/core/models/PipelineEventGroup.h b/core/models/PipelineEventGroup.h index f1339c45c3..47b569b4ac 100644 --- a/core/models/PipelineEventGroup.h +++ b/core/models/PipelineEventGroup.h @@ -59,7 +59,7 @@ enum class EventGroupMetaKey { PROMETHEUS_SCRAPE_TIMESTAMP_MILLISEC, PROMETHEUS_UP_STATE, - COLLECT_TIME, + HOST_MONITOR_COLLECT_INTERVAL, SOURCE_ID }; diff --git a/core/plugin/input/InputHostMeta.cpp b/core/plugin/input/InputHostMeta.cpp index 0a5c06117a..d9b32e1182 100644 --- a/core/plugin/input/InputHostMeta.cpp +++ b/core/plugin/input/InputHostMeta.cpp @@ -36,9 +36,7 @@ bool InputHostMeta::Start() { bool InputHostMeta::Stop(bool isPipelineRemoving) { LOG_INFO(sLogger, ("input host meta stop", mContext->GetConfigName())); - if (isPipelineRemoving) { - HostMonitorInputRunner::GetInstance()->RemoveCollector(); - } + HostMonitorInputRunner::GetInstance()->RemoveCollector(); return true; } diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp index a4508e64b1..b9ff4eb7a8 100644 --- a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -24,6 +24,7 @@ #include "QueueKeyManager.h" #include "common/timer/Timer.h" #include "unittest/Unittest.h" +#include "unittest/host_monitor/MockCollector.h" using namespace std; @@ -35,6 +36,12 @@ class HostMonitorInputRunnerUnittest : public testing::Test { void TestScheduleOnce() const; private: + static void SetUpTestCase() { + auto collector = std::make_unique(); + HostMonitorInputRunner::GetInstance()->mRegisteredCollectorMap.emplace(MockCollector::sName, + CollectorInstance(std::move(collector))); + } + void TearDown() override { HostMonitorInputRunner::GetInstance()->Stop(); Timer::GetInstance()->Clear(); @@ -44,11 +51,12 @@ class HostMonitorInputRunnerUnittest : public testing::Test { void HostMonitorInputRunnerUnittest::TestUpdateAndRemoveCollector() const { auto runner = HostMonitorInputRunner::GetInstance(); runner->Init(); - runner->UpdateCollector({"process_entity"}, QueueKey{}, 0); - APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid("test", "process_entity")); + runner->UpdateCollector({MockCollector::sName}, QueueKey{}, 0); + APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid(MockCollector::sName)); APSARA_TEST_TRUE_FATAL(runner->HasRegisteredPlugins()); + APSARA_TEST_EQUAL_FATAL(1, Timer::GetInstance()->mQueue.size()); runner->RemoveCollector(); - APSARA_TEST_FALSE_FATAL(runner->IsCollectTaskValid("test", "process_entity")); + APSARA_TEST_FALSE_FATAL(runner->IsCollectTaskValid(MockCollector::sName)); APSARA_TEST_FALSE_FATAL(runner->HasRegisteredPlugins()); runner->Stop(); } @@ -57,17 +65,20 @@ void HostMonitorInputRunnerUnittest::TestScheduleOnce() const { auto runner = HostMonitorInputRunner::GetInstance(); runner->Init(); runner->mThreadPool.Start(); + runner->mRegisteredCollectorMap.find(MockCollector::sName)->second.Enable(); std::string configName = "test"; auto queueKey = QueueKeyManager::GetInstance()->GetKey(configName); auto ctx = PipelineContext(); ctx.SetConfigName(configName); ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(queueKey, 0, ctx); - HostMonitorTimerEvent::CollectConfig collectConfig("process_entity", queueKey, 0, std::chrono::seconds(60)); - runner->ScheduleOnce(collectConfig); - std::this_thread::sleep_for(std::chrono::seconds(1)); - runner->ScheduleOnce(collectConfig); + HostMonitorTimerEvent::CollectConfig collectConfig(MockCollector::sName, queueKey, 0, std::chrono::seconds(60)); + std::chrono::time_point now = std::chrono::steady_clock::now(); + runner->ScheduleOnce(now, collectConfig); std::this_thread::sleep_for(std::chrono::seconds(1)); + APSARA_TEST_EQUAL_FATAL(1, Timer::GetInstance()->mQueue.size()); + APSARA_TEST_EQUAL_FATAL((now + std::chrono::seconds(60)).time_since_epoch().count(), + Timer::GetInstance()->mQueue.top()->GetExecTime().time_since_epoch().count()); auto item = std::unique_ptr(new ProcessQueueItem(std::make_shared(), 0)); ProcessQueueManager::GetInstance()->EnablePop(configName); APSARA_TEST_TRUE_FATAL(ProcessQueueManager::GetInstance()->PopItem(0, item, configName)); diff --git a/core/unittest/host_monitor/MockCollector.h b/core/unittest/host_monitor/MockCollector.h new file mode 100644 index 0000000000..b10a5b4f4e --- /dev/null +++ b/core/unittest/host_monitor/MockCollector.h @@ -0,0 +1,42 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "host_monitor/collector/BaseCollector.h" + +namespace logtail { + +class MockCollector : public BaseCollector { +public: + MockCollector() = default; + ~MockCollector() = default; + + void Collect(PipelineEventGroup& group) { + auto event = group.AddLogEvent(); + time_t logtime = time(nullptr); + event->SetTimestamp(logtime); + std::string key = "mock_key"; + std::string value = "mock_value"; + event->SetContent(key, value); + } + static const std::string sName; + const std::string& Name() const { return sName; } +}; + +const std::string MockCollector::sName = "mock"; + +} // namespace logtail \ No newline at end of file From f2e18852bea8efd2eb1fbd8a099fa532ed4f3dde Mon Sep 17 00:00:00 2001 From: abingcbc Date: Mon, 6 Jan 2025 09:59:48 +0800 Subject: [PATCH 10/13] fix --- core/common/FileSystemUtil.h | 5 ++-- core/host_monitor/collector/BaseCollector.h | 3 +++ .../collector/ProcessEntityCollector.cpp | 14 ++++++----- core/prometheus/PrometheusInputRunner.cpp | 3 --- core/prometheus/PrometheusInputRunner.h | 2 -- .../prometheus/schedulers/ScrapeScheduler.cpp | 3 +-- core/prometheus/schedulers/ScrapeScheduler.h | 2 +- .../schedulers/TargetSubscriberScheduler.cpp | 24 +++++++++---------- 8 files changed, 26 insertions(+), 30 deletions(-) diff --git a/core/common/FileSystemUtil.h b/core/common/FileSystemUtil.h index f3fb744980..0adab15636 100644 --- a/core/common/FileSystemUtil.h +++ b/core/common/FileSystemUtil.h @@ -26,6 +26,7 @@ #include #endif #include +#include #include "DevInode.h" #include "ErrorUtil.h" @@ -264,9 +265,7 @@ class PathStat { int64_t GetFileSize() const; // GetMode returns st_mode. - int GetMode() const { - return static_cast(mRawStat.st_mode); - } + int GetMode() const { return static_cast(mRawStat.st_mode); } }; } // namespace fsutil diff --git a/core/host_monitor/collector/BaseCollector.h b/core/host_monitor/collector/BaseCollector.h index 8716307708..4c9115ea8e 100755 --- a/core/host_monitor/collector/BaseCollector.h +++ b/core/host_monitor/collector/BaseCollector.h @@ -27,6 +27,9 @@ class BaseCollector { virtual void Collect(PipelineEventGroup& group) = 0; virtual const std::string& Name() const = 0; + +protected: + bool mValidState = false; }; } // namespace logtail diff --git a/core/host_monitor/collector/ProcessEntityCollector.cpp b/core/host_monitor/collector/ProcessEntityCollector.cpp index a519722ee6..2949bc44fb 100755 --- a/core/host_monitor/collector/ProcessEntityCollector.cpp +++ b/core/host_monitor/collector/ProcessEntityCollector.cpp @@ -16,12 +16,12 @@ #include "ProcessEntityCollector.h" -#include -#include - #include #include #include +#include +#include + #include #include #include @@ -29,12 +29,11 @@ #include #include -#include "BaseCollector.h" -#include "Common.h" #include "FileSystemUtil.h" #include "Logger.h" #include "PipelineEventGroup.h" #include "StringTools.h" +#include "common/HashUtil.h" #include "constants/EntityConstants.h" #include "host_monitor/SystemInformationTools.h" @@ -60,6 +59,9 @@ ProcessEntityCollector::ProcessEntityCollector() : mProcessSilentCount(INT32_FLA }; void ProcessEntityCollector::Collect(PipelineEventGroup& group) { + if (!mValidState) { + return; + } std::vector processes; GetSortedProcess(processes, ProcessTopN); for (auto process : processes) { @@ -273,7 +275,7 @@ bool ProcessEntityCollector::WalkAllProcess(const std::filesystem::path& root, const std::string ProcessEntityCollector::GetProcessEntityID(StringView pid, StringView createTime) { std::ostringstream oss; oss << mHostEntityID << pid << createTime; - auto bigID = sdk::CalcMD5(oss.str()); + auto bigID = CalcMD5(oss.str()); std::transform(bigID.begin(), bigID.end(), bigID.begin(), ::tolower); return bigID; } diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index bf02f9600d..4018e3284c 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -50,9 +50,6 @@ PrometheusInputRunner::PrometheusInputRunner() mPodName(STRING_FLAG(_pod_name_)), mEventPool(true), mUnRegisterMs(0) { - mClient = std::make_unique(); - mTimer = std::make_shared(); - // self monitor MetricLabels labels; labels.emplace_back(METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS); diff --git a/core/prometheus/PrometheusInputRunner.h b/core/prometheus/PrometheusInputRunner.h index 0f7c237ed3..4519b37e39 100644 --- a/core/prometheus/PrometheusInputRunner.h +++ b/core/prometheus/PrometheusInputRunner.h @@ -74,8 +74,6 @@ class PrometheusInputRunner : public InputRunner { int32_t mServicePort; std::string mPodName; - std::unique_ptr mClient; - std::shared_ptr mTimer; EventPool mEventPool; mutable ReadWriteLock mSubscriberMapRWLock; diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index 79b5e3d3a3..195c5fd63f 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -107,8 +107,7 @@ string ScrapeScheduler::GetId() const { return mHash; } -void ScrapeScheduler::SetComponent(shared_ptr timer, EventPool* eventPool) { - mTimer = std::move(timer); +void ScrapeScheduler::SetComponent(EventPool* eventPool) { mEventPool = eventPool; mPromStreamScraper.mEventPool = mEventPool; } diff --git a/core/prometheus/schedulers/ScrapeScheduler.h b/core/prometheus/schedulers/ScrapeScheduler.h index 551ada0da9..697e15ec31 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.h +++ b/core/prometheus/schedulers/ScrapeScheduler.h @@ -48,7 +48,7 @@ class ScrapeScheduler : public BaseScheduler { std::string GetId() const; - void SetComponent(std::shared_ptr timer, EventPool* eventPool); + void SetComponent(EventPool* eventPool); void ScheduleNext() override; void ScrapeOnce(std::chrono::steady_clock::time_point execTime); diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index b671d3f9cd..f1eeffff6e 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -111,19 +111,17 @@ void TargetSubscriberScheduler::UpdateScrapeScheduler( auto tmpRandSleepMilliSec = GetRandSleepMilliSec( v->GetId(), mScrapeConfigPtr->mScrapeIntervalSeconds, tmpCurrentMilliSeconds); - // zero-cost upgrade - if (mUnRegisterMs > 0 - && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - - (uint64_t)mScrapeConfigPtr->mScrapeIntervalSeconds * 1000 - > mUnRegisterMs) - && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - - (uint64_t)mScrapeConfigPtr->mScrapeIntervalSeconds * 1000 * 2 - < mUnRegisterMs)) { - // scrape once just now - LOG_INFO(sLogger, ("scrape zero cost", ToString(tmpCurrentMilliSeconds))); - v->SetScrapeOnceTime(chrono::steady_clock::now(), chrono::system_clock::now()); - } - v->ScheduleNext(); + // zero-cost upgrade + if (mUnRegisterMs > 0 + && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec + - (uint64_t)mScrapeConfigPtr->mScrapeIntervalSeconds * 1000 + > mUnRegisterMs) + && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec + - (uint64_t)mScrapeConfigPtr->mScrapeIntervalSeconds * 1000 * 2 + < mUnRegisterMs)) { + // scrape once just now + LOG_INFO(sLogger, ("scrape zero cost", ToString(tmpCurrentMilliSeconds))); + v->SetScrapeOnceTime(chrono::steady_clock::now(), chrono::system_clock::now()); } v->ScheduleNext(); } From 3b27c1beeb31955b8fc89e69cda95f552e88f72d Mon Sep 17 00:00:00 2001 From: abingcbc Date: Mon, 6 Jan 2025 11:45:09 +0800 Subject: [PATCH 11/13] lint --- core/common/FileSystemUtil.cpp | 3 ++- core/common/FileSystemUtil.h | 4 +++- core/host_monitor/HostMonitorTimerEvent.h | 1 + core/host_monitor/SystemInformationTools.h | 1 + core/host_monitor/collector/ProcessEntityCollector.h | 4 ++-- core/pipeline/PipelineManager.cpp | 3 +-- core/plugin/input/InputHostMeta.cpp | 3 ++- core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp | 1 + 8 files changed, 13 insertions(+), 7 deletions(-) mode change 100755 => 100644 core/host_monitor/collector/ProcessEntityCollector.h diff --git a/core/common/FileSystemUtil.cpp b/core/common/FileSystemUtil.cpp index bef4b8aa4a..8096916354 100644 --- a/core/common/FileSystemUtil.cpp +++ b/core/common/FileSystemUtil.cpp @@ -183,7 +183,8 @@ int GetFileLines(const std::filesystem::path& filename, std::vector& res, bool enableEmptyLine, std::string* errorMessage) { - return GetLines(filename, enableEmptyLine, [&res](const std::string& s) { res.push_back(s); }, errorMessage); + return GetLines( + filename, enableEmptyLine, [&res](const std::string& s) { res.push_back(s); }, errorMessage); } bool OverwriteFile(const std::string& fileName, const std::string& content) { diff --git a/core/common/FileSystemUtil.h b/core/common/FileSystemUtil.h index 0adab15636..f199733c5f 100644 --- a/core/common/FileSystemUtil.h +++ b/core/common/FileSystemUtil.h @@ -265,7 +265,9 @@ class PathStat { int64_t GetFileSize() const; // GetMode returns st_mode. - int GetMode() const { return static_cast(mRawStat.st_mode); } + int GetMode() const { + return static_cast(mRawStat.st_mode); + } }; } // namespace fsutil diff --git a/core/host_monitor/HostMonitorTimerEvent.h b/core/host_monitor/HostMonitorTimerEvent.h index a4d9465c53..3eefb3e520 100644 --- a/core/host_monitor/HostMonitorTimerEvent.h +++ b/core/host_monitor/HostMonitorTimerEvent.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include diff --git a/core/host_monitor/SystemInformationTools.h b/core/host_monitor/SystemInformationTools.h index c0e1e5189b..054d87f7e0 100644 --- a/core/host_monitor/SystemInformationTools.h +++ b/core/host_monitor/SystemInformationTools.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include diff --git a/core/host_monitor/collector/ProcessEntityCollector.h b/core/host_monitor/collector/ProcessEntityCollector.h old mode 100755 new mode 100644 index d189a694ed..cc001d5028 --- a/core/host_monitor/collector/ProcessEntityCollector.h +++ b/core/host_monitor/collector/ProcessEntityCollector.h @@ -16,12 +16,12 @@ #pragma once +#include +#include #include #include #include -#include -#include #include #include #include diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 69f2941860..85eafad39b 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -41,8 +41,7 @@ PipelineManager::PipelineManager() : mInputRunners({ PrometheusInputRunner::GetInstance(), #if defined(__linux__) && !defined(__ANDROID__) - ebpf::eBPFServer::GetInstance(), - HostMonitorInputRunner::GetInstance(), + ebpf::eBPFServer::GetInstance(), HostMonitorInputRunner::GetInstance(), #endif }) { } diff --git a/core/plugin/input/InputHostMeta.cpp b/core/plugin/input/InputHostMeta.cpp index d9b32e1182..3b5ff099fb 100644 --- a/core/plugin/input/InputHostMeta.cpp +++ b/core/plugin/input/InputHostMeta.cpp @@ -14,10 +14,11 @@ #include "InputHostMeta.h" +#include "json/value.h" + #include "HostMonitorInputRunner.h" #include "Logger.h" #include "constants/EntityConstants.h" -#include "json/value.h" namespace logtail { diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp index b9ff4eb7a8..88504f9d5d 100644 --- a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include + #include #include "HostMonitorInputRunner.h" From 2cb4b7b288e3e1e1e69777e15cf8f427753e437c Mon Sep 17 00:00:00 2001 From: abingcbc Date: Mon, 6 Jan 2025 15:29:53 +0800 Subject: [PATCH 12/13] fix --- core/host_monitor/HostMonitorInputRunner.cpp | 7 +++++-- core/host_monitor/HostMonitorInputRunner.h | 10 ++++++++-- core/host_monitor/HostMonitorTimerEvent.cpp | 2 +- .../host_monitor/HostMonitorInputRunnerUnittest.cpp | 6 ++++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/host_monitor/HostMonitorInputRunner.cpp b/core/host_monitor/HostMonitorInputRunner.cpp index 16fb5f4154..dcea6809ef 100755 --- a/core/host_monitor/HostMonitorInputRunner.cpp +++ b/core/host_monitor/HostMonitorInputRunner.cpp @@ -16,6 +16,8 @@ #include "HostMonitorInputRunner.h" +#include + #include #include #include @@ -100,13 +102,14 @@ bool HostMonitorInputRunner::HasRegisteredPlugins() const { return false; } -bool HostMonitorInputRunner::IsCollectTaskValid(const std::string& collectorName) { +bool HostMonitorInputRunner::IsCollectTaskValid(std::chrono::steady_clock::time_point execTime, + const std::string& collectorName) { std::shared_lock lock(mRegisteredCollectorMapMutex); auto it = mRegisteredCollectorMap.find(collectorName); if (it == mRegisteredCollectorMap.end()) { return false; } - return it->second.IsEnabled(); + return it->second.IsEnabled() && (it->second.mLastEnableTime <= execTime); } void HostMonitorInputRunner::ScheduleOnce(std::chrono::steady_clock::time_point execTime, diff --git a/core/host_monitor/HostMonitorInputRunner.h b/core/host_monitor/HostMonitorInputRunner.h index a9f5fe6d1f..20cffb73f3 100644 --- a/core/host_monitor/HostMonitorInputRunner.h +++ b/core/host_monitor/HostMonitorInputRunner.h @@ -16,6 +16,8 @@ #pragma once +#include + #include #include #include @@ -40,10 +42,14 @@ struct CollectorInstance { std::unique_ptr mCollector; bool mIsEnabled = false; + std::chrono::steady_clock::time_point mLastEnableTime; BaseCollector* GetCollector() const { return mCollector.get(); } bool IsEnabled() const { return mIsEnabled; } - void Enable() { mIsEnabled = true; } + void Enable() { + mIsEnabled = true; + mLastEnableTime = std::chrono::steady_clock::now(); + } void Disable() { mIsEnabled = false; } }; @@ -67,7 +73,7 @@ class HostMonitorInputRunner : public InputRunner { void Stop() override; bool HasRegisteredPlugins() const override; - bool IsCollectTaskValid(const std::string& collectorName); + bool IsCollectTaskValid(std::chrono::steady_clock::time_point execTime, const std::string& collectorName); void ScheduleOnce(std::chrono::steady_clock::time_point execTime, HostMonitorTimerEvent::CollectConfig& collectConfig); diff --git a/core/host_monitor/HostMonitorTimerEvent.cpp b/core/host_monitor/HostMonitorTimerEvent.cpp index 62c46259df..944925a619 100755 --- a/core/host_monitor/HostMonitorTimerEvent.cpp +++ b/core/host_monitor/HostMonitorTimerEvent.cpp @@ -22,7 +22,7 @@ namespace logtail { bool HostMonitorTimerEvent::IsValid() const { - return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(mCollectConfig.mCollectorName); + return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(GetExecTime(), mCollectConfig.mCollectorName); } bool HostMonitorTimerEvent::Execute() { diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp index 88504f9d5d..795521d679 100644 --- a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -53,11 +53,13 @@ void HostMonitorInputRunnerUnittest::TestUpdateAndRemoveCollector() const { auto runner = HostMonitorInputRunner::GetInstance(); runner->Init(); runner->UpdateCollector({MockCollector::sName}, QueueKey{}, 0); - APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid(MockCollector::sName)); + APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid(std::chrono::steady_clock::now(), MockCollector::sName)); + APSARA_TEST_FALSE_FATAL( + runner->IsCollectTaskValid(std::chrono::steady_clock::now() - std::chrono::seconds(60), MockCollector::sName)); APSARA_TEST_TRUE_FATAL(runner->HasRegisteredPlugins()); APSARA_TEST_EQUAL_FATAL(1, Timer::GetInstance()->mQueue.size()); runner->RemoveCollector(); - APSARA_TEST_FALSE_FATAL(runner->IsCollectTaskValid(MockCollector::sName)); + APSARA_TEST_FALSE_FATAL(runner->IsCollectTaskValid(std::chrono::steady_clock::now(), MockCollector::sName)); APSARA_TEST_FALSE_FATAL(runner->HasRegisteredPlugins()); runner->Stop(); } From c4614a8636cf46fc22eec205f5c7eb5f58873c93 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Mon, 6 Jan 2025 19:32:33 +0800 Subject: [PATCH 13/13] fix --- core/app_config/AppConfig.cpp | 1 + core/common/MachineInfoUtil.cpp | 2 +- core/common/MachineInfoUtil.h | 7 +- core/constants/EntityConstants.cpp | 17 +++-- core/constants/EntityConstants.h | 14 +++- core/host_monitor/HostMonitorInputRunner.cpp | 15 ++-- core/host_monitor/HostMonitorInputRunner.h | 7 +- core/host_monitor/HostMonitorTimerEvent.cpp | 1 - .../collector/ProcessEntityCollector.cpp | 68 ++++++++++++++----- .../collector/ProcessEntityCollector.h | 10 +-- core/plugin/input/InputHostMeta.cpp | 16 ++++- core/plugin/input/InputHostMeta.h | 6 ++ .../HostMonitorInputRunnerUnittest.cpp | 2 +- .../ProcessEntityCollectorUnittest.cpp | 3 +- .../plugins/input/native/input-host-meta.md | 43 ++++++++++++ 15 files changed, 169 insertions(+), 43 deletions(-) create mode 100644 docs/cn/plugins/input/native/input-host-meta.md diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index 0bc9d1c322..5350250456 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -165,6 +165,7 @@ DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator s DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888); DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000); DEFINE_FLAG_STRING(_pod_name_, "agent pod name", ""); +DEFINE_FLAG_INT32(host_monitor_default_interval, "default interval for host monitor", 60); DEFINE_FLAG_INT32(process_collect_silent_count, "number of process scanned between a sleep", 1000); DEFINE_FLAG_STRING(app_info_file, "", "app_info.json"); diff --git a/core/common/MachineInfoUtil.cpp b/core/common/MachineInfoUtil.cpp index 0efa6dcee0..5c233493f6 100644 --- a/core/common/MachineInfoUtil.cpp +++ b/core/common/MachineInfoUtil.cpp @@ -504,13 +504,13 @@ size_t FetchECSMetaCallback(char* buffer, size_t size, size_t nmemb, std::string HostIdentifier::HostIdentifier() { #ifdef __ENTERPRISE__ getECSMetaFromFile(); - updateHostId(); #else ECSMeta ecsMeta; if (FetchECSMeta(ecsMeta)) { UpdateECSMetaAndHostid(ecsMeta); } #endif + updateHostId(); } bool HostIdentifier::UpdateECSMetaAndHostid(const ECSMeta& meta) { diff --git a/core/common/MachineInfoUtil.h b/core/common/MachineInfoUtil.h index 4264360f44..9cd0656f2f 100644 --- a/core/common/MachineInfoUtil.h +++ b/core/common/MachineInfoUtil.h @@ -81,10 +81,15 @@ class HostIdentifier { } bool UpdateECSMetaAndHostid(const ECSMeta& meta); - bool FetchECSMeta(ECSMeta& metaObj); void DumpECSMeta(); +#ifdef __ENTERPRISE__ + bool FetchECSMeta(ECSMeta& metaObj); +#endif private: +#ifndef __ENTERPRISE__ + bool FetchECSMeta(ECSMeta& metaObj); +#endif void getECSMetaFromFile(); // 从云助手获取序列号 void getSerialNumberFromEcsAssist(); diff --git a/core/constants/EntityConstants.cpp b/core/constants/EntityConstants.cpp index 16bd9a20cf..fe0d364732 100644 --- a/core/constants/EntityConstants.cpp +++ b/core/constants/EntityConstants.cpp @@ -18,13 +18,13 @@ namespace logtail { -const std::string DEFAULT_HOST_TYPE_ECS = "ecs"; -const std::string DEFAULT_HOST_TYPE_HOST = "host"; const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE = "__entity_type__"; const std::string DEFAULT_CONTENT_KEY_ENTITY_ID = "__entity_id__"; const std::string DEFAULT_CONTENT_KEY_DOMAIN = "__domain__"; const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS = "acs"; const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA = "infra"; +const std::string DEFAULT_HOST_TYPE_ECS = "acs.ecs.instance"; +const std::string DEFAULT_HOST_TYPE_HOST = "acs.host.instance"; const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME = "__first_observed_time__"; const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME = "__last_observed_time__"; const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS = "__keep_alive_seconds__"; @@ -32,8 +32,9 @@ const std::string DEFAULT_CONTENT_KEY_METHOD = "__method__"; const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE = "update"; const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE = "expire"; -// for process entity -const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS = "process"; +// process entity +const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_ECS_PROCESS = "acs.ecs.process"; +const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_HOST_PROCESS = "infra.host.process"; const std::string DEFAULT_CONTENT_KEY_PROCESS_PID = "pid"; const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID = "ppid"; const std::string DEFAULT_CONTENT_KEY_PROCESS_USER = "user"; @@ -44,4 +45,12 @@ const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY = "binary"; const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS = "arguments"; const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE = "language"; const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID = "container_id"; + +const std::string DEFAULT_CONTENT_KEY_SRC_DOMAIN = "__src_domain__"; +const std::string DEFAULT_CONTENT_KEY_SRC_ENTITY_TYPE = "__src_entity_type__"; +const std::string DEFAULT_CONTENT_KEY_SRC_ENTITY_ID = "__src_entity_id__"; +const std::string DEFAULT_CONTENT_KEY_DEST_DOMAIN = "__dest_domain__"; +const std::string DEFAULT_CONTENT_KEY_DEST_ENTITY_TYPE = "__dest_entity_type__"; +const std::string DEFAULT_CONTENT_KEY_DEST_ENTITY_ID = "__dest_entity_id__"; +const std::string DEFAULT_CONTENT_KEY_RELATION_TYPE = "__relation_type__"; } // namespace logtail diff --git a/core/constants/EntityConstants.h b/core/constants/EntityConstants.h index acb6c4cccc..c86a723671 100644 --- a/core/constants/EntityConstants.h +++ b/core/constants/EntityConstants.h @@ -33,8 +33,9 @@ extern const std::string DEFAULT_CONTENT_KEY_METHOD; extern const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE; extern const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE; -// for process entity -extern const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS; +// process entity +extern const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_ECS_PROCESS; +extern const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_HOST_PROCESS; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PID; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_USER; @@ -45,4 +46,13 @@ extern const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE; extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID; + +// link +extern const std::string DEFAULT_CONTENT_KEY_SRC_DOMAIN; +extern const std::string DEFAULT_CONTENT_KEY_SRC_ENTITY_TYPE; +extern const std::string DEFAULT_CONTENT_KEY_SRC_ENTITY_ID; +extern const std::string DEFAULT_CONTENT_KEY_DEST_DOMAIN; +extern const std::string DEFAULT_CONTENT_KEY_DEST_ENTITY_TYPE; +extern const std::string DEFAULT_CONTENT_KEY_DEST_ENTITY_ID; +extern const std::string DEFAULT_CONTENT_KEY_RELATION_TYPE; } // namespace logtail diff --git a/core/host_monitor/HostMonitorInputRunner.cpp b/core/host_monitor/HostMonitorInputRunner.cpp index dcea6809ef..470130b5a9 100755 --- a/core/host_monitor/HostMonitorInputRunner.cpp +++ b/core/host_monitor/HostMonitorInputRunner.cpp @@ -41,8 +41,9 @@ HostMonitorInputRunner::HostMonitorInputRunner() : mThreadPool(ThreadPool(3)) { void HostMonitorInputRunner::UpdateCollector(const std::vector& newCollectors, QueueKey processQueueKey, - int inputIndex) { - std::unique_lock lock(mRegisteredCollectorMapMutex); + int inputIndex, + int interval) { + std::unique_lock lock(mRegisteredCollectorMapMutex); for (auto& collector : newCollectors) { auto iter = mRegisteredCollectorMap.find(collector); if (iter == mRegisteredCollectorMap.end()) { @@ -53,7 +54,7 @@ void HostMonitorInputRunner::UpdateCollector(const std::vector& new iter->second.Enable(); // add timer event HostMonitorTimerEvent::CollectConfig collectConfig( - collector, processQueueKey, inputIndex, std::chrono::seconds(DEFAULT_SCHEDULE_INTERVAL)); + collector, processQueueKey, inputIndex, std::chrono::seconds(interval)); auto now = std::chrono::steady_clock::now(); auto event = std::make_unique(now, collectConfig); Timer::GetInstance()->PushEvent(std::move(event)); @@ -62,7 +63,7 @@ void HostMonitorInputRunner::UpdateCollector(const std::vector& new } void HostMonitorInputRunner::RemoveCollector() { - std::unique_lock lock(mRegisteredCollectorMapMutex); + std::unique_lock lock(mRegisteredCollectorMapMutex); for (auto& collector : mRegisteredCollectorMap) { collector.second.Disable(); } @@ -93,7 +94,7 @@ void HostMonitorInputRunner::Stop() { } bool HostMonitorInputRunner::HasRegisteredPlugins() const { - std::shared_lock lock(mRegisteredCollectorMapMutex); + std::shared_lock lock(mRegisteredCollectorMapMutex); for (auto& collector : mRegisteredCollectorMap) { if (collector.second.IsEnabled()) { return true; @@ -104,7 +105,7 @@ bool HostMonitorInputRunner::HasRegisteredPlugins() const { bool HostMonitorInputRunner::IsCollectTaskValid(std::chrono::steady_clock::time_point execTime, const std::string& collectorName) { - std::shared_lock lock(mRegisteredCollectorMapMutex); + std::shared_lock lock(mRegisteredCollectorMapMutex); auto it = mRegisteredCollectorMap.find(collectorName); if (it == mRegisteredCollectorMap.end()) { return false; @@ -118,7 +119,7 @@ void HostMonitorInputRunner::ScheduleOnce(std::chrono::steady_clock::time_point PipelineEventGroup group(std::make_shared()); group.SetMetadata(EventGroupMetaKey::HOST_MONITOR_COLLECT_INTERVAL, std::to_string(config.mInterval.count())); - std::unique_lock lock(mRegisteredCollectorMapMutex); + std::unique_lock lock(mRegisteredCollectorMapMutex); auto collector = mRegisteredCollectorMap.find(config.mCollectorName); if (collector == mRegisteredCollectorMap.end()) { LOG_ERROR( diff --git a/core/host_monitor/HostMonitorInputRunner.h b/core/host_monitor/HostMonitorInputRunner.h index 20cffb73f3..e3de66c2ee 100644 --- a/core/host_monitor/HostMonitorInputRunner.h +++ b/core/host_monitor/HostMonitorInputRunner.h @@ -35,8 +35,6 @@ namespace logtail { -const int DEFAULT_SCHEDULE_INTERVAL = 10; - struct CollectorInstance { CollectorInstance(std::unique_ptr&& collector) : mCollector(std::move(collector)) {} @@ -66,7 +64,10 @@ class HostMonitorInputRunner : public InputRunner { } // Only support singleton mode - void UpdateCollector(const std::vector& collectorNames, QueueKey processQueueKey, int inputIndex); + void UpdateCollector(const std::vector& collectorNames, + QueueKey processQueueKey, + int inputIndex, + int interval); void RemoveCollector(); void Init() override; diff --git a/core/host_monitor/HostMonitorTimerEvent.cpp b/core/host_monitor/HostMonitorTimerEvent.cpp index 944925a619..df794e1753 100755 --- a/core/host_monitor/HostMonitorTimerEvent.cpp +++ b/core/host_monitor/HostMonitorTimerEvent.cpp @@ -16,7 +16,6 @@ #include "HostMonitorTimerEvent.h" -#include "common/timer/Timer.h" #include "host_monitor/HostMonitorInputRunner.h" namespace logtail { diff --git a/core/host_monitor/collector/ProcessEntityCollector.cpp b/core/host_monitor/collector/ProcessEntityCollector.cpp index 2949bc44fb..d89417c423 100755 --- a/core/host_monitor/collector/ProcessEntityCollector.cpp +++ b/core/host_monitor/collector/ProcessEntityCollector.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include #include "FileSystemUtil.h" +#include "Flags.h" #include "Logger.h" #include "PipelineEventGroup.h" #include "StringTools.h" @@ -52,10 +54,6 @@ ProcessEntityCollector::ProcessEntityCollector() : mProcessSilentCount(INT32_FLA } else { mValidState = true; } - // TODO: fix after host id ready - mHostEntityID = ""; // FetchHostId(); - mDomain = ""; - mEntityType = ""; }; void ProcessEntityCollector::Collect(PipelineEventGroup& group) { @@ -73,23 +71,27 @@ void ProcessEntityCollector::Collect(PipelineEventGroup& group) { = std::to_string(duration_cast(process->startTime.time_since_epoch()).count()); // common fields - event->SetContent(DEFAULT_CONTENT_KEY_ENTITY_TYPE, mEntityType); - event->SetContent(DEFAULT_CONTENT_KEY_ENTITY_ID, - GetProcessEntityID(std::to_string(process->pid), processCreateTime)); - event->SetContent(DEFAULT_CONTENT_KEY_DOMAIN, mDomain); + std::string domain, entityType, hostEntityID, hostEntityType; + FetchDomainInfo(domain, entityType, hostEntityID, hostEntityType); + event->SetContent(DEFAULT_CONTENT_KEY_DOMAIN, domain); + event->SetContent(DEFAULT_CONTENT_KEY_ENTITY_TYPE, entityType); + auto entityID = GetProcessEntityID(std::to_string(process->pid), processCreateTime, hostEntityID); + event->SetContent(DEFAULT_CONTENT_KEY_ENTITY_ID, entityID); + event->SetContent(DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME, processCreateTime); event->SetContent(DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME, std::to_string(logtime)); auto interval = group.GetMetadata(EventGroupMetaKey::HOST_MONITOR_COLLECT_INTERVAL); + int keepAliveSeconds = INT32_FLAG(host_monitor_default_interval) * 2; if (!interval.empty()) { try { - auto keepAliveSeconds = std::stoi(interval.data()) * 2; - event->SetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS, std::to_string(keepAliveSeconds)); - } catch (const std::invalid_argument& e) { - LOG_ERROR(sLogger, ("process entity collector", "invalid interval")("interval", interval)); - } catch (const std::out_of_range& e) { - LOG_ERROR(sLogger, ("process entity collector", "interval out of range")("interval", interval)); + keepAliveSeconds = std::stoi(interval.data()) * 2; + } catch (const std::exception& e) { + LOG_ERROR(sLogger, + ("process entity collector", + "invalid interval")("interval", interval)("use default interval instead", keepAliveSeconds)); } } + event->SetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS, std::to_string(keepAliveSeconds)); // custom fields event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_PID, std::to_string(process->pid)); @@ -102,6 +104,21 @@ void ProcessEntityCollector::Collect(PipelineEventGroup& group) { // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS, ""); TODO: get arguments // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE, ""); TODO: get language // event->SetContent(DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID, ""); TODO: get container id + + // process -> host link + auto linkEvent = group.AddLogEvent(); + linkEvent->SetTimestamp(logtime); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_SRC_DOMAIN, domain); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_SRC_ENTITY_TYPE, entityType); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_SRC_ENTITY_ID, entityID); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_DEST_DOMAIN, domain); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_DEST_ENTITY_TYPE, hostEntityType); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_DEST_ENTITY_ID, hostEntityID); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_RELATION_TYPE, DEFAULT_CONTENT_VALUE_METHOD_UPDATE); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_RELATION_TYPE, DEFAULT_CONTENT_VALUE_METHOD_UPDATE); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME, processCreateTime); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME, std::to_string(logtime)); + linkEvent->SetContent(DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS, std::to_string(keepAliveSeconds)); } } @@ -272,12 +289,31 @@ bool ProcessEntityCollector::WalkAllProcess(const std::filesystem::path& root, return true; } -const std::string ProcessEntityCollector::GetProcessEntityID(StringView pid, StringView createTime) { +const std::string +ProcessEntityCollector::GetProcessEntityID(StringView pid, StringView createTime, const std::string& hostEntityID) { std::ostringstream oss; - oss << mHostEntityID << pid << createTime; + oss << hostEntityID << pid << createTime; auto bigID = CalcMD5(oss.str()); std::transform(bigID.begin(), bigID.end(), bigID.begin(), ::tolower); return bigID; } +void ProcessEntityCollector::FetchDomainInfo(std::string& domain, + std::string& entityType, + std::string& hostEntityID, + std::string& hostEntityType) { + ECSMeta metaObj = HostIdentifier::Instance()->GetECSMeta(); + if (metaObj.isValid) { + domain = DEFAULT_CONTENT_VALUE_DOMAIN_ACS; + entityType = DEFAULT_CONTENT_VALUE_ENTITY_TYPE_ECS_PROCESS; + hostEntityID = metaObj.instanceID; + hostEntityType = DEFAULT_HOST_TYPE_ECS; + } else { + domain = DEFAULT_CONTENT_VALUE_DOMAIN_INFRA; + entityType = DEFAULT_CONTENT_VALUE_ENTITY_TYPE_HOST_PROCESS; + hostEntityID = HostIdentifier::Instance()->GetHostId().id; + hostEntityType = DEFAULT_HOST_TYPE_HOST; + } +} + } // namespace logtail diff --git a/core/host_monitor/collector/ProcessEntityCollector.h b/core/host_monitor/collector/ProcessEntityCollector.h index cc001d5028..b08c33bb91 100644 --- a/core/host_monitor/collector/ProcessEntityCollector.h +++ b/core/host_monitor/collector/ProcessEntityCollector.h @@ -36,6 +36,7 @@ #include "host_monitor/collector/BaseCollector.h" DECLARE_FLAG_INT32(process_collect_silent_count); +DECLARE_FLAG_INT32(host_monitor_default_interval); using namespace std::chrono; @@ -158,16 +159,17 @@ class ProcessEntityCollector : public BaseCollector { ProcessStatPtr ParseProcessStat(pid_t pid, std::string& line); bool WalkAllProcess(const std::filesystem::path& root, const std::function& callback); - const std::string GetProcessEntityID(StringView pid, StringView createTime); + const std::string GetProcessEntityID(StringView pid, StringView createTime, const std::string& hostEntityID); + void FetchDomainInfo(std::string& domain, + std::string& entityType, + std::string& hostEntityID, + std::string& hostEntityType); steady_clock::time_point mProcessSortTime; std::vector mSortProcessStats; std::unordered_map mPrevProcessStat; const int mProcessSilentCount; - std::string mHostEntityID; - std::string mEntityType; - std::string mDomain; #ifdef APSARA_UNIT_TEST_MAIN friend class ProcessEntityCollectorUnittest; diff --git a/core/plugin/input/InputHostMeta.cpp b/core/plugin/input/InputHostMeta.cpp index 3b5ff099fb..b07bd3ee5e 100644 --- a/core/plugin/input/InputHostMeta.cpp +++ b/core/plugin/input/InputHostMeta.cpp @@ -18,6 +18,7 @@ #include "HostMonitorInputRunner.h" #include "Logger.h" +#include "ParamExtractor.h" #include "constants/EntityConstants.h" namespace logtail { @@ -25,13 +26,26 @@ namespace logtail { const std::string InputHostMeta::sName = "input_host_meta"; bool InputHostMeta::Init(const Json::Value& config, Json::Value& optionalGoPipeline) { + std::string errorMsg; + if (!GetOptionalIntParam(config, "Interval", mInterval, errorMsg)) { + PARAM_WARNING_DEFAULT(mContext->GetLogger(), + mContext->GetAlarm(), + errorMsg, + false, + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); + } return true; } bool InputHostMeta::Start() { LOG_INFO(sLogger, ("input host meta start", mContext->GetConfigName())); HostMonitorInputRunner::GetInstance()->Init(); - HostMonitorInputRunner::GetInstance()->UpdateCollector({"process_entity"}, mContext->GetProcessQueueKey(), mIndex); + HostMonitorInputRunner::GetInstance()->UpdateCollector( + {"process_entity"}, mContext->GetProcessQueueKey(), mIndex, mInterval); return true; } diff --git a/core/plugin/input/InputHostMeta.h b/core/plugin/input/InputHostMeta.h index cdae8b6f2e..b28e26c769 100644 --- a/core/plugin/input/InputHostMeta.h +++ b/core/plugin/input/InputHostMeta.h @@ -16,8 +16,11 @@ #pragma once +#include "Flags.h" #include "pipeline/plugin/interface/Input.h" +DECLARE_FLAG_INT32(host_monitor_default_interval); + namespace logtail { class InputHostMeta : public Input { @@ -30,6 +33,9 @@ class InputHostMeta : public Input { bool Stop(bool isPipelineRemoving) override; bool SupportAck() const override { return true; } +private: + int mInterval = INT32_FLAG(host_monitor_default_interval); + #ifdef APSARA_UNIT_TEST_MAIN friend class InputHostMetaUnittest; #endif diff --git a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp index 795521d679..338d8993a0 100644 --- a/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp +++ b/core/unittest/host_monitor/HostMonitorInputRunnerUnittest.cpp @@ -52,7 +52,7 @@ class HostMonitorInputRunnerUnittest : public testing::Test { void HostMonitorInputRunnerUnittest::TestUpdateAndRemoveCollector() const { auto runner = HostMonitorInputRunner::GetInstance(); runner->Init(); - runner->UpdateCollector({MockCollector::sName}, QueueKey{}, 0); + runner->UpdateCollector({MockCollector::sName}, QueueKey{}, 0, 60); APSARA_TEST_TRUE_FATAL(runner->IsCollectTaskValid(std::chrono::steady_clock::now(), MockCollector::sName)); APSARA_TEST_FALSE_FATAL( runner->IsCollectTaskValid(std::chrono::steady_clock::now() - std::chrono::seconds(60), MockCollector::sName)); diff --git a/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp index 13e52a2a70..6f2b636436 100644 --- a/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp +++ b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp @@ -53,8 +53,7 @@ void ProcessEntityCollectorUnittest::TestSortProcessByCpu() const { void ProcessEntityCollectorUnittest::TestGetProcessEntityID() const { ProcessEntityCollector collect; - collect.mHostEntityID = "123"; - APSARA_TEST_EQUAL(collect.GetProcessEntityID("123", "123"), "f5bb0c8de146c67b44babbf4e6584cc0"); + APSARA_TEST_EQUAL(collect.GetProcessEntityID("123", "123", "123"), "f5bb0c8de146c67b44babbf4e6584cc0"); } UNIT_TEST_CASE(ProcessEntityCollectorUnittest, TestGetNewProcessStat); diff --git a/docs/cn/plugins/input/native/input-host-meta.md b/docs/cn/plugins/input/native/input-host-meta.md new file mode 100644 index 0000000000..43fa3280d1 --- /dev/null +++ b/docs/cn/plugins/input/native/input-host-meta.md @@ -0,0 +1,43 @@ +# 主机元信息采集 + +## 简介 + +`input_host_meta` 定时采集主机元数据，包括主机、进程及其之间的关系。 + +## 版本 + +[Beta](../../stability-level.md) + +## 配置参数 + +| 参数 | 类型，默认值 | 说明 | +| - | - | - | +| Type | String，无默认值（必填） | 插件类型，固定为`input_host_meta`。 | +| Interval | int, 60 | 采集间隔时间，单位为秒。 | + +## 样例 + +* 采集配置 + +```yaml +enable: true +inputs: + - Type: input_host_meta +``` + +* 输出 + +```json +{ + "__domain__":"infra", + "__entity_type__":"infra.host.process", + "__entity_id__":"8a1aee58dcdea68434e058e48e39f965", + "__first_observed_time__":"1735348941", + "__last_observed_time__":"1736163039", + "__keep_alive_seconds__":"120", + "pid":"84450", + "ppid":"0", + "comm":"sh", + "ktime":"1735348941" +} +```