feat: support host monitor

core/CMakeLists.txt

@@ -129,6 +129,7 @@ set(SUB_DIRECTORIES_LIST
         prometheus prometheus/labels prometheus/schedulers prometheus/async prometheus/component
         ebpf ebpf/observer ebpf/security ebpf/handler
         parser
+        host_monitor host_monitor/collector
         )
 if (LINUX)
     if (ENABLE_ENTERPRISE)

core/app_config/AppConfig.cpp

@@ -165,6 +165,8 @@ DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator s
 DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888);
 DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000);
 DEFINE_FLAG_STRING(_pod_name_, "agent pod name", "");
+DEFINE_FLAG_INT32(host_monitor_default_interval, "default interval for host monitor", 60);
+DEFINE_FLAG_INT32(process_collect_silent_count, "number of process scanned between a sleep", 1000);
 
 DEFINE_FLAG_STRING(app_info_file, "", "app_info.json");
 DEFINE_FLAG_STRING(crash_stack_file_name, "crash stack back trace file name", "backtrace.dat");

core/application/Application.cpp

@@ -14,6 +14,8 @@
 
 #include "application/Application.h"
 
+#include "timer/Timer.h"
+
 #ifndef LOGTAIL_NO_TC_MALLOC
 #include "gperftools/malloc_extension.h"
 #endif
@@ -265,6 +267,7 @@ void Application::Start() { // GCOVR_EXCL_START
 
     // TODO: this should be refactored to internal pipeline
     AlarmManager::GetInstance()->Init();
+    Timer::GetInstance()->Init();
 
     time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, lastCheckTagsTime = 0, lastQueueGCTime = 0;
 #ifndef LOGTAIL_NO_TC_MALLOC

core/common/FileSystemUtil.cpp

@@ -142,6 +142,51 @@ bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t
     return true;
 }
 
+int GetLines(std::istream& is,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage) {
+    std::string line;
+    // 此处必须判断eof，具体原因参见:
+    // https://stackoverflow.com/questions/40561482/getline-throws-basic-iosclear-exception-after-reading-the-last-line
+    while (!is.eof() && std::getline(is, line)) {
+        if (enableEmptyLine || !line.empty()) {
+            pushBack(line);
+        }
+    }
+    return 0;
+}
+
+int GetLines(const std::filesystem::path& filename,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage) {
+    int ret = 0;
+    std::ifstream fin;
+    try {
+        fin.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+        fin.open(filename.string(), std::ios_base::in);
+        fin.exceptions(std::ifstream::goodbit);
+        GetLines(fin, enableEmptyLine, pushBack, errorMessage);
+        fin.close();
+    } catch (const std::exception& fail) {
+        if (errorMessage != nullptr) {
+            LOG_ERROR(sLogger, ("open file fail", filename)("errno", strerror(errno)));
+            ret = -1;
+        }
+        fin.close();
+    }
+    return ret;
+}
+
+int GetFileLines(const std::filesystem::path& filename,
+                 std::vector<std::string>& res,
+                 bool enableEmptyLine,
+                 std::string* errorMessage) {
+    return GetLines(
+        filename, enableEmptyLine, [&res](const std::string& s) { res.push_back(s); }, errorMessage);
+}
+
 bool OverwriteFile(const std::string& fileName, const std::string& content) {
     FILE* pFile = fopen(fileName.c_str(), "w");
     if (pFile == NULL) {

core/common/FileSystemUtil.h

@@ -25,6 +25,9 @@
 #elif defined(_MSC_VER)
 #include <Windows.h>
 #endif
+#include <filesystem>
+#include <functional>
+
 #include "DevInode.h"
 #include "ErrorUtil.h"
 #include "LogtailCommonFlags.h"
@@ -86,6 +89,19 @@ void TrimLastSeperator(std::string& path);
 // ReadFileContent reads all content of @fileName to @content.
 bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t maxFileSize = 8192);
 
+int GetLines(std::istream& is,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage);
+int GetLines(const std::filesystem::path& filename,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage);
+int GetFileLines(const std::filesystem::path& filename,
+                 std::vector<std::string>& res,
+                 bool enableEmptyLine = true,
+                 std::string* errorMessage = nullptr);
+
 // OverwriteFile overwrides @fileName with @content.
 bool OverwriteFile(const std::string& fileName, const std::string& content);
 

core/common/MachineInfoUtil.cpp

@@ -504,13 +504,13 @@ size_t FetchECSMetaCallback(char* buffer, size_t size, size_t nmemb, std::string
 HostIdentifier::HostIdentifier() {
 #ifdef __ENTERPRISE__
     getECSMetaFromFile();
-    updateHostId();
 #else
     ECSMeta ecsMeta;
     if (FetchECSMeta(ecsMeta)) {
         UpdateECSMetaAndHostid(ecsMeta);
     }
 #endif
+    updateHostId();
 }
 
 bool HostIdentifier::UpdateECSMetaAndHostid(const ECSMeta& meta) {

core/common/MachineInfoUtil.h

@@ -81,10 +81,15 @@ class HostIdentifier {
     }
 
     bool UpdateECSMetaAndHostid(const ECSMeta& meta);
-    bool FetchECSMeta(ECSMeta& metaObj);
     void DumpECSMeta();
+#ifdef __ENTERPRISE__
+    bool FetchECSMeta(ECSMeta& metaObj);
+#endif
 
 private:
+#ifndef __ENTERPRISE__
+    bool FetchECSMeta(ECSMeta& metaObj);
+#endif
     void getECSMetaFromFile();
     // 从云助手获取序列号
     void getSerialNumberFromEcsAssist();

core/common/StringTools.cpp

@@ -363,4 +363,12 @@ void RemoveFilePathTrailingSlash(std::string& filePath) {
     filePath = path.string();
 }
 
+bool IsInt(const char* sz) {
+    bool ok = (sz != nullptr && *sz != '\0');
+    for (auto* it = sz; ok && *it; ++it) {
+        ok = (0 != std::isdigit(*it));
+    }
+    return ok;
+}
+
 } // namespace logtail

core/common/StringTools.h

@@ -147,6 +147,12 @@ bool NormalizeTopicRegFormat(std::string& regStr);
 
 void RemoveFilePathTrailingSlash(std::string& path);
 
+bool IsInt(const char* sz);
+
+inline bool IsInt(const std::string& str) {
+    return IsInt(str.c_str());
+}
+
 #if defined(_MSC_VER)
 // TODO: Test it.
 #define FNM_PATHNAME 0

core/common/timer/Timer.cpp

@@ -23,6 +23,9 @@ namespace logtail {
 void Timer::Init() {
     {
         lock_guard<mutex> lock(mThreadRunningMux);
+        if (mIsThreadRunning) {
+            return;
+        }
         mIsThreadRunning = true;
     }
     mThreadRes = async(launch::async, &Timer::Run, this);
@@ -31,6 +34,9 @@ void Timer::Init() {
 void Timer::Stop() {
     {
         lock_guard<mutex> lock(mThreadRunningMux);
+        if (!mIsThreadRunning) {
+            return;
+        }
         mIsThreadRunning = false;
     }
     mCV.notify_one();
@@ -88,4 +94,13 @@ void Timer::Run() {
     }
 }
 
+#ifdef APSARA_UNIT_TEST_MAIN
+void Timer::Clear() {
+    lock_guard<mutex> lock(mQueueMux);
+    while (!mQueue.empty()) {
+        mQueue.pop();
+    }
+}
+#endif
+
 } // namespace logtail

core/common/timer/Timer.h

@@ -34,11 +34,24 @@ struct TimerEventCompare {
 
 class Timer {
 public:
+    Timer(const Timer&) = delete;
+    Timer(Timer&&) = delete;
+    Timer& operator=(const Timer&) = delete;
+    Timer& operator=(Timer&&) = delete;
+    ~Timer() = default;
+    static Timer* GetInstance() {
+        static Timer sInstance;
+        return &sInstance;
+    }
     void Init();
     void Stop();
     void PushEvent(std::unique_ptr<TimerEvent>&& e);
+#ifdef APSARA_UNIT_TEST_MAIN
+    void Clear();
+#endif
 
 private:
+    Timer() = default;
     void Run();
 
     mutable std::mutex mQueueMux;
@@ -47,12 +60,13 @@ class Timer {
 
     std::future<void> mThreadRes;
     mutable std::mutex mThreadRunningMux;
-    bool mIsThreadRunning = true;
+    bool mIsThreadRunning = false;
     mutable std::condition_variable mCV;
 
 #ifdef APSARA_UNIT_TEST_MAIN
     friend class TimerUnittest;
     friend class ScrapeSchedulerUnittest;
+    friend class HostMonitorInputRunnerUnittest;
 #endif
 };
 

core/common/timer/TimerEvent.h

@@ -29,6 +29,7 @@ class TimerEvent {
     virtual bool Execute() = 0;
 
     std::chrono::steady_clock::time_point GetExecTime() const { return mExecTime; }
+    void SetExecTime(std::chrono::steady_clock::time_point nextExecTime) { mExecTime = nextExecTime; }
 
 private:
     std::chrono::steady_clock::time_point mExecTime;

core/constants/EntityConstants.cpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "EntityConstants.h"
+
+namespace logtail {
+
+const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE = "__entity_type__";
+const std::string DEFAULT_CONTENT_KEY_ENTITY_ID = "__entity_id__";
+const std::string DEFAULT_CONTENT_KEY_DOMAIN = "__domain__";
+const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS = "acs";
+const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA = "infra";
+const std::string DEFAULT_HOST_TYPE_ECS = "acs.ecs.instance";
+const std::string DEFAULT_HOST_TYPE_HOST = "acs.host.instance";
+const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME = "__first_observed_time__";
+const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME = "__last_observed_time__";
+const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS = "__keep_alive_seconds__";
+const std::string DEFAULT_CONTENT_KEY_METHOD = "__method__";
+const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE = "update";
+const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE = "expire";
+
+// process entity
+const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_ECS_PROCESS = "acs.ecs.process";
+const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_HOST_PROCESS = "infra.host.process";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_PID = "pid";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID = "ppid";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_USER = "user";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM = "comm";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_KTIME = "ktime";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD = "cwd";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY = "binary";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS = "arguments";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE = "language";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID = "container_id";
+
+const std::string DEFAULT_CONTENT_KEY_SRC_DOMAIN = "__src_domain__";
+const std::string DEFAULT_CONTENT_KEY_SRC_ENTITY_TYPE = "__src_entity_type__";
+const std::string DEFAULT_CONTENT_KEY_SRC_ENTITY_ID = "__src_entity_id__";
+const std::string DEFAULT_CONTENT_KEY_DEST_DOMAIN = "__dest_domain__";
+const std::string DEFAULT_CONTENT_KEY_DEST_ENTITY_TYPE = "__dest_entity_type__";
+const std::string DEFAULT_CONTENT_KEY_DEST_ENTITY_ID = "__dest_entity_id__";
+const std::string DEFAULT_CONTENT_KEY_RELATION_TYPE = "__relation_type__";
+} // namespace logtail

core/constants/EntityConstants.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+
+namespace logtail {
+
+extern const std::string DEFAULT_ENV_KEY_HOST_TYPE;
+extern const std::string DEFAULT_HOST_TYPE_ECS;
+extern const std::string DEFAULT_HOST_TYPE_HOST;
+extern const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE;
+extern const std::string DEFAULT_CONTENT_KEY_ENTITY_ID;
+extern const std::string DEFAULT_CONTENT_KEY_DOMAIN;
+extern const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS;
+extern const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA;
+extern const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME;
+extern const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME;
+extern const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS;
+extern const std::string DEFAULT_CONTENT_KEY_METHOD;
+extern const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE;
+extern const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE;
+
+// process entity
+extern const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_ECS_PROCESS;
+extern const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_HOST_PROCESS;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PID;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_USER;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_KTIME;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID;
+
+// link
+extern const std::string DEFAULT_CONTENT_KEY_SRC_DOMAIN;
+extern const std::string DEFAULT_CONTENT_KEY_SRC_ENTITY_TYPE;
+extern const std::string DEFAULT_CONTENT_KEY_SRC_ENTITY_ID;
+extern const std::string DEFAULT_CONTENT_KEY_DEST_DOMAIN;
+extern const std::string DEFAULT_CONTENT_KEY_DEST_ENTITY_TYPE;
+extern const std::string DEFAULT_CONTENT_KEY_DEST_ENTITY_ID;
+extern const std::string DEFAULT_CONTENT_KEY_RELATION_TYPE;
+} // namespace logtail