Merge pull request #1090 from stratosphereips/develop

Slips v1.1.4
stratosphereips · Nov 29, 2024 · fb6478e · fb6478e
2 parents 6e6bc6f + 1feaa36
commit fb6478e
Show file tree

Hide file tree

Showing 75 changed files with 9,912 additions and 1,112 deletions.
diff --git a/.github/workflows/install-slips-dependencies.yml b/.github/workflows/install-slips-dependencies.yml
@@ -0,0 +1,64 @@
+name: Install Slips Dependencies
+
+on:
+  # workflow_call make this workflow re-usable
+  workflow_call:
+    # these are like variables to make the workflow more clean
+    # we can pass these variable from another workflows if we want
+    inputs:
+      zeek-repo-url:
+        description: 'Zeek repository URL'
+        required: false
+        default: 'http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/'
+        type: string
+      zeek-key-url:
+        description: 'Zeek key URL'
+        required: false
+        default: 'https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key'
+        type: string
+      python-version:
+        description: 'Python version to set up'
+        required: false
+        default: '3.10.12'
+        type: string
+
+jobs:
+  install-dependencies:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.ref }}
+        fetch-depth: ''
+
+    - name: Enable memory overcommit (for Redis)
+      run: sysctl vm.overcommit_memory=1
+
+    - name: Install APT dependencies
+      run: |
+        sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install $(cat install/apt_dependencies.txt)
+        sudo apt-get -y install font-manager
+
+    - name: Save APT Cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          /var/cache/apt/archives
+          /var/lib/apt/lists
+        key: apt-cache
+
+    - name: Set up Python with caching enabled
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: 'pip'
+
+    - name: Install Python dependencies
+      run: python3 -m pip install -r install/requirements.txt
+
+    - name: Install Zeek
+      run: |
+        echo "deb ${{ inputs.zeek-repo-url }} /" | sudo tee /etc/apt/sources.list.d/security:zeek.list
+        curl -fsSL ${{ inputs.zeek-key-url }} | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/security_zeek.gpg
+        sudo apt update && sudo apt install -y --no-install-recommends zeek
+        sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -7,25 +7,45 @@ on:
       - 'develop'
 
 jobs:
-  tests:
+  # uses the common workflow that builds slips
+  install-dependencies-using-reusable-workflow:
+    uses: ./.github/workflows/install-slips-dependencies.yml
+
+
+  integration-tests:
     runs-on: ubuntu-22.04
-    timeout-minutes: 7200
+    timeout-minutes: 1800
+    # make this job depend on the first job
+    needs: install-dependencies-using-reusable-workflow
 
     strategy:
       matrix:
         test_file:
-          - tests/integration_tests/test_config_files.py
-          - tests/integration_tests/test_portscans.py
-          - tests/integration_tests/test_dataset.py
+          - test_config_files.py
+          - test_portscans.py
+          - test_dataset.py
 
     steps:
     - uses: actions/checkout@v4
       with:
         ref: ${{ github.ref }}
         fetch-depth: ''
 
-    - name: Install slips dependencies
-      run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin
+    - name: Restore Zeek Build from Cache
+      id: zeek-cache
+      uses: actions/cache@v4
+      with:
+        path: /opt/zeek
+        key: zeek-cache
+
+    - name: Restore APT cache
+      id: apt-cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          /var/cache/apt/archives
+          /var/lib/apt/lists
+        key: apt-cache
 
     - name: Install Zeek
       run: |
@@ -34,22 +54,25 @@ jobs:
         sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek
         sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro
 
-    - name: Set up Python 3.10.12
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.10.12"
 
-    - name: Install Python dependencies
+    - name: Install apt dependencies (from cache if possible)
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y $(cat install/apt_dependencies.txt)
+
+    - name: Install Python dependencies (from cache if possible)
       run: |
-        python -m pip install --upgrade pip
-        python3 -m pip install --no-cache-dir -r install/requirements.txt
-        python3 -m pip install pytest-timeout
+        python3 -m pip install --upgrade pip
+        python3 -m pip install -r install/requirements.txt
+
+
 
     - name: Start redis server
       run: redis-server --daemonize yes
 
     - name: Run Integration Tests for ${{ matrix.test_file }}
-      run: python3 -m pytest ${{ matrix.test_file }} -vvv -s
+      run: |
+        python3 -m pytest tests/integration_tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 5
 
     - name: Upload Artifacts
       if: success() || failure()

diff --git a/.github/workflows/publish-slips-image.yml → .github/workflows/publish-slips-images.yml b/.github/workflows/publish-slips-image.yml → .github/workflows/publish-slips-images.yml
@@ -1,4 +1,4 @@
-name: CI-production-publishing-slips-image
+name: CI-production-publishing-slips-images
 
 on:
   push:
@@ -40,7 +40,7 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
-    - name: Build and push Slips image using dockerfile
+    - name: Build and push the main Slips image
       id: docker_build_slips
       uses: docker/build-push-action@v6
       with:
@@ -51,3 +51,15 @@ jobs:
           stratosphereips/slips:latest
           stratosphereips/slips:${{ env.SLIPS_VERSION }}
         push: true
+
+    - name: Build and push the light Slips image
+      id: docker_build_light_slips
+      uses: docker/build-push-action@v6
+      with:
+        allow: network.host
+        context: ./
+        file: ./docker/light/Dockerfile
+        tags: |
+          stratosphereips/slips_light:latest
+          stratosphereips/slips_light:${{ env.SLIPS_VERSION }}
+        push: true
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -6,100 +6,110 @@ on:
       - 'master'
       - 'develop'
 
+
 jobs:
-  tests:
+  # uses the common workflow that builds slips
+  install-dependencies-using-reusable-workflow:
+    uses: ./.github/workflows/install-slips-dependencies.yml
+
+  unit-tests:
     runs-on: ubuntu-22.04
     timeout-minutes: 1800
+    # make this job depend on the first job
+    needs: install-dependencies-using-reusable-workflow
 
     strategy:
       matrix:
         test_file:
-          - tests/test_inputProc.py
-          - tests/test_main.py
-          - tests/test_conn.py
-          - tests/test_downloaded_file.py
-          - tests/test_ssl.py
-          - tests/test_tunnel.py
-          - tests/test_ssh.py
-          - tests/test_dns.py
-          - tests/test_notice.py
-          - tests/test_software.py
-          - tests/test_smtp.py
-          - tests/test_whitelist.py
-          - tests/test_arp.py
-          - tests/test_blocking.py
-          - tests/test_flow_handler.py
-          - tests/test_horizontal_portscans.py
-          - tests/test_http_analyzer.py
-          - tests/test_vertical_portscans.py
-          - tests/test_network_discovery.py
-          - tests/test_virustotal.py
-          - tests/test_update_file_manager.py
-          - tests/test_threat_intelligence.py
-          - tests/test_slips_utils.py
-          - tests/test_slips.py
-          - tests/test_profiler.py
-          - tests/test_leak_detector.py
-          - tests/test_ip_info.py
-          - tests/test_evidence.py
-          - tests/test_asn_info.py
-          - tests/test_urlhaus.py
-          - tests/test_markov_chain.py
-          - tests/test_daemon.py
-          - tests/test_go_director.py
-          - tests/test_notify.py
-          - tests/test_checker.py
-          - tests/test_base_model.py
-          - tests/test_set_evidence.py
-          - tests/test_trustdb.py
-          - tests/test_cesnet.py
-          - tests/test_output.py
-          - tests/test_riskiq.py
-          - tests/test_spamhaus.py
-          - tests/test_circllu.py
-          - tests/test_evidence_handler.py
-          - tests/test_alert_handler.py
-          - tests/test_redis_manager.py
-          - tests/test_ioc_handler.py
-          - tests/test_timeline.py
-          - tests/test_database.py
-          - tests/test_symbols_handler.py
+          - test_inputProc.py
+          - test_main.py
+          - test_conn.py
+          - test_downloaded_file.py
+          - test_ssl.py
+          - test_tunnel.py
+          - test_ssh.py
+          - test_dns.py
+          - test_notice.py
+          - test_software.py
+          - test_smtp.py
+          - test_whitelist.py
+          - test_arp.py
+          - test_blocking.py
+          - test_flow_handler.py
+          - test_horizontal_portscans.py
+          - test_http_analyzer.py
+          - test_vertical_portscans.py
+          - test_network_discovery.py
+          - test_virustotal.py
+          - test_update_file_manager.py
+          - test_threat_intelligence.py
+          - test_slips_utils.py
+          - test_slips.py
+          - test_profiler.py
+          - test_leak_detector.py
+          - test_ip_info.py
+          - test_evidence.py
+          - test_asn_info.py
+          - test_urlhaus.py
+          - test_markov_chain.py
+          - test_daemon.py
+          - test_go_director.py
+          - test_notify.py
+          - test_checker.py
+          - test_base_model.py
+          - test_set_evidence.py
+          - test_trustdb.py
+          - test_cesnet.py
+          - test_output.py
+          - test_riskiq.py
+          - test_spamhaus.py
+          - test_circllu.py
+          - test_evidence_handler.py
+          - test_alert_handler.py
+          - test_redis_manager.py
+          - test_ioc_handler.py
+          - test_timeline.py
+          - test_database.py
+          - test_symbols_handler.py
 
     steps:
     - uses: actions/checkout@v4
       with:
         ref: ${{ github.ref }}
         fetch-depth: ''
 
-    - name: Enable memory overcommit (for redis)
-      run: sysctl vm.overcommit_memory=1
 
-    - name: Install slips dependencies
-      run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin
+    - name: Restore APT cache
+      id: apt-cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          /var/cache/apt/archives
+          /var/lib/apt/lists
+        key: apt-cache
+
+    - if: ${{ steps.apt-cache.outputs.cache-hit == 'true' }}
+      name: Echo restored from cache
+      continue-on-error: true
+      run: echo "Restored APT dependencies from cache successfully"
 
-    - name: Install Zeek
+    - name: Install Python dependencies (from cache if possible)
       run: |
-        sudo echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list
-        curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
-        sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek
-        sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro
-
-    - name: Set up Python 3.10.12
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.10.12"
+        python3 -m pip install --upgrade pip
+        python3 -m pip install -r install/requirements.txt
 
-    - name: Install Python dependencies
+    - name: Install apt dependencies (from cache if possible)
       run: |
-        python -m pip install --upgrade pip
-        python3 -m pip install --no-cache-dir -r install/requirements.txt
+        sudo apt-get update
+        sudo apt-get install -y $(cat install/apt_dependencies.txt)
+
 
     - name: Start redis server
       run: redis-server --daemonize yes
 
     - name: Run Unit Tests for ${{ matrix.test_file }}
       run: |
-        python3 -m pytest ${{ matrix.test_file }} -p no:warnings -vv -s -n 5
+        python3 -m pytest tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 5
 
     - name: Upload Artifacts
       if: success() || failure()

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,11 +16,12 @@ repos:
       hooks:
         - id: trailing-whitespace
         - id: check-added-large-files
+          exclude: ^config/local_ti_files/known_fp_hashes\.csv$
         - id: check-docstring-first
         - id: check-merge-conflict
         - id: end-of-file-fixer
         - id: detect-private-key
-          exclude: .*dataset/.*|
+          exclude: .*dataset/.* |
                 (?x)(
                      ^config/$|
                       .*test.* |

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,10 @@
+- 1.1.4 (Nov 29th, 2024)
+- Fix changing the used database in the web interface.
+- Reduce false positive evidence about malicious downloaded files.
+- Fix datetime errors when running on interface
+- Improve the detection of "DNS without connection".
+- Add support for a light Slips docker image.
+
 - 1.1.3 (October 30th, 2024)
 - Enhanced Slips shutdown process for smoother operations.
 - Optimized resource management in Slips, resolving issues with lingering threads in memory.