Release v0.15.0 (#698)

.editorconfig

@@ -10,7 +10,7 @@ indent_style = space
 indent_size = 2
 trim_trailing_whitespace = true
 
-[*.{py, pyi}]
+[*.{py,pyi}]
 indent_style = space
 indent_size = 4
 

.env.sample

@@ -1,84 +1,70 @@
+# TODO generate heroku app.json from sample
 # Telegram Bot API Token, obtain it here: https://t.me/BotFather. This is the only required configuration variable.
-TG_BOT_TOKEN="166849652:AAEPCgHuDf3K6HvI2OkrJmDN2k9R6mcfmLs"
+TG_BOT_TOKEN="166849652:AAEPCgHuDf3K6HvI2OkrJmDN2k9Rexample"
+# TODO
 #TG_BOT_API="https://api.telegram.org"
 #TG_BOT_API="http://127.0.0.1:8081"
+# Chat ID of bot owner for alerts and permissions
+TG_BOT_OWNER_CHAT_ID="1265343"
 
+# TODO
+CHAT_STORAGE="/home/gpchelkin/scdlbot.pickle"
 # (Absolute?) path to parent directory for downloads directories, default: /tmp/scdlbot
 DL_DIR="/tmp/scdlbot"
-
+# TODO
+BIN_PATH=""
+# TODO
+WORKERS="2"
 # Download timeout in seconds, stop downloading if it takes longer than allowed
 DL_TIMEOUT="300"
-
+# TODO
+CHECK_URL_TIMEOUT="30"
+# TODO
+COMMON_CONNECTION_TIMEOUT="10"
+# Telegram upload file size limit (in bytes). API limit is 50 MB, Local mode API limit is 2000 MB (so you can change it to 1900_000_000).
+MAX_TG_FILE_SIZE = "45_000_000"
+# Bot will not try to split and send files bigger than this (in bytes)
+MAX_CONVERT_FILE_SIZE="80_000_000"
 # Comma-separated chat IDs with no replying and caption spam
 NO_FLOOD_CHAT_IDS="-10018859218,-1011068201"
-
-ALERT_CHAT_IDS="1265343,3265143"
-
-BIN_PATH=""
-
-CHAT_STORAGE="/home/gpchelkin/scdlbotdata"
-
-# For using inline mode bot needs to store audios somewhere. ID of that chat.
-STORE_CHAT_ID="-1795100"
-
+# HTTP or local path with cookies file for Instagram and/or Yandex.Music
+COOKIES_FILE="https://example.com/cookies.txt"
+# TODO
+PROXIES="http://127.0.0.1:3187,http://127.0.0.1:3188,"
+# TODO
+SOURCE_IPS="9.21.18.2,9.21.16.9"
+# A space separated list of domains which should be considered whitelisted - the bot will only process these domains. Example of domain: example.com if the domain has a subdomain this needs to be included: subdomain.example.com **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.
+WHITELIST_DOMAINS="example.com,subdomain.example.com"
+# A space separated list of domains which should be considered blacklisted - the bot will not process these domains. Example of domain: example.com if the domain has a subdomain this needs to be included: subdomain.example.com **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.
+BLACKLIST_DOMAINS="example.com,subdomain.example.com"
+#BLACKLIST_DOMAINS="invidious.tube,invidious.kavin.rocks,invidious.himiko.cloud,invidious.namazso.eu,dev.viewtube.io,tube.cadence.moe,piped.kavin.rocks"
 A space separated list of chat_ids which should be considered whitelisted - the bot will only join those chats **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.
-WHITELIST_CHATS ="-1795100, 1795102"
+WHITELIST_CHATS ="-1795100,1795102"
 # A space separated list of chat_ids which should be considered blacklisted - the bot will not join those chats. **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.
-BLACKLIST_CHATS ="-1795100, 1795102"
-
-SOURCE_IPS="9.21.18.2,9.21.16.9"
-PROXIES="socks5://127.0.0.1:1080,socks5://127.0.0.1:1081,"
-SERVE_AUDIO="0"
-# Host and port for metrics
-METRICS_HOST="127.0.0.1"
-METRICS_PORT="8000"
-
-# HTTP or local path with cookies file for Yandex.Music
-COOKIES_FILE="http://test.test.com/cookies.txt"
-
-# Don't try to split and send files over this number of bytes
-MAX_CONVERT_FILE_SIZE="80_000_000"
-
-# Telegram upload file size limit
-MAX_TG_FILE_SIZE = "45_000_000"
-#MAX_TG_FILE_SIZE = "1900_000_000"
-
-#Hostname to show in Syslog messages. In most cases it is already set in environment, you may want to use it on Heroku.
-HOSTNAME="prod-aws"
-
-# Syslog server, for example: logsX.papertrailapp.com:55555
-SYSLOG_ADDRESS="logs2.papertrailapp.com:51181"
-
-# Set to 1 to enable verbose debug logging
-SYSLOG_DEBUG="0"
-
-
-# Webhooks: These links should help. In NGINX use TG_BOT_TOKEN without ":" symbol instead of TOKEN1, and port in proxy_pass according to PORT environment variable.
-# https://github.com/python-telegram-bot/python-telegram-bot/wiki/Webhooks#using-nginx-with-one-domainport-for-all-bots
-# https://nginx.org/en/linux_packages.html#mainline
-# https://certbot.eff.org/lets-encrypt/ubuntubionic-nginx
+BLACKLIST_CHATS ="-1795100,1795102"
 
+### Webhook:
 # Use webhook for bot updates: 1, use polling (default): 0, more info https://core.telegram.org/bots/api#getting-updates
-USE_WEBHOOK="0"
-
+WEBHOOK_ENABLE="0"
 # Host and port for webhook to listen to; Heroku sets PORT automatically for web dynos, so you shouldn't set it for Heroku
 HOST="0.0.0.0"
 PORT="5000"
-
-# Your host URL like https://scdlbot.herokuapp.com/, required for webhook mode
-APP_URL="https://yourapp.heroku.com/"
-
-# A space separated list of domains which should be considered whitelisted - the bot will only process these domains. Example of domain: example.com if the domain has a subdomain this needs to be included: subdomain.example.com **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.
-WHITELIST_DOMS="example.com subdomain.example.com"
-# A space separated list of domains which should be considered blacklisted - the bot will not process these domains. Example of domain: example.com if the domain has a subdomain this needs to be included: subdomain.example.com **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.
-BLACKLIST_DOMS="example.com subdomain.example.com"
+# Your host URL like https://scdlbot.herokuapp.com, required for webhook mode
+WEBHOOK_APP_URL_ROOT="https://yourapp.heroku.com"
 # Your URL path for webhook to listen
-URL_PATH="166849652AAEPCgHuDf3K6HvI2OkrJmDN2k9R6mcfmLs"
-
-# Amount of threads in the thread pool for functions decorated with @run_async:
-# https://docs.python-telegram-bot.org/en/stable/telegram.ext.updater.html
-WORKERS=4
+WEBHOOK_APP_URL_PATH="166849652AAEPCgHuDf3K6HvI2OkrJmDN2k9Rexample"
+# TODO
+WEBHOOK_SECRET_TOKEN="CHANGEME"
+# These links should help. In NGINX use WEBHOOK_APP_URL_PATH (TG_BOT_TOKEN without ":"), and port in proxy_pass according to PORT environment variable.
+# https://github.com/python-telegram-bot/python-telegram-bot/wiki/Webhooks#using-nginx-with-one-domainport-for-all-bots
 
-# Variables for some specific deployments, you don't need it:
-#LC_ALL="en_US.UTF8"
-#PYENV_VERSION="3.8.2"
+### Monitoring and logging:
+# Host and port for metrics in Prometheus OpenMetrics format:
+METRICS_HOST="127.0.0.1"
+METRICS_PORT="8000"
+# Set to DEBUG to enable verbose debug logging
+LOGLEVEL="INFO"
+# Syslog server, for example: logsX.papertrailapp.com:55555
+SYSLOG_ADDRESS="logs2.papertrailapp.com:51181"
+# Hostname to show in Syslog messages. In most cases it is already set in environment, you may want to set it manually in Heroku.
+HOSTNAME="prod-aws"

.github/dependabot.yml

@@ -4,19 +4,19 @@ updates:
   directory: "/"
   schedule:
     interval: daily
-    time: '12:00'
+    time: "12:00"
     timezone: Europe/Moscow
   open-pull-requests-limit: 99
   reviewers:
   - gpchelkin
   assignees:
   - gpchelkin
 
-- package-ecosystem: "github-actions"
+- package-ecosystem: github-actions
   directory: "/"
   schedule:
     interval: "daily"
-    time: '12:00'
+    time: "12:00"
     timezone: Europe/Moscow
   open-pull-requests-limit: 99
   reviewers:

.github/workflows/build.yml

@@ -12,6 +12,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -32,7 +35,7 @@ jobs:
       run: |
         pip install --upgrade pip
         pip install --upgrade setuptools wheel
-        export POETRY_VERSION=1.3.1
+        export POETRY_VERSION=1.3.2
         wget --output-document=install-poetry.py https://install.python-poetry.org
         python "install-poetry.py"
         rm -f "install-poetry.py"

.github/workflows/release.yml

@@ -23,7 +23,7 @@ jobs:
       run: |
         pip install --upgrade pip
         pip install --upgrade setuptools wheel
-        export POETRY_VERSION=1.3.1
+        export POETRY_VERSION=1.3.2
         wget --output-document=install-poetry.py https://install.python-poetry.org
         python "install-poetry.py"
         rm -f "install-poetry.py"

.gitignore

@@ -148,9 +148,8 @@ ENV/
 .idea
 
 #### scdlbot ####
-scdlbotdata
-scdlbotdata.dat
-scdlbotdata.dir
-scdlbotdata.bak
+*.pickle
 *.mp3
 *.mp4
+*.webm
+notes.txt

AUTHORS.rst

@@ -10,4 +10,4 @@ Contributors
 ------------
 
 * Leonid Runyshkin @leovp
-* Vadim Larionov @vadimlarionov <[email protected]>
+* Vadim Larionov @vadimlarionov

CHANGELOG.rst

@@ -1,6 +1,10 @@
 Version history
 ===============
 
+0.15.0 (2023-02-20)
+-----------------------
+* refactor and fix many things
+
 0.14.2 (2022-12-19)
 -----------------------
 * merge #508:

LICENSE

@@ -1,17 +1,17 @@
 
  The MIT License (MIT)
- Copyright (c) 2019 George Pchelkin
- 
+ Copyright (c) 2023 George Pchelkin
+
  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
  in the Software without restriction, including without limitation the rights
  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  copies of the Software, and to permit persons to whom the Software is
  furnished to do so, subject to the following conditions:
- 
+
  The above copyright notice and this permission notice shall be included in all
  copies or substantial portions of the Software.
- 
+
  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.

Makefile

@@ -3,7 +3,7 @@ SHELL:=/usr/bin/env bash
 .PHONY: format
 format:
 	poetry run isort .
-	poetry run black .
+	poetry run black --extend-exclude docs/ .
 
 .PHONY: lint
 lint:
@@ -18,8 +18,25 @@ package:
 	poetry run pip check
 	poetry run safety check --full-report
 
+.PHONY: update
+update:
+	poetry self lock
+	poetry self install --sync
+	poetry self update
+	poetry update --with main,dev,docs
+	poetry export --only main --without-hashes -f requirements.txt -o requirements.txt
+	poetry export --only docs --without-hashes -f requirements.txt -o requirements-docs.txt
+	poetry export --only dev  --without-hashes -f requirements.txt -o requirements-dev.txt
+
 .PHONY: test
 test: lint package
 
+.PHONY: run_dev
+run_dev:
+	ps -ef | grep '[s]cdlbot/scdlbot.py' | grep -v bash | awk '{print $$2}' | xargs --no-run-if-empty kill -15
+	set -o allexport; \
+	source .env-dev; \
+	poetry run python scdlbot/scdlbot.py
+
 .DEFAULT:
 	@cd docs && $(MAKE) $@

README.rst

@@ -82,8 +82,8 @@ scdlbot is standing on the shoulders of giants:
 -  `Bandcamp <https://bandcamp.com>`__:
    `bandcamp-dl <https://github.com/iheanyi/bandcamp-dl>`__
 -  `YouTube <https://www.youtube.com/>`__, `Yandex.Music <https://music.yandex.com/>`__,
-   `Mixcloud <https://www.mixcloud.com/>`__, and almost everything from this `list <https://yt-dl.org/supportedsites.html>`__:
-   `youtube-dl <https://yt-dl.org>`__
+   `Mixcloud <https://www.mixcloud.com/>`__, and almost `everything from this list <https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md>`__:
+   `yt-dlp <https://github.com/yt-dlp/yt-dlp>`__
 
 Run your own scdlbot
 --------------------
@@ -163,9 +163,9 @@ You will need `Heroku CLI <https://cli.heroku.com/>`__ installed.
     # If you've installed from PyPI - download Procfile first (otherwise already present in Git repository):
     curl -O https://raw.githubusercontent.com/gpchelkin/scdlbot/master/Procfile
 
-    # For long polling mode (when USE_WEBHOOK=0):
+    # For long polling mode (when WEBHOOK_ENABLE=0):
     heroku local -e .env worker
-    # For webhook mode (when USE_WEBHOOK=1):
+    # For webhook mode (when WEBHOOK_ENABLE=1):
     heroku local -e .env web
 
 Using Python only

app.json

@@ -29,85 +29,25 @@
       "description": "Telegram Bot API Token",
       "required": true
     },
-    "DL_DIR": {
-      "description": "Parent directory for downloads directories",
-      "value": "/tmp/scdlbot",
-      "required": false
-    },
-    "DL_TIMEOUT": {
-      "description": "Download timeout in seconds, stop downloading if it takes longer than allowed",
-      "value": "300",
-      "required": false
-    },
-    "NO_FLOOD_CHAT_IDS": {
-      "description": "Comma-separated chat IDs with no replying and caption hashtags",
-      "required": false
-    },
-    "ALERT_CHAT_IDS": {
-      "description": "Comma-separated chat IDs with no replying and caption hashtags",
-      "required": false
-    },
-    "BIN_PATH": {
-      "description": "Custom directory where scdl and bandcamp-dl binaries are available",
-      "required": false
-    },
-    "CHAT_STORAGE": {
-      "description": "Chat storage",
-      "value": "/tmp/scdlbotdata",
-      "required": false
-    },
-    "STORE_CHAT_ID": {
-      "description": "Chat ID for storing audios for inline mode",
-      "required": false
-    },
-    "WHITELIST_CHATS": {
-      "description": "A space separated list of chat_ids which should be considered whitelisted - the bot will only join those chats **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.",
-      "required": false
-    },
-    "BLACKLIST_CHATS": {
-      "description": "A space separated list of chat_ids which should be considered blacklisted - the bot will not join those chats. **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.",
+    "WEBHOOK_ENABLE": {
+      "description": "Use webhook for bot updates: 1, use polling (default): 0",
+      "value": "0",
       "required": false
     },
-    "COOKIES_FILE": {
-      "description": "HTTP or local path with cookies file for Yandex.Music",
+    "WEBHOOK_APP_URL_ROOT": {
+      "description": "Your host URL like https://scdlbot.herokuapp.com, required for webhook mode",
       "required": false
     },
-    "MAX_CONVERT_FILE_SIZE": {
-      "description": "Don't try to split and send files over this number of bytes",
-      "value": "80_000_000",
+    "HOST": {
+      "description": "Hostname to show in Syslog messages",
+      "value": "0.0.0.0",
       "required": false
     },
     "HOSTNAME": {
       "description": "Hostname to show in Syslog messages",
       "value": "heroku",
       "required": false
     },
-    "SYSLOG_ADDRESS": {
-      "description": "Syslog server, for example: logsX.papertrailapp.com:55555",
-      "required": false
-    },
-    "SYSLOG_DEBUG": {
-      "description": "Set to 1 to enable verbose debug logging",
-      "value": "0",
-      "required": false
-    },
-    "USE_WEBHOOK": {
-      "description": "Use webhook for bot updates: 1, use polling (default): 0",
-      "value": "0",
-      "required": false
-    },
-    "APP_URL": {
-      "description": "Your host URL like https://scdlbot.herokuapp.com/, required for webhook mode",
-      "required": false
-    },
-    "WHITELIST_DOMS": {
-      "description": "A space separated list of domains which should be considered whitelisted - the bot will only process these domains. Example of domain: example.com if the domain has a subdomain this needs to be included: subdomain.example.com **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.",
-      "required": false
-    },
-    "BLACKLIST_DOMS": {
-      "description": "A space separated list of domains which should be considered blacklisted - the bot will not process these domains. Example of domain: example.com if the domain has a subdomain this needs to be included: subdomain.example.com **NOTE** that if both whitelist and blacklist will be used, only the blacklist will be taken into consideration.",
-      "required": false
-    }
   },
   "formation": {
     "web": {