feat: Allow instrumenting edxapp with Datadog APM (#7136)

Introduces `COMMON_ENABLE_DATADOG_APP` for APM instrumention with Datadog,
as distinct from `COMMON_ENABLE_DATADOG` that just installs the agent
(for infrastructure monitoring).

See edx/edx-arch-experiments#574

CHANGELOG.md

@@ -4,6 +4,9 @@
 All notable changes to this project will be documented in this file.
 Add any new changes to the top (right below this line).
 
+- 2024-03-20
+  - Add `COMMON_ENABLE_DATADOG_APP` for APM instrumention, supported in LMS and CMS so far. Disabled by default.
+
 - 2024-01-25
   - Role: mfe
     - Added `MFE_ENVIRONMENT_DEFAULT_EXTRA` to allow operators to add extra environment variables to all MFEs when

playbooks/roles/common_vars/defaults/main.yml

@@ -98,7 +98,10 @@ COMMON_MYSQL_MIGRATE_PASS: 'password'
 
 COMMON_MONGO_READ_ONLY_USER: 'read_only'
 COMMON_MONGO_READ_ONLY_PASS: !!null
+# Enable installation of the Datadog agent (infrastructure monitoring)
 COMMON_ENABLE_DATADOG: False
+# Enable APM monitoring with Datadog (metrics, traces, and logs)
+COMMON_ENABLE_DATADOG_APP: False
 COMMON_ENABLE_NGINXTRA: False
 COMMON_ENABLE_SPLUNKFORWARDER: False
 COMMON_ENABLE_NEWRELIC: False

playbooks/roles/edxapp/tasks/deploy.yml

@@ -216,6 +216,23 @@
     - install
     - install:app-requirements
 
+- name: "Install Datadog APM requirements"
+  when: COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP
+  pip:
+    name:
+      - ddtrace
+    extra_args: "--exists-action w {{ item.extra_args|default('') }}"
+    virtualenv: "{{ edxapp_venv_dir }}"
+    state: present
+  become_user: "{{ edxapp_user }}"
+  register: edxapp_install_datadog_reqs
+  until: edxapp_install_datadog_reqs is succeeded
+  retries: 5
+  delay: 15
+  tags:
+    - install
+    - install:app-requirements
+
 # Pulling Atlas translations into the repo needs to happen after
 # Python dependencies have been installed. Note: This task leaves the
 # git working directory in a "dirty" state.

playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2

@@ -4,8 +4,10 @@
 
 {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %}
 
+{% set executable = edxapp_venv_bin + '/gunicorn' %}
+
 {% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/gunicorn' %}
+{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + executable %}
 
 export NEW_RELIC_DISTRIBUTED_TRACING_ENABLED="{{ EDXAPP_CMS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}"
 export NEW_RELIC_APP_NAME="{{ EDXAPP_NEWRELIC_CMS_APPNAME }}"
@@ -16,10 +18,11 @@ if command -v ec2metadata >/dev/null 2>&1; then
   export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID"
 fi
 export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
+{% endif -%}
 
-{% else %}
-{% set executable = edxapp_venv_bin + '/gunicorn' %}
-{% endif %}
+{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}
+{% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %}
+{% endif -%}
 
 export PORT="{{ edxapp_cms_gunicorn_port }}"
 export ADDRESS="{{ edxapp_cms_gunicorn_host }}"

playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2

@@ -4,8 +4,10 @@
 
 {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %}
 
+{% set executable = edxapp_venv_bin + '/gunicorn' %}
+
 {% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/gunicorn' %}
+{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + executable %}
 
 export NEW_RELIC_DISTRIBUTED_TRACING_ENABLED="{{ EDXAPP_LMS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}"
 export NEW_RELIC_APP_NAME="{{ EDXAPP_NEWRELIC_LMS_APPNAME }}"
@@ -16,10 +18,10 @@ if command -v ec2metadata >/dev/null 2>&1; then
   export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID"
 fi
 export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
+{% endif -%}
 
-{% else %}
-{% set executable = edxapp_venv_bin + '/gunicorn' %}
-
+{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}
+{% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %}
 {% endif -%}
 
 export PORT="{{ edxapp_lms_gunicorn_port }}"