Merge pull request #378 from silinternational/develop

Release 6.10.0 - Enable external-group sync-errors email notification
silinternational · Oct 9, 2024 · 1e83429 · 1e83429
2 parents 34951b5 + 176b911
commit 1e83429
Show file tree

Hide file tree

Showing 13 changed files with 489 additions and 112 deletions.
diff --git a/README.md b/README.md
@@ -71,3 +71,105 @@ Calls are made to Google Analytics regarding users' mfas and whether a password
 
 If you want to have an indication that those calls are likely to succeed, run 
 `$ make callGA`.
+
+## Adding groups to SAML `member` attribute from a Google Sheet
+
+The `local.env.dist` file shows how to add the necessary environment variables
+in order to sync values from a Google Sheet to the `user.groups_external` field
+in the database, which are then included in the SAML `member` attribute that can
+be sent to the website that the user is signing into. See the
+`EXTERNAL_GROUPS_SYNC_*` entries in the `local.env.dist` file.
+
+### How Many of What?
+
+You will need...
+
+* One Google Cloud Console Project.
+  * Example:  
+    `My IDPs External Groups Sync`
+* One Google Sheet per application that needs custom groups.
+  * Examples:  
+    `App A SSO Groups`  
+    `App B SSO Groups`
+* One Service Account (in that Project) per IDP that you want to sync the
+  custom groups into for that application.
+  * Examples:  
+    `IDP 1 groups for App A`  
+    `IDP 2 groups for App A`
+* One tab in each Google Sheet per IDP that you want to sync that application's
+  custom groups into.
+  * Examples:  
+    `idp1`  
+    `idp2`
+
+### Specific How-To Steps
+
+To do this...
+
+1. Use at least version 6.8.0 of ID Broker.
+2. Create a Google Sheet named for the application that needs the groups
+   (e.g. `App A SSO groups`).
+3. Create a tab (in that Google Sheet) named after the short/code name of your
+   IDP (e.g. `idp1`) with two columns: `email` and `groups`.
+   - To add groups for a specific user, put the user's (lowercase) email address
+     for that IDP in the `email` cell in their row.
+   - Only use one row per user.
+   - Put all of a user's desired groups in their `groups` cell, separated by
+     commas. Example: "ext-appa-managers, ext-appa-designers"
+   - Group names must begin with your chosen prefix and a dash
+     (e.g. "ext-appa-").
+4. Create a Google Cloud Console Project (e.g. `My IDPs External Groups Sync`).
+5. Add a Service Account to that Project.
+   - I recommend naming it after both the IDP you will use it for and the
+     application that needs the groups (e.g. `IDP 1 groups for App A`).
+6. Create a JSON Key for that Service Account.
+7. Share the Google Sheet that you created earlier with the `client_email` value
+   in that JSON Key file (as a Viewer, no notification).
+8. Set the following environment variables for your ID Broker instance:
+   - `EXTERNAL_GROUPS_SYNC_set1AppPrefix`
+     - Set this to some prefix starting with "ext-", e.g. `ext-appa`
+   - `EXTERNAL_GROUPS_SYNC_set1GoogleSheetId`
+     - Set this to the ID of the Google Sheet you created earlier.
+   - `EXTERNAL_GROUPS_SYNC_set1JsonAuthString`
+     - Use the JSON key you just created here, compacted to a single line by
+       something like this command:  
+       `cat service-account-key-from-google-abcdef123456.json | jq -c "."`
+9. You can also set the following environment variable if you want to send a
+   notification email any time the sync runs and encounters errors (such as
+   "No user found for email address ..." or "The given group (ext-appb-users)
+   does not start with the given prefix (ext-appa)"):
+   - `EXTERNAL_GROUPS_SYNC_set1ErrorsEmailRecipient`
+     - Set this to a single email address.
+10. If you need to sync those custom groups to another IDP...
+    - Ensure that IDP is also running a recent enough version of ID Broker.
+    - Create another tab in your Google Sheet.
+    - Create another Service Account and JSON Key.
+    - Share the Google Account with that new JSON Key's `client_email`.
+    - Set the above environment variables in that other IDP, using the same
+      app-prefix and Google Sheet ID, but the JSON Auth String from the new JSON
+      Key that you created.
+11. If you need to sync custom groups for _another app_ to your IDP...
+    - Create another Google Sheet similarly, but named for that other app, with
+      a tab for each of the relevant IDPs.
+    - Create another Service Account (and JSON Key) in that existing Google
+      Cloud Console Project.
+    - Share the Google Account with that JSON Key's `client_email`.
+    - Add another set of the above environment variables, but with the next
+      number in the lowercased portion
+      (e.g. `EXTERNAL_GROUPS_SYNC_set2AppPrefix`), using an app-prefix for that
+      other app, the new Google Sheet's ID, and the new JSON Key (as the JSON
+      Auth String).
+
+### Rotating external-groups sync credentials
+
+You can easily rotate the credentials for a Service Account by creating a new
+JSON Key for it. Then simply update the
+`EXTERNAL_GROUPS_SYNC_set(NUMBER)JsonAuthString` environment variable to use the
+contents of that new JSON Key.
+
+Since you can have multiple Keys on Google Cloud for a given Service Account,
+you can create a new Key in the Google Cloud Console, switch from the old one to
+the new one here, then remove the old one from the Google Cloud Console. In
+other words, you can wait to delete the previous Key from that Service Account
+until you have deployed the new credentials, if desired, to avoid service
+interruption.
diff --git a/application/common/components/Emailer.php b/application/common/components/Emailer.php
@@ -49,6 +49,8 @@ class Emailer extends Component
 
     public const SUBJ_ABANDONED_USER_ACCOUNTS = 'Unused {idpDisplayName} Identity Accounts';
 
+    public const SUBJ_EXT_GROUP_SYNC_ERRORS = "Errors while syncing '{appPrefix}' external-groups to the {idpDisplayName} IDP";
+
     public const PROP_SUBJECT = 'subject';
     public const PROP_TO_ADDRESS = 'to_address';
     public const PROP_CC_ADDRESS = 'cc_address';
@@ -141,6 +143,8 @@ class Emailer extends Component
 
     public $subjectForAbandonedUsers;
 
+    public $subjectForExtGroupSyncErrors;
+
     /* The email to contact for HR notifications */
     public $hrNotificationsEmail;
 
@@ -196,7 +200,7 @@ protected function assertConfigIsValid()
      * @param string $textBody The email body (as plain text).
      * @param string $ccAddress Optional. Email address to include as 'cc'.
      * @param string $bccAddress Optional. Email address to include as 'bcc'.
-     * @param string $delaySeconds Number of seconds to delay sending the email. Default = no delay.
+     * @param int $delaySeconds Number of seconds to delay sending the email. Default = no delay.
      * @throws \Sil\EmailService\Client\EmailServiceClientException
      */
     protected function email(
@@ -302,11 +306,14 @@ public function init()
 
         $this->subjectForAbandonedUsers = $this->subjectForAbandonedUsers ?? self::SUBJ_ABANDONED_USER_ACCOUNTS;
 
+        $this->subjectForExtGroupSyncErrors = $this->subjectForExtGroupSyncErrors ?? self::SUBJ_EXT_GROUP_SYNC_ERRORS;
+
         $this->subjects = [
             EmailLog::MESSAGE_TYPE_INVITE => $this->subjectForInvite,
             EmailLog::MESSAGE_TYPE_MFA_RATE_LIMIT => $this->subjectForMfaRateLimit,
             EmailLog::MESSAGE_TYPE_PASSWORD_CHANGED => $this->subjectForPasswordChanged,
             EmailLog::MESSAGE_TYPE_WELCOME => $this->subjectForWelcome,
+            EmailLog::MESSAGE_TYPE_EXT_GROUP_SYNC_ERRORS => $this->subjectForExtGroupSyncErrors,
             EmailLog::MESSAGE_TYPE_GET_BACKUP_CODES => $this->subjectForGetBackupCodes,
             EmailLog::MESSAGE_TYPE_REFRESH_BACKUP_CODES => $this->subjectForRefreshBackupCodes,
             EmailLog::MESSAGE_TYPE_LOST_SECURITY_KEY => $this->subjectForLostSecurityKey,
@@ -338,15 +345,20 @@ public function init()
      *
      * @param string $messageType The message type. Must be one of the
      *     EmailLog::MESSAGE_TYPE_* values.
-     * @param User $user The intended recipient.
+     * @param ?User $user The intended recipient, if applicable. If not provided, a 'toAddress'
+     *     must be in the $data parameter.
      * @param string[] $data Data fields for email template. Include key 'toAddress' to override
      *     sending to primary address in User object.
      * @param int $delaySeconds Number of seconds to delay sending the email. Default = no delay.
      * @throws \Sil\EmailService\Client\EmailServiceClientException
      */
-    public function sendMessageTo(string $messageType, User $user, array $data = [], int $delaySeconds = 0)
-    {
-        if ($user->active === 'no') {
+    public function sendMessageTo(
+        string $messageType,
+        ?User $user = null,
+        array $data = [],
+        int $delaySeconds = 0
+    ) {
+        if ($user && $user->active === 'no') {
             \Yii::warning([
                 'action' => 'send message',
                 'status' => 'canceled',
@@ -357,7 +369,7 @@ public function sendMessageTo(string $messageType, User $user, array $data = [],
         }
 
         $dataForEmail = ArrayHelper::merge(
-            $user->getAttributesForEmail(),
+            $user ? $user->getAttributesForEmail() : [],
             $this->otherDataForEmails,
             $data
         );
@@ -372,7 +384,9 @@ public function sendMessageTo(string $messageType, User $user, array $data = [],
 
         $this->email($toAddress, $subject, $htmlBody, strip_tags($htmlBody), $ccAddress, $bccAddress, $delaySeconds);
 
-        EmailLog::logMessage($messageType, $user->id);
+        if ($user !== null) {
+            EmailLog::logMessage($messageType, $user->id);
+        }
     }
 
     /**
@@ -740,6 +754,38 @@ public function sendPasswordExpiredEmails()
         ]));
     }
 
+    public function sendExternalGroupSyncErrorsEmail(
+        string $appPrefix,
+        array $errors,
+        string $recipient,
+        string $googleSheetUrl
+    ) {
+        $logData = [
+            'action' => 'send external-groups sync errors email',
+            'status' => 'starting',
+        ];
+
+        $this->logger->info(array_merge($logData, [
+            'errors' => count($errors)
+        ]));
+
+        $this->sendMessageTo(
+            EmailLog::MESSAGE_TYPE_EXT_GROUP_SYNC_ERRORS,
+            null,
+            [
+                'toAddress' => $recipient,
+                'appPrefix' => $appPrefix,
+                'errors' => $errors,
+                'googleSheetUrl' => $googleSheetUrl,
+                'idpDisplayName' => \Yii::$app->params['idpDisplayName'],
+            ]
+        );
+
+        $this->logger->info(array_merge($logData, [
+            'status' => 'finished',
+        ]));
+    }
+
     /**
      * Sends email alert to HR with all abandoned users, if any
      */

diff --git a/application/common/components/ExternalGroupsSync.php b/application/common/components/ExternalGroupsSync.php
@@ -17,6 +17,7 @@ public static function syncAllSets(array $syncSetsParams)
             $appPrefixKey = sprintf('set%uAppPrefix', $i);
             $googleSheetIdKey = sprintf('set%uGoogleSheetId', $i);
             $jsonAuthStringKey = sprintf('set%uJsonAuthString', $i);
+            $errorsEmailRecipientKey = sprintf('set%uErrorsEmailRecipient', $i);
 
             if (! array_key_exists($appPrefixKey, $syncSetsParams)) {
                 Yii::warning(sprintf(
@@ -29,6 +30,7 @@ public static function syncAllSets(array $syncSetsParams)
             $appPrefix = $syncSetsParams[$appPrefixKey] ?? '';
             $googleSheetId = $syncSetsParams[$googleSheetIdKey] ?? '';
             $jsonAuthString = $syncSetsParams[$jsonAuthStringKey] ?? '';
+            $errorsEmailRecipient = $syncSetsParams[$errorsEmailRecipientKey] ?? '';
 
             if (empty($appPrefix) || empty($googleSheetId) || empty($jsonAuthString)) {
                 Yii::error(sprintf(
@@ -45,23 +47,63 @@ public static function syncAllSets(array $syncSetsParams)
                     $appPrefix,
                     $googleSheetId
                 ));
-                self::syncSet($appPrefix, $googleSheetId, $jsonAuthString);
+                self::syncSet(
+                    $appPrefix,
+                    $googleSheetId,
+                    $jsonAuthString,
+                    $errorsEmailRecipient
+                );
             }
         }
     }
 
-    private static function syncSet(
+    /**
+     * Sync the specified external-groups data with the specified Google Sheet.
+     *
+     * @param string $appPrefix
+     * @param string $googleSheetId
+     * @param string $jsonAuthString
+     * @param string $errorsEmailRecipient
+     * @throws \Google\Service\Exception
+     */
+    public static function syncSet(
         string $appPrefix,
         string $googleSheetId,
-        string $jsonAuthString
+        string $jsonAuthString,
+        string $errorsEmailRecipient = ''
     ) {
         $desiredExternalGroups = self::getExternalGroupsFromGoogleSheet(
             $googleSheetId,
             $jsonAuthString
         );
+        self::processUpdates(
+            $appPrefix,
+            $desiredExternalGroups,
+            $errorsEmailRecipient,
+            $googleSheetId
+        );
+    }
+
+    /**
+     * Update users' external-groups using the given data, and handle (and
+     * return) any errors.
+     *
+     * @param string $appPrefix
+     * @param array $desiredExternalGroups
+     * @param string $errorsEmailRecipient
+     * @param string $googleSheetIdForEmail -- The Google Sheet's ID, for use in
+     *     the sync-error notification email.
+     * @return string[] -- The resulting error messages.
+     */
+    public static function processUpdates(
+        string $appPrefix,
+        array $desiredExternalGroups,
+        string $errorsEmailRecipient = '',
+        string $googleSheetIdForEmail = ''
+    ): array {
         $errors = User::updateUsersExternalGroups($appPrefix, $desiredExternalGroups);
         Yii::warning(sprintf(
-            "Ran sync for '%s' external groups.",
+            "Updated '%s' external groups.",
             $appPrefix
         ));
 
@@ -76,7 +118,21 @@ private static function syncSet(
                 $errorSummary = substr($errorSummary, 0, 997) . '...';
             }
             Yii::error($errorSummary);
+
+            if (!empty($errorsEmailRecipient)) {
+                $googleSheetUrl = '';
+                if (!empty($googleSheetIdForEmail)) {
+                    $googleSheetUrl = 'https://docs.google.com/spreadsheets/d/' . $googleSheetIdForEmail;
+                }
+                self::sendSyncErrorsEmail(
+                    $appPrefix,
+                    $errors,
+                    $errorsEmailRecipient,
+                    $googleSheetUrl
+                );
+            }
         }
+        return $errors;
     }
 
     /**
@@ -122,4 +178,27 @@ private static function getExternalGroupsFromGoogleSheet(
         }
         return $data;
     }
+
+    /**
+     * @param string $appPrefix
+     * @param string[] $errors
+     * @param string $recipient
+     * @param string $googleSheetUrl
+     * @return void
+     */
+    private static function sendSyncErrorsEmail(
+        string $appPrefix,
+        array $errors,
+        string $recipient,
+        string $googleSheetUrl
+    ) {
+        /* @var $emailer Emailer */
+        $emailer = \Yii::$app->emailer;
+        $emailer->sendExternalGroupSyncErrorsEmail(
+            $appPrefix,
+            $errors,
+            $recipient,
+            $googleSheetUrl
+        );
+    }
 }
diff --git a/application/common/config/main.php b/application/common/config/main.php
@@ -82,6 +82,7 @@
             'otherDataForEmails' => [
                 'emailSignature' => Env::get('EMAIL_SIGNATURE', ''),
                 'helpCenterUrl' => Env::get('HELP_CENTER_URL'),
+                'idpName' => $idpName,
                 'idpDisplayName' => $idpDisplayName,
                 'passwordProfileUrl' => $passwordProfileUrl . '/#',
                 'supportEmail' => Env::get('SUPPORT_EMAIL'),
@@ -125,6 +126,7 @@
             'subjectForPasswordExpiring' => Env::get('SUBJECT_FOR_PASSWORD_EXPIRING'),
             'subjectForPasswordExpired' => Env::get('SUBJECT_FOR_PASSWORD_EXPIRED'),
             'subjectForAbandonedUsers' => Env::get('SUBJECT_FOR_ABANDONED_USERS'),
+            'subjectForExtGroupSyncErrors' => Env::get('SUBJECT_FOR_EXT_GROUP_SYNC_ERRORS'),
 
             'lostSecurityKeyEmailDays' => Env::get('LOST_SECURITY_KEY_EMAIL_DAYS', 62),
             'minimumBackupCodesBeforeNag' => Env::get('MINIMUM_BACKUP_CODES_BEFORE_NAG', 4),