Custom properties, token refactor, open access teams

- Removes CENTRAL_OPERATIONS_TOKEN
- System teams - open access
  - adds open access concept, which is a broad access team anyone in the org can join without approval
  - open access teams are not recommended the same way as broad access teams during new repo setup
- TypeScript: prefer types to interfaces
- GitHub Apps and REST APIs:
  - Simplifying bound function calls
  - Relocated app and token management files
  - Improves types for header/tokens
  - Allow custom app purposes to retrieve app token instances
  - Custom app purpose debug display fix
  - PAT/app token type identification helper method
  - Collections now expose "collectAllPages" and "collectAllPagesViaHttpGet" to move specific code out of the file
  - Fix for custom apps initialized after startup
- Custom Properties Beta support

.secrets.env.example

@@ -1,4 +1,3 @@
-GITHUB_CENTRAL_OPERATIONS_TOKEN=
 GITHUB_CLIENT_ID=
 GITHUB_CLIENT_SECRET=
 GITHUB_CALLBACK_URL=http://localhost:3000/auth/github/callback

api/client/context/organization/index.ts

@@ -115,7 +115,6 @@ router.get(
       }
       const userAggregateContext = activeContext.aggregations;
       const maintainedTeams = new Set<string>();
-      const broadTeams = new Set<number>(req.organization.broadAccessTeams);
       const userTeams = userAggregateContext.reduceOrganizationTeams(
         organization,
         await userAggregateContext.teams()

api/client/newOrgRepo.ts

@@ -51,6 +51,7 @@ router.get(
       const userAggregateContext = req.apiContext.aggregations;
       const maintainedTeams = new Set<string>();
       const broadTeams = new Set<number>(req.organization.broadAccessTeams);
+      const openAccessTeams = new Set<number>(req.organization.openAccessTeams);
       const userTeams = userAggregateContext.reduceOrganizationTeams(
         organization,
         await userAggregateContext.teams()
@@ -62,6 +63,7 @@ router.get(
       const personalizedTeams = Array.from(combinedTeams.values()).map((combinedTeam) => {
         return {
           broad: broadTeams.has(Number(combinedTeam.id)),
+          isOpenAccessTeam: openAccessTeams.has(Number(combinedTeam.id)),
           description: combinedTeam.description,
           id: Number(combinedTeam.id),
           name: combinedTeam.name,
@@ -86,6 +88,7 @@ router.get(
     const queryCache = providers.queryCache;
     const organization = req.organization as Organization;
     const broadTeams = new Set(organization.broadAccessTeams);
+    const openAccessTeams = new Set<number>(req.organization.openAccessTeams);
     if (req.query.refresh === undefined && queryCache && queryCache.supportsTeams) {
       // Use the newer method in this case...
       const organizationTeams = await queryCache.organizationTeams(organization.id.toString());
@@ -96,6 +99,9 @@ router.get(
           if (broadTeams.has(Number(t.id))) {
             t['broad'] = true;
           }
+          if (openAccessTeams.has(Number(t.id))) {
+            t['openAccess'] = true;
+          }
           return t;
         }),
       }) as unknown as void;

business/account.ts

@@ -10,15 +10,15 @@ import * as common from './common';
 import { wrapError } from '../utils';
 import { corporateLinkToJson } from './corporateLink';
 import { Organization } from './organization';
-import { AppPurpose } from './githubApps';
+import { AppPurpose } from '../lib/github/appPurposes';
 import { ILinkProvider } from '../lib/linkProviders';
 import { CacheDefault, getMaxAgeSeconds } from '.';
 import {
   AccountJsonFormat,
   CoreCapability,
   ICacheOptions,
   ICorporateLink,
-  IGetAuthorizationHeader,
+  GetAuthorizationHeader,
   IGitHubAccountDetails,
   IOperationsInstance,
   IOperationsLinks,
@@ -46,7 +46,7 @@ const secondaryAccountProperties = [];
 
 export class Account {
   private _operations: IOperationsInstance;
-  private _getAuthorizationHeader: IGetAuthorizationHeader;
+  private _getAuthorizationHeader: GetAuthorizationHeader;
 
   private _link: ICorporateLink;
   private _id: number;
@@ -137,7 +137,7 @@ export class Account {
     return this._originalEntity ? this._originalEntity.name : undefined;
   }
 
-  constructor(entity, operations: IOperationsInstance, getAuthorizationHeader: IGetAuthorizationHeader) {
+  constructor(entity, operations: IOperationsInstance, getAuthorizationHeader: GetAuthorizationHeader) {
     common.assignKnownFieldsPrefixed(
       this,
       entity,
@@ -150,7 +150,7 @@ export class Account {
     this._getAuthorizationHeader = getAuthorizationHeader;
   }
 
-  overrideAuthorization(getAuthorizationHeader: IGetAuthorizationHeader) {
+  overrideAuthorization(getAuthorizationHeader: GetAuthorizationHeader) {
     this._getAuthorizationHeader = getAuthorizationHeader;
   }
 
@@ -571,11 +571,8 @@ export class Account {
     return { history, error };
   }
 
-  private authorize(purpose: AppPurpose): IGetAuthorizationHeader | string {
-    const getAuthorizationHeader = this._getAuthorizationHeader.bind(
-      this,
-      purpose
-    ) as IGetAuthorizationHeader;
+  private authorize(purpose: AppPurpose): GetAuthorizationHeader | string {
+    const getAuthorizationHeader = this._getAuthorizationHeader.bind(this, purpose) as GetAuthorizationHeader;
     return getAuthorizationHeader;
   }
 }

business/application.ts

@@ -7,7 +7,7 @@ import { OrganizationSetting } from '../entities/organizationSettings/organizati
 import {
   IOperationsGitHubRestLibrary,
   IOperationsDefaultCacheTimes,
-  IGetAuthorizationHeader,
+  GetAuthorizationHeader,
   IGitHubAppInstallation,
   ICacheOptions,
 } from '../interfaces';
@@ -50,7 +50,7 @@ export default class GitHubApplication {
     public id: number,
     public slug: string,
     public friendlyName: string,
-    private getAuthorizationHeader: IGetAuthorizationHeader
+    private getAuthorizationHeader: GetAuthorizationHeader
   ) {}
 
   static PrimaryInstallationProperties = primaryInstallationProperties;
@@ -115,7 +115,7 @@ export default class GitHubApplication {
   async getInstallations(options?: ICacheOptions): Promise<IGitHubAppInstallation[]> {
     options = options || {};
     const operations = this.operations;
-    const getAuthorizationHeader = this.getAuthorizationHeader.bind(this) as IGetAuthorizationHeader;
+    const getAuthorizationHeader = this.getAuthorizationHeader.bind(this) as GetAuthorizationHeader;
     const github = operations.github;
     const caching = {
       maxAgeSeconds: options.maxAgeSeconds || operations.defaults.orgRepoDetailsStaleSeconds, // borrowing from another value
@@ -132,8 +132,8 @@ export default class GitHubApplication {
     return installations;
   }
 
-  private authorize(): IGetAuthorizationHeader | string {
-    const getAuthorizationHeader = this.getAuthorizationHeader.bind(this) as IGetAuthorizationHeader;
+  private authorize(): GetAuthorizationHeader | string {
+    const getAuthorizationHeader = this.getAuthorizationHeader.bind(this) as GetAuthorizationHeader;
     return getAuthorizationHeader;
   }
 }

business/domains.ts

@@ -3,13 +3,13 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 
-import { AppPurpose, AppPurposeTypes } from './githubApps';
+import { AppPurpose, AppPurposeTypes } from '../lib/github/appPurposes';
 import { Organization } from '.';
 import {
   IOperationsInstance,
-  IPurposefulGetAuthorizationHeader,
+  PurposefulGetAuthorizationHeader,
   throwIfNotGitHubCapable,
-  IGetAuthorizationHeader,
+  GetAuthorizationHeader,
 } from '../interfaces';
 import {
   decorateIterable,
@@ -44,15 +44,15 @@ export class OrganizationDomains {
   private _organization: Organization;
   private _operations: IOperationsInstance;
 
-  private _getAuthorizationHeader: IPurposefulGetAuthorizationHeader;
-  private _getSpecificAuthorizationHeader: IPurposefulGetAuthorizationHeader;
+  private _getAuthorizationHeader: PurposefulGetAuthorizationHeader;
+  private _getSpecificAuthorizationHeader: PurposefulGetAuthorizationHeader;
   private _purpose: AppPurpose;
 
   constructor(
     organization: Organization,
     operations: IOperationsInstance,
-    getAuthorizationHeader: IPurposefulGetAuthorizationHeader,
-    getSpecificAuthorizationHeader: IPurposefulGetAuthorizationHeader
+    getAuthorizationHeader: PurposefulGetAuthorizationHeader,
+    getSpecificAuthorizationHeader: PurposefulGetAuthorizationHeader
   ) {
     this._getAuthorizationHeader = getAuthorizationHeader;
     this._getSpecificAuthorizationHeader = getSpecificAuthorizationHeader;
@@ -110,19 +110,16 @@ export class OrganizationDomains {
     }
   }
 
-  private authorize(purpose: AppPurpose = this._purpose): IGetAuthorizationHeader {
-    const getAuthorizationHeader = this._getAuthorizationHeader.bind(
-      this,
-      purpose
-    ) as IGetAuthorizationHeader;
+  private authorize(purpose: AppPurpose = this._purpose): GetAuthorizationHeader {
+    const getAuthorizationHeader = this._getAuthorizationHeader.bind(this, purpose) as GetAuthorizationHeader;
     return getAuthorizationHeader;
   }
 
-  private authorizeSpecificPurpose(purpose: AppPurposeTypes): IGetAuthorizationHeader | string {
+  private authorizeSpecificPurpose(purpose: AppPurposeTypes): GetAuthorizationHeader | string {
     const getAuthorizationHeader = this._getSpecificAuthorizationHeader.bind(
       this,
       purpose
-    ) as IGetAuthorizationHeader;
+    ) as GetAuthorizationHeader;
     return getAuthorizationHeader;
   }
 }

business/operations/core.ts

@@ -4,8 +4,13 @@
 //
 
 import { OrganizationSetting } from '../../entities/organizationSettings/organizationSetting';
-import { GitHubAppAuthenticationType, AppPurpose, ICustomAppPurpose, AppPurposeTypes } from '../githubApps';
-import { GitHubTokenManager } from '../githubApps/tokenManager';
+import {
+  GitHubAppAuthenticationType,
+  AppPurpose,
+  ICustomAppPurpose,
+  AppPurposeTypes,
+} from '../../lib/github/appPurposes';
+import { GitHubTokenManager } from '../../lib/github/tokenManager';
 import {
   IProviders,
   ICacheDefaultTimes,
@@ -19,9 +24,11 @@ import {
   throwIfNotGitHubCapable,
   throwIfNotCapable,
   IOperationsCentralOperationsToken,
-  IAuthorizationHeaderValue,
+  AuthorizationHeaderValue,
   SiteConfiguration,
   ExecutionEnvironment,
+  IPagedCacheOptions,
+  ICacheOptionsWithPurpose,
 } from '../../interfaces';
 import { RestLibrary } from '../../lib/github';
 import { CreateError } from '../../transitional';
@@ -32,6 +39,28 @@ import GitHubApplication from '../application';
 import Debug from 'debug';
 const debugGitHubTokens = Debug('github:tokens');
 
+const symbolCost = Symbol('cost');
+const symbolHeaders = Symbol('headers');
+
+export function symbolizeApiResponse<T>(response: any): T {
+  if (response && response.headers) {
+    response[symbolHeaders] = response.headers;
+    delete response.headers;
+  }
+  if (response && response.cost) {
+    response[symbolCost] = response.cost;
+    delete response.cost;
+  }
+  return response;
+}
+
+export function getApiSymbolMetadata(response: any) {
+  if (response) {
+    return { headers: response[symbolHeaders], cost: response[symbolCost] };
+  }
+  throw CreateError.ParameterRequired('response');
+}
+
 export interface IOperationsCoreOptions {
   github: RestLibrary;
   providers: IProviders;
@@ -63,6 +92,7 @@ export enum CacheDefault {
   teamDetailStaleSeconds = 'teamDetailStaleSeconds',
   orgRepoWebhooksStaleSeconds = 'orgRepoWebhooksStaleSeconds',
   teamRepositoryPermissionStaleSeconds = 'teamRepositoryPermissionStaleSeconds',
+  defaultStaleSeconds = 'defaultStaleSeconds',
 }
 
 // defaults could move to configuration alternatively
@@ -90,6 +120,7 @@ const defaults: ICacheDefaultTimes = {
   [CacheDefault.teamDetailStaleSeconds]: 60 * 60 * 2 /* 2h */,
   [CacheDefault.orgRepoWebhooksStaleSeconds]: 60 * 60 * 8 /* 8h */,
   [CacheDefault.teamRepositoryPermissionStaleSeconds]: 0 /* 0m */,
+  [CacheDefault.defaultStaleSeconds]: 60 /* 1m */,
 };
 
 export const DefaultPageSize = 100;
@@ -108,6 +139,46 @@ export function getPageSize(operations: IOperationsInstance, options?: IOptionWi
   return DefaultPageSize;
 }
 
+export function createCacheOptions(
+  operations: IOperationsInstance,
+  options?: ICacheOptions,
+  cacheDefault: CacheDefault = CacheDefault.defaultStaleSeconds
+) {
+  const cacheOptions: ICacheOptions = {
+    maxAgeSeconds: getMaxAgeSeconds(operations, cacheDefault, options, 60),
+  };
+  if (options.backgroundRefresh !== undefined) {
+    cacheOptions.backgroundRefresh = options.backgroundRefresh;
+  }
+  return cacheOptions;
+}
+
+export function createPagedCacheOptions(
+  operations: IOperationsInstance,
+  options?: IPagedCacheOptions,
+  cacheDefault: CacheDefault = CacheDefault.defaultStaleSeconds
+) {
+  const cacheOptions: IPagedCacheOptions = {
+    maxAgeSeconds: getMaxAgeSeconds(operations, cacheDefault, options, 60),
+  };
+  if (options.pageRequestDelay !== undefined) {
+    cacheOptions.pageRequestDelay = options.pageRequestDelay;
+  }
+  if (options.backgroundRefresh !== undefined) {
+    cacheOptions.backgroundRefresh = options.backgroundRefresh;
+  }
+  return cacheOptions;
+}
+
+export function popPurpose(options: ICacheOptionsWithPurpose, defaultPurpose: AppPurposeTypes) {
+  if (options.purpose) {
+    const purpose = options.purpose;
+    delete options.purpose;
+    return purpose;
+  }
+  return defaultPurpose;
+}
+
 export function getMaxAgeSeconds(
   operations: IOperationsInstance,
   cacheDefault: CacheDefault,
@@ -197,7 +268,7 @@ export abstract class OperationsCore
   async getAccountByUsername(username: string, options?: ICacheOptions): Promise<Account> {
     options = options || {};
     const operations = throwIfNotGitHubCapable(this);
-    const centralOperations = throwIfNotCapable<IOperationsCentralOperationsToken>(
+    const ops = throwIfNotCapable<IOperationsCentralOperationsToken>(
       this,
       CoreCapability.GitHubCentralOperations
     );
@@ -214,10 +285,9 @@ export abstract class OperationsCore
       cacheOptions.backgroundRefresh = options.backgroundRefresh;
     }
     try {
-      const getHeaderFunction = centralOperations.getCentralOperationsToken();
-      const authorizationHeader = await getHeaderFunction(AppPurpose.Data);
+      const getHeaderFunction = ops.getPublicAuthorizationToken();
       const entity = await operations.github.call(
-        authorizationHeader,
+        getHeaderFunction,
         'users.getByUsername',
         parameters,
         cacheOptions
@@ -337,17 +407,16 @@ export abstract class OperationsCore
     organizationName: string,
     organizationSettings: OrganizationSetting,
     legacyOwnerToken: string,
-    centralOperationsFallbackToken: string,
     appAuthenticationType: GitHubAppAuthenticationType,
     purpose: AppPurposeTypes
-  ): Promise<IAuthorizationHeaderValue> {
+  ): Promise<AuthorizationHeaderValue> {
     const customPurpose = purpose as ICustomAppPurpose;
     const isCustomPurpose = customPurpose?.isCustomAppPurpose === true;
     if (
       !isCustomPurpose &&
       !this.tokenManager.organizationSupportsAnyPurpose(organizationName, organizationSettings)
     ) {
-      const legacyTokenValue = legacyOwnerToken || centralOperationsFallbackToken;
+      const legacyTokenValue = legacyOwnerToken;
       if (!legacyTokenValue) {
         throw new Error(
           `Organization ${organizationName} is not configured with a GitHub app, Personal Access Token ownerToken configuration value, or a fallback central operations token for the ${
@@ -358,7 +427,7 @@ export abstract class OperationsCore
       return {
         value: `token ${legacyTokenValue}`,
         purpose: null,
-        source: legacyOwnerToken ? 'legacyOwnerToken' : 'centralOperationsFallbackToken',
+        source: 'legacyOwnerToken',
       };
     }
     if (!purpose) {