Merge pull request #3885 from terascope/typescript-implicit-any-index…

…-errors

[all] Update to TypeScript v5.7.2, remove "suppressImplicitAnyIndexErrors" rule

e2e/package.json

@@ -43,9 +43,9 @@
         "ms": "~2.1.3"
     },
     "devDependencies": {
-        "@terascope/types": "~1.3.2",
+        "@terascope/types": "~1.4.0",
         "bunyan": "~1.8.15",
-        "elasticsearch-store": "~1.6.0",
+        "elasticsearch-store": "~1.7.0",
         "fs-extra": "~11.2.0",
         "ms": "~2.1.3",
         "nanoid": "~5.0.9",

package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "2.10.0",
+    "version": "2.11.0",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {
@@ -67,10 +67,10 @@
         "jest-extended": "~4.0.2",
         "jest-watch-typeahead": "~2.2.2",
         "node-notifier": "~10.0.1",
-        "patch-package": "^8.0.0",
-        "postinstall-postinstall": "^2.1.0",
+        "patch-package": "~8.0.0",
+        "postinstall-postinstall": "~2.1.0",
         "ts-jest": "~29.2.5",
-        "typescript": "~5.2.2"
+        "typescript": "~5.7.2"
     },
     "engines": {
         "node": ">=18.18.0",

packages/data-mate/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-mate",
     "displayName": "Data-Mate",
-    "version": "1.6.0",
+    "version": "1.7.0",
     "description": "Library of data validations/transformations",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme",
     "repository": {
@@ -30,9 +30,9 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-types": "~1.6.0",
-        "@terascope/types": "~1.3.2",
-        "@terascope/utils": "~1.6.0",
+        "@terascope/data-types": "~1.7.0",
+        "@terascope/types": "~1.4.0",
+        "@terascope/utils": "~1.7.0",
         "@types/validator": "~13.12.2",
         "awesome-phonenumber": "~7.2.0",
         "date-fns": "~4.1.0",
@@ -45,7 +45,7 @@
         "uuid": "~11.0.3",
         "valid-url": "~1.0.9",
         "validator": "~13.12.0",
-        "xlucene-parser": "~1.6.0"
+        "xlucene-parser": "~1.7.0"
     },
     "devDependencies": {
         "@types/ip6addr": "~0.2.6",

packages/data-mate/src/index.ts

@@ -29,6 +29,7 @@ declare module './aggregation-frame/AggregationFrame.d.ts' {
          * @returns the new columns
         */
         run(): Promise<DataFrame<T>>;
+        [key: string | symbol]: unknown;
     }
 }
 

packages/data-types/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-types",
     "displayName": "Data Types",
-    "version": "1.6.0",
+    "version": "1.7.0",
     "description": "A library for defining the data structures and mapping",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme",
     "bugs": {
@@ -27,8 +27,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "~1.3.2",
-        "@terascope/utils": "~1.6.0",
+        "@terascope/types": "~1.4.0",
+        "@terascope/utils": "~1.7.0",
         "graphql": "~16.9.0",
         "yargs": "~17.7.2"
     },

packages/elasticsearch-api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "4.6.0",
+    "version": "4.7.0",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -24,16 +24,16 @@
         "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "~1.3.2",
-        "@terascope/utils": "~1.6.0",
+        "@terascope/types": "~1.4.0",
+        "@terascope/utils": "~1.7.0",
         "bluebird": "~3.7.2",
         "setimmediate": "~1.0.5"
     },
     "devDependencies": {
         "@opensearch-project/opensearch": "~1.2.0",
         "@types/elasticsearch": "~5.0.43",
         "elasticsearch": "~15.4.1",
-        "elasticsearch-store": "~1.6.0",
+        "elasticsearch-store": "~1.7.0",
         "elasticsearch6": "npm:@elastic/elasticsearch@~6.8.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@~7.17.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@~8.15.0"

packages/elasticsearch-store/package.json

@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-store",
     "displayName": "Elasticsearch Store",
-    "version": "1.6.0",
+    "version": "1.7.0",
     "description": "An API for managing an elasticsearch index, with versioning and migration support.",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
     "bugs": {
@@ -30,10 +30,10 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-mate": "~1.6.0",
-        "@terascope/data-types": "~1.6.0",
-        "@terascope/types": "~1.3.2",
-        "@terascope/utils": "~1.6.0",
+        "@terascope/data-mate": "~1.7.0",
+        "@terascope/data-types": "~1.7.0",
+        "@terascope/types": "~1.4.0",
+        "@terascope/utils": "~1.7.0",
         "ajv": "~8.17.1",
         "ajv-formats": "~3.0.1",
         "elasticsearch6": "npm:@elastic/elasticsearch@~6.8.0",
@@ -43,7 +43,7 @@
         "opensearch2": "npm:@opensearch-project/opensearch@~2.12.0",
         "setimmediate": "~1.0.5",
         "uuid": "~11.0.3",
-        "xlucene-translator": "~1.6.0"
+        "xlucene-translator": "~1.7.0"
     },
     "devDependencies": {
         "@types/uuid": "~10.0.0"

packages/elasticsearch-store/src/elasticsearch-client/method-helpers/search.ts

@@ -41,7 +41,7 @@ export function convertSearchParams(
 }
 
 function qDependentFieldsCheck(params: ClientParams.SearchParams) {
-    const requiresQ = [
+    const requiresQ: Array<keyof ClientParams.SearchParams> = [
         'analyzer',
         'analyze_wildcard',
         'default_operator',

packages/elasticsearch-store/src/index-model.ts

@@ -1,7 +1,7 @@
 import {
     isTest, debugLogger, concat,
     Logger, makeISODate, toSafeString,
-    trim, trimAndToLower, TSError
+    trim, trimAndToLower, TSError, isKey
 } from '@terascope/utils';
 import { JoinBy } from '@terascope/data-mate';
 import { QueryAccess, RestrictOptions } from 'xlucene-translator';
@@ -23,7 +23,7 @@ export abstract class IndexModel<T extends i.IndexModelRecord> extends IndexStor
     readonly logger: Logger;
 
     private _uniqueFields: readonly (keyof T)[];
-    private _sanitizeFields: i.SanitizeFields;
+    private _sanitizeFields: i.SanitizeFields<T> | undefined;
 
     constructor(
         client: Client,
@@ -85,7 +85,9 @@ export abstract class IndexModel<T extends i.IndexModelRecord> extends IndexStor
         this.logger = options.logger || debugLogger(debugLoggerName);
 
         this._uniqueFields = concat('_key', uniqueFields);
-        this._sanitizeFields = sanitizeFields || {};
+        if (sanitizeFields) {
+            this._sanitizeFields = sanitizeFields;
+        }
 
         this.readHooks.add((doc) => {
             if (doc._deleted) return false;
@@ -212,24 +214,46 @@ export abstract class IndexModel<T extends i.IndexModelRecord> extends IndexStor
         return count === ids.length;
     }
 
+    private _setStringValueOrThrow<K extends keyof T>(obj: T, key: K, value: string) {
+        if (typeof obj[key] === 'string') {
+            obj[key] = value as T[K];
+        } else {
+            throw new Error(`Cannot assign a string to a property of type ${typeof obj[key]}`);
+        }
+    }
+
     protected _sanitizeRecord(record: T): T {
-        const entries = Object.entries(this._sanitizeFields);
-
-        for (const [field, method] of entries) {
-            if (!record[field]) continue;
-
-            switch (method) {
-                case 'trim':
-                    record[field] = trim(record[field]);
-                    break;
-                case 'trimAndToLower':
-                    record[field] = trimAndToLower(record[field]);
-                    break;
-                case 'toSafeString':
-                    record[field] = toSafeString(record[field]);
-                    break;
-                default:
-                    continue;
+        if (this._sanitizeFields) {
+            const entries = Object.entries(this._sanitizeFields);
+
+            for (const [field, method] of entries) {
+                if (isKey(record, field) && record[field]) {
+                    switch (method) {
+                        case 'trim':
+                            this._setStringValueOrThrow(
+                                record,
+                                field,
+                                trim(record[field])
+                            );
+                            break;
+                        case 'trimAndToLower':
+                            this._setStringValueOrThrow(
+                                record,
+                                field,
+                                trimAndToLower(record[field] as string)
+                            );
+                            break;
+                        case 'toSafeString':
+                            this._setStringValueOrThrow(
+                                record,
+                                field,
+                                toSafeString(record[field] as string)
+                            );
+                            break;
+                        default:
+                            continue;
+                    }
+                }
             }
         }
 

packages/elasticsearch-store/src/interfaces.ts

@@ -245,7 +245,7 @@ export type IndexModelConfig<T extends IndexModelRecord> = Omit<
     unique_fields?: (keyof T)[];
 
     /** Sanitize / cleanup fields mapping, like trim or trimAndToLower */
-    sanitize_fields?: SanitizeFields;
+    sanitize_fields?: SanitizeFields<T>;
 
     /** Specify whether the data should be strictly validated, defaults to true */
     strict_mode?: boolean;
@@ -256,8 +256,8 @@ export type IndexModelConfig<T extends IndexModelRecord> = Omit<
     timeseries?: boolean;
 };
 
-export type SanitizeFields = {
-    [field: string]: 'trimAndToLower' | 'trim' | 'toSafeString';
+export type SanitizeFields<T> = {
+    [field in string & keyof T]: 'trimAndToLower' | 'trim' | 'toSafeString';
 };
 
 /**

packages/elasticsearch-store/src/utils/elasticsearch.ts

@@ -246,7 +246,7 @@ export type FlattenProperties = Record<string, [type: ESFieldType, extra?: strin
 export function getFlattenedNamesAndTypes(config: ESTypeMapping): FlattenProperties {
     const output: FlattenProperties = Object.create(null);
     for (const field of Object.keys(config).sort()) {
-        const { type: _type, properties, ...extra } = config[field];
+        const { type: _type, properties, ...extra } = config[field as keyof ESTypeMapping];
 
         // if there is no type, elasticsearch returns "undefined" for the type
         // but this will cause conflicts, we should set it to "object"

packages/elasticsearch-store/src/utils/model.ts

@@ -2,7 +2,8 @@ import { FieldType, DataTypeFields } from '@terascope/types';
 import { DataType, LATEST_VERSION } from '@terascope/data-types';
 import {
     cloneDeep, isPlainObject, concat,
-    getWordParts, firstToUpper, isNumberLike
+    getWordParts, firstToUpper, isNumberLike,
+    isKey
 } from '@terascope/utils';
 
 /** JSON Schema */
@@ -57,18 +58,18 @@ export function addDefaultSchema(input: Record<string, any>): Record<string, any
 /**
  * Deep copy two levels deep (useful for mapping and schema)
  */
-export function mergeDefaults<T>(source: T, from: Partial<T>): T {
+export function mergeDefaults<T extends object>(source: T, from: Partial<T>): T {
     const output = cloneDeep(source);
     const _mapping = from ? cloneDeep(from) : {};
 
     for (const [key, val] of Object.entries(_mapping)) {
-        if (output[key] != null) {
+        if (isKey(output, key) && output[key] != null) {
             if (isPlainObject(val)) {
                 output[key] = Object.assign(output[key], val);
             } else if (Array.isArray(val)) {
-                output[key] = concat(output[key], val);
+                output[key] = concat(output[key], val) as T[string & keyof T];
             } else {
-                output[key] = val;
+                output[key] = val as T[string & keyof T];
             }
         }
     }