Merge pull request #108 from NYPL/NOREF-identify-ids-by-sierra

Add [wip] jidentify-ids-by-sierra-query

config/production-bulk-index.env

@@ -6,6 +6,7 @@ NYPL_OAUTH_SECRET=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAIcwgYQGCS
 ENCRYPTED_ELASTICSEARCH_URI=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAJYwgZMGCSqGSIb3DQEHBqCBhTCBggIBADB9BgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFWw8ECX9Pz81z0kvAIBEIBQGec9PCpwuvEgLH6imhqP6tx1fj8Vlf2ZipnUy06jzmpE262Qvk9LPAq7sIYPVkTCZctwilwcU9oC6yxasVoUlK87la77v03CeZsPIDwciFY=
 ENCRYPTED_ELASTICSEARCH_API_KEY=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAJ4wgZsGCSqGSIb3DQEHBqCBjTCBigIBADCBhAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAx+oI1oizn/wsntbkoCARCAV+yRGmMGm4f+1fFRw9pDR4loT1LQrEEh0f6Yxavt8xGIyu77iyLboTHkXdg/s/y2OOM07crt9MEhDfbeEvlU0rjuyDS/bugNaZvsUrd/cu4gwUUZJswbkA==
 ELASTIC_RESOURCES_INDEX_NAME=resources-2024-10-22
+ES_VERSION=8
 
 LOG_LEVEL=info
 STREAM_ENVIRONMENT=production
@@ -26,3 +27,8 @@ HOLDINGS_SERVICE_DB_PW=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHIwc
 HOLDINGS_SERVICE_DB_USER=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGswaQYJKoZIhvcNAQcGoFwwWgIBADBVBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHONL7PAQLdPrClNiQIBEIAo4WacUaBZn5I6veE4qgJVDNqVuC2JuObjbgYzAT98ywhpXd76gg699A==
 HOLDINGS_SERVICE_DB_HOST=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAJcwgZQGCSqGSIb3DQEHBqCBhjCBgwIBADB+BgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFRq7HQ2Lk6gfCdVxAIBEIBRdKyNsdHtfTC4l6RckfyOjzPhplNFnUVg13UwTygKvi9Ouw+9rKGQ1pFCSveUrSYpCgZnn8ErBhyxZ195Tt06mJDVdUHVNBTUWP3WeuTCAZtM
 HOLDINGS_SERVICE_DB_NAME=holdings
+
+# For use with scripts/identify-ids-by-sierra-query:
+SIERRA_BASE=https://ilsstaff.nypl.org:443/iii/sierra-api/v6
+SIERRA_KEY=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHoweAYJKoZIhvcNAQcGoGswaQIBADBkBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBqMn6CoPoFs7+mkAgIBEIA3xn2bR+eXvq2hyeX9qf9vWrY6hWlN4mkJXjNO0RHj+OepwmkZgOBVvkYwXSNrk6wP7lwuEE/yuQ==
+SIERRA_SECRET=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAIcwgYQGCSqGSIb3DQEHBqB3MHUCAQAwcAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyFHPRx3UmwbT7kuMICARCAQ6Uu9rxy78WLnkoCaAXyAnXzlcUpRnpY8d0ZsDzaLhtiy4l3mUzHvGGFq6OzSa+KPgfvXxbjfvAMBaZDPG/bW1UuLSE=

config/qa-bulk-index.env

@@ -2,8 +2,12 @@ NYPL_API_BASE_URL=https://qa-platform.nypl.org/api/v0.1/
 NYPL_OAUTH_URL=https://isso.nypl.org/
 NYPL_OAUTH_KEY=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGswaQYJKoZIhvcNAQcGoFwwWgIBADBVBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDNQozUGkaz8WYD2lUAIBEIAo/SzNMA9LowO6gcnTUCcMjBaAU1RH/L3EAS14fjJCUpyZppkuEDUd7w==
 NYPL_OAUTH_SECRET=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAIcwgYQGCSqGSIb3DQEHBqB3MHUCAQAwcAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxN3indXvk2ueiE6CwCARCAQ018FdIVXXwfTuKH1vp/ZTfjBinxKTDosMmzyWB9/CtiFtgOu09iiyZEpC3AyGOt8ExywHZoHOZQuLdGGNFgbusmldw=
-ELASTICSEARCH_CONNECTION_URI=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAALcwgbQGCSqGSIb3DQEHBqCBpjCBowIBADCBnQYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwLEPDqsqAvuJdR8ckCARCAcKo/xhIswLW0MazFXGYVn+qrAFMOTEOzUiMsLNAzHJ7n4Fi1AJztcQSHIOgizHZuDXM0ULUebUaES8mmOwsH8pxi30UmX5/rtTakE5/dbVKZN7yvzt7FsEMlhfhBbloTDK0HaQcdsnL2BcnwxztXRUQ=
-ELASTIC_RESOURCES_INDEX_NAME=resources-2024-04-15
+
+ENCRYPTED_ELASTICSEARCH_URI=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAJYwgZMGCSqGSIb3DQEHBqCBhTCBggIBADB9BgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMIkDoQ9C/cCDCAq1wIBEIBQ+L3OgUGeOW9rs1CWkhpBjwM4LbbVRFIWedqew4UXIeSNMJ8cO9SNe4YGCUIoKwCDYt7W7ip3VtDRRRMVvz6QJw+Eg8ugTMVs2pbNFGNvaAQ=
+ENCRYPTED_ELASTICSEARCH_API_KEY=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAJ4wgZsGCSqGSIb3DQEHBqCBjTCBigIBADCBhAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxFghFNijuLqfqLD9gCARCAVyNd/BIq2x2op/mEoOL7/yNqUX51nM/fmZJ4BswSkofXSeOZFVfdE5Jf0vJLXGYZX164DdfWz7VHpYLUye+CtLztL2HBeLzcYUJNqYQzH31l0go3Cs6Drw==
+ELASTIC_RESOURCES_INDEX_NAME=resources-2024-10-17
+ES_VERSION=8
+
 LOG_LEVEL=info
 STREAM_ENVIRONMENT=qa
 SCSB_URL=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAH8wfQYJKoZIhvcNAQcGoHAwbgIBADBpBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDG1u2jd2nCAGR/6jAgIBEIA8BTTHJsdRqHKYyVW1EqiVklaFbPJ4w+0osLcpIWp/JAljVl4AHrhQk0oUVqylK0a/nSjfLlz2+UwZqe7A
@@ -24,3 +28,8 @@ HOLDINGS_SERVICE_DB_PW=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAG8wb
 HOLDINGS_SERVICE_DB_USER=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGswaQYJKoZIhvcNAQcGoFwwWgIBADBVBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDIB8EpIsaJMl3BzfygIBEIAoRTruPiXba5K0ZHUfEBKv+vVA2PGep53pE/j8ziiU6qgJOkaBeWuerA==
 HOLDINGS_SERVICE_DB_HOST=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAJUwgZIGCSqGSIb3DQEHBqCBhDCBgQIBADB8BgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBcVigY1/5EGzU4i0QIBEIBPcld4+sa3s1PnlAOYFAbos+bUW3oDHUOViJWqfIyn7FDHtaSJWMXOZHNo4o6RMNI8jV/mE9suLUMG/i+Jk8sZd5wuAavECkWGFxTEve9hQg==
 HOLDINGS_SERVICE_DB_NAME=holdings
+
+# For use with scripts/identify-ids-by-sierra-query:
+SIERRA_BASE=https://nypl-sierra-test.nypl.org:443/iii/sierra-api/v6
+SIERRA_KEY=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHoweAYJKoZIhvcNAQcGoGswaQIBADBkBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJGDDlDYRAgm6Kq1CAIBEIA37Kuya0YgwuI3MPf83rCxeV75tN6Nh4pPGOIt5YInT+kdD89xTO3D/6B9OlK6T3+MWtymsJ+/Kg==
+SIERRA_SECRET=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGowaAYJKoZIhvcNAQcGoFswWQIBADBUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDKmT3cEELMoZ2huyrwIBEIAnibbT57jJuV6ah2iATkcglCYHhQze8aQ291egun6IOpv6NTAMJ4Hv

package.json

@@ -47,6 +47,7 @@
   },
   "devDependencies": {
     "@aws-sdk/credential-providers": "^3.614.0",
+    "@nypl/sierra-wrapper": "^1.1.1",
     "chai": "^4.3.7",
     "chai-as-promised": "^7.1.1",
     "csv-parser": "^3.0.0",

scripts/bulk-index.js

@@ -570,7 +570,7 @@ const processCsvBatch = async (batches, index = 0, options) => {
   )
 
   // Log out progress so far:
-  printProgress(options.count + batch.length, options.total, options.startTime)
+  printProgress(options.count + batch.length, options.total, options.batchSize, options.startTime)
 
   if (batches.length > index + 1) {
     // Update `count` (for progress stats):