Skip to content

Commit

Permalink
fix: skip deprecated rows and trim all values
Browse files Browse the repository at this point in the history
  • Loading branch information
stdavis committed Mar 25, 2024
1 parent 4d0803d commit 63af879
Showing 1 changed file with 41 additions and 11 deletions.
52 changes: 41 additions & 11 deletions src/scripts/validate-sgid-index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,21 @@ const spreadsheet = new GoogleSpreadsheet(spreadsheetId, client);
await spreadsheet.loadInfo();
const worksheet = spreadsheet.sheetsByTitle['SGID Index'];

const fieldNames = {
id: 'id',
displayName: 'displayName',
openSgid: 'openSgid',
openSgidTableName: 'openSgidTableName',
productPage: 'productPage',
itemId: 'itemId',
serverLayerId: 'serverLayerId',
};

const errors = [];
function recordError(message, row) {
errors.push({
displayName: row.get('displayName'),
id: row.get('id'),
displayName: row.get(fieldNames.displayName),
id: row.get(fieldNames.id),
Error: message,
});
}
Expand All @@ -43,11 +53,11 @@ function toBoolean(value) {
}

async function openSGIDTableName(row) {
if (!toBoolean(row.get('openSgid'))) {
if (!toBoolean(row.get(fieldNames.openSgid))) {
return;
}

const cellValue = row.get('openSgidTableName');
const cellValue = row.get(fieldNames.openSgidTableName);
if (!cellValue) {
recordError('openSgidTableName is empty', row);
return;
Expand All @@ -61,8 +71,22 @@ async function openSGIDTableName(row) {
}
}

function trimFields(row) {
let changed = false;

for (const field in fieldNames) {
const cellValue = row.get(fieldNames[field]);
if (cellValue && cellValue.trim() !== cellValue) {
row.set(fieldNames[field], cellValue.trim());
changed = true;
}
}

return changed;
}

async function productPage(row) {
const cellValue = row.get('productPage');
const cellValue = row.get(fieldNames.productPage);

if (!cellValue) {
// productPage is not a required field
Expand All @@ -83,7 +107,7 @@ async function productPage(row) {
}

async function idGuid(row) {
const cellValue = row.get('id');
const cellValue = row.get(fieldNames.id);

if (!cellValue) {
row.set('id', uuid());
Expand All @@ -93,7 +117,7 @@ async function idGuid(row) {
}

async function itemId(row) {
const cellValue = row.get('itemId')?.trim();
const cellValue = row.get(fieldNames.itemId)?.trim();

if (!cellValue) {
// TODO: internal datasets _should_ have an itemId
Expand All @@ -109,7 +133,9 @@ async function itemId(row) {

let hubData;
try {
hubData = await ky(`https://opendata.arcgis.com/api/v3/datasets/${cellValue}_${row.get('serverLayerId')}`).json();
hubData = await ky(
`https://opendata.arcgis.com/api/v3/datasets/${cellValue}_${row.get(fieldNames.serverLayerId)}`,
).json();
} catch (error) {
recordError(`itemId hub request error: ${error.message}`, row);
return;
Expand Down Expand Up @@ -148,7 +174,7 @@ async function duplicates(row) {
}

async function downloadMetadataCheck(row) {
const name = row.get('hubName');
const name = row.get(fieldNames.hubName);
const metadata = downloadMetadata.dataPages[name];

if (!metadata || metadata.featureServiceId === null) {
Expand Down Expand Up @@ -222,13 +248,17 @@ const skipStatuses = ['deprecated', 'shelved'];
for (const row of rows) {
progressBar.tick();

if (skipStatuses.includes(row.get('ugrcStatus')?.toLowerCase())) {
if (skipStatuses.includes(row.get(fieldNames.ugrcStatus)?.toLowerCase())) {
continue;
}

let changed;
try {
changed = (await Promise.all(checks.map((check) => check(row)))).some((result) => result);
changed = trimFields(row); // we want trim to run first

const checksChanged = (await Promise.all(checks.map((check) => check(row)))).some((result) => result);

changed = changed || checksChanged;
} catch (error) {
recordError(error.message, row);
}
Expand Down

0 comments on commit 63af879

Please sign in to comment.