From 01c38df56b2dd0fa84745c0e5953702b559a5631 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Wed, 29 Jan 2025 14:16:48 -0500 Subject: [PATCH] changelog, more test cases --- CHANGES.rst | 15 +++ astroquery/mast/tests/data/README.rst | 12 +++ .../mast/tests/data/mast_relative_path.json | 10 ++ astroquery/mast/tests/test_mast.py | 96 ++++++++++++++++++- astroquery/mast/tests/test_mast_remote.py | 3 + astroquery/mast/utils.py | 3 +- 6 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 astroquery/mast/tests/data/mast_relative_path.json diff --git a/CHANGES.rst b/CHANGES.rst index 15ea2f63f7..e6cdd3ca4f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,14 @@ New Tools and Services ---------------------- +mast +^^^^ + +- Handle coordinates that are not in the ICRS frame in query functions. [#3164] + +- Handle a MAST URI string as input for ``Observations.get_cloud_uri`` and a list of MAST URIs as input for + ``Observations.get_cloud_uris``. [#3193] + Service fixes and enhancements ------------------------------ @@ -13,6 +21,13 @@ ipac.nexsci.nasa_exoplanet_archive - Fixed InvalidTableError for DI_STARS_EXEP and TD tables. [#3189] +mast +^^^^ + +- Bugfix where users are unnecessarily warned about a query limit while fetching products in ``MastMissions.get_product_list``. [#3193] + +- Bugfix where ``Observations.get_cloud_uri`` and ``Observations.get_cloud_uris`` fail if the MAST relative path is not found. [#3193] + Infrastructure, Utility and Other Changes and Additions ------------------------------------------------------- diff --git a/astroquery/mast/tests/data/README.rst b/astroquery/mast/tests/data/README.rst index 42001794db..a99f0123e2 100644 --- a/astroquery/mast/tests/data/README.rst +++ b/astroquery/mast/tests/data/README.rst @@ -36,3 +36,15 @@ To generate `~astroquery.mast.tests.data.mission_products.json`, use the followi >>> resp = utils._simple_request('https://mast.stsci.edu/search/hst/api/v0.1/list_products', {'dataset_ids': 'Z14Z0104T'}) >>> with open('panstarrs_columns.json', 'w') as file: ... json.dump(resp.json(), file, indent=4) # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.mast_relative_path.json`, use the following: + +.. doctest-remote-data:: + + >>> import json + >>> from astroquery.mast import utils + ... + >>> resp = utils._simple_request('https://mast.stsci.edu/api/v0.1/path_lookup/', + ... {'uri': ['mast:HST/product/u9o40504m_c3m.fits', 'mast:HST/product/does_not_exist.fits']}) + >>> with open('mast_relative_path.json', 'w') as file: + ... json.dump(resp.json(), file, indent=4) # doctest: +SKIP diff --git a/astroquery/mast/tests/data/mast_relative_path.json b/astroquery/mast/tests/data/mast_relative_path.json new file mode 100644 index 0000000000..3d7e110b8b --- /dev/null +++ b/astroquery/mast/tests/data/mast_relative_path.json @@ -0,0 +1,10 @@ +{ + "mast:HST/product/u9o40504m_c3m.fits": { + "status_code": 200, + "path": "/hst/public/u9o4/u9o40504m/u9o40504m_c3m.fits" + }, + "mast:HST/product/does_not_exist.fits": { + "status_code": 404, + "path": null + } +} \ No newline at end of file diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index b5532b48e0..d911881713 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -4,6 +4,7 @@ import os import re from shutil import copyfile +from unittest.mock import patch import pytest @@ -16,7 +17,8 @@ from astroquery.mast.services import _json_to_table from astroquery.utils.mocks import MockResponse -from astroquery.exceptions import InvalidQueryError, InputWarning, MaxResultsWarning +from astroquery.exceptions import (InvalidQueryError, InputWarning, MaxResultsWarning, NoResultsWarning, + RemoteServiceError) from astroquery import mast @@ -48,6 +50,7 @@ 'Mast.HscMatches.Db.v3': 'matchid.json', 'Mast.HscMatches.Db.v2': 'matchid.json', 'Mast.HscSpectra.Db.All': 'spectra.json', + 'mast_relative_path': 'mast_relative_path.json', 'panstarrs': 'panstarrs.json', 'panstarrs_columns': 'panstarrs_columns.json', 'tess_cutout': 'astrocut_107.27_-70.0_5x5.zip', @@ -142,6 +145,8 @@ def request_mockreturn(url, params={}): filename = data_path(DATA_FILES["Mast.Name.Lookup"]) elif 'panstarrs' in url: filename = data_path(DATA_FILES['panstarrs_columns']) + elif 'path_lookup' in url: + filename = data_path(DATA_FILES['mast_relative_path']) with open(filename, 'rb') as infile: content = infile.read() return MockResponse(content) @@ -678,6 +683,95 @@ def test_observations_download_file(patch_post, tmpdir): assert result == ('COMPLETE', None, None) +@patch('boto3.client') +def test_observations_get_cloud_uri(mock_client, patch_post): + pytest.importorskip("boto3") + + mast_uri = 'mast:HST/product/u9o40504m_c3m.fits' + expected = 's3://stpubdata/hst/public/u9o4/u9o40504m/u9o40504m_c3m.fits' + + # Error without cloud connection + with pytest.raises(RemoteServiceError): + mast.Observations.get_cloud_uri('mast:HST/product/u9o40504m_c3m.fits') + + # Enable access to public AWS S3 bucket + mast.Observations.enable_cloud_dataset() + + # Row input + product = Table() + product['dataURI'] = [mast_uri] + uri = mast.Observations.get_cloud_uri(product[0]) + assert isinstance(uri, str) + assert uri == expected + + # String input + uri = mast.Observations.get_cloud_uri(mast_uri) + assert uri == expected + + mast.Observations.disable_cloud_dataset() + + +@patch('boto3.client') +def test_observations_get_cloud_uris(mock_client, patch_post): + pytest.importorskip("boto3") + + mast_uri = 'mast:HST/product/u9o40504m_c3m.fits' + expected = 's3://stpubdata/hst/public/u9o4/u9o40504m/u9o40504m_c3m.fits' + + # Error without cloud connection + with pytest.raises(RemoteServiceError): + mast.Observations.get_cloud_uris(['mast:HST/product/u9o40504m_c3m.fits']) + + # Enable access to public AWS S3 bucket + mast.Observations.enable_cloud_dataset() + + # Get the cloud URIs + # Table input + product = Table() + product['dataURI'] = [mast_uri] + uris = mast.Observations.get_cloud_uris([mast_uri]) + assert isinstance(uris, list) + assert len(uris) == 1 + assert uris[0] == expected + + # List input + uris = mast.Observations.get_cloud_uris([mast_uri]) + assert isinstance(uris, list) + assert len(uris) == 1 + assert uris[0] == expected + + # Warn if attempting to filter with list input + with pytest.warns(InputWarning, match='Filtering is not supported'): + mast.Observations.get_cloud_uris([mast_uri], + extension='png') + + # Warn if not found + with pytest.warns(NoResultsWarning, match='Failed to retrieve MAST relative path'): + mast.Observations.get_cloud_uris(['mast:HST/product/does_not_exist.fits']) + + +@patch('boto3.client') +def test_observations_get_cloud_uris_query(mock_client, patch_post): + pytest.importorskip("boto3") + + # enable access to public AWS S3 bucket + mast.Observations.enable_cloud_dataset() + + # get uris with streamlined function + uris = mast.Observations.get_cloud_uris(target_name=234295610, + filter_products={'productSubGroupDescription': 'C3M'}) + assert isinstance(uris, list) + + # check that InvalidQueryError is thrown if neither data_products or **criteria are defined + with pytest.raises(InvalidQueryError): + mast.Observations.get_cloud_uris(filter_products={'productSubGroupDescription': 'C3M'}) + + # warn if no data products match filters + with pytest.warns(NoResultsWarning, match='No matching products'): + mast.Observations.get_cloud_uris(target_name=234295610, + filter_products={'productSubGroupDescription': 'LC'}) + + ###################### # CatalogClass tests # ###################### diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 90525ff030..1c864c936d 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -805,6 +805,7 @@ def test_observations_get_cloud_uris(self, test_obs_id): extension='png') def test_observations_get_cloud_uris_list_input(self): + pytest.importorskip("boto3") uri_list = ['mast:HST/product/u24r0102t_c1f.fits', 'mast:PS1/product/rings.v3.skycell.1334.061.stk.r.unconv.exp.fits'] expected = ['s3://stpubdata/hst/public/u24r/u24r0102t/u24r0102t_c1f.fits', @@ -851,6 +852,8 @@ def test_observations_get_cloud_uris_query(self): Observations.get_cloud_uris(target_name=234295611) def test_observations_get_cloud_uris_no_duplicates(self, msa_product_table): + pytest.importorskip("boto3") + # Get a product list with 6 duplicate JWST MSA config files products = msa_product_table diff --git a/astroquery/mast/utils.py b/astroquery/mast/utils.py index 2e70f44211..69755d4f30 100644 --- a/astroquery/mast/utils.py +++ b/astroquery/mast/utils.py @@ -185,7 +185,8 @@ def mast_relative_path(mast_uri): result = [] for chunk in uri_list_chunks: response = _simple_request("https://mast.stsci.edu/api/v0.1/path_lookup/", - {"uri": chunk}) + {"uri": [mast_uri[1] for mast_uri in chunk]}) + json_response = response.json() for uri in chunk: