changelog, more test cases

CHANGES.rst

@@ -4,6 +4,14 @@
 New Tools and Services
 ----------------------
 
+mast
+^^^^
+
+- Handle coordinates that are not in the ICRS frame in query functions. [#3164]
+
+- Handle a MAST URI string as input for ``Observations.get_cloud_uri`` and a list of MAST URIs as input for
+  ``Observations.get_cloud_uris``. [#3193]
+
 
 Service fixes and enhancements
 ------------------------------
@@ -13,6 +21,13 @@ ipac.nexsci.nasa_exoplanet_archive
 
 - Fixed InvalidTableError for DI_STARS_EXEP and TD tables. [#3189]
 
+mast
+^^^^
+
+- Bugfix where users are unnecessarily warned about a query limit while fetching products in ``MastMissions.get_product_list``. [#3193]
+
+- Bugfix where ``Observations.get_cloud_uri`` and ``Observations.get_cloud_uris`` fail if the MAST relative path is not found. [#3193]
+
 
 Infrastructure, Utility and Other Changes and Additions
 -------------------------------------------------------

astroquery/mast/tests/data/README.rst

@@ -36,3 +36,15 @@ To generate `~astroquery.mast.tests.data.mission_products.json`, use the followi
     >>> resp = utils._simple_request('https://mast.stsci.edu/search/hst/api/v0.1/list_products', {'dataset_ids': 'Z14Z0104T'})
     >>> with open('panstarrs_columns.json', 'w') as file:
     ...     json.dump(resp.json(), file, indent=4)  # doctest: +SKIP
+
+To generate `~astroquery.mast.tests.data.mast_relative_path.json`, use the following:
+
+.. doctest-remote-data::
+
+    >>> import json
+    >>> from astroquery.mast import utils
+    ...
+    >>> resp = utils._simple_request('https://mast.stsci.edu/api/v0.1/path_lookup/',
+    ...                              {'uri': ['mast:HST/product/u9o40504m_c3m.fits', 'mast:HST/product/does_not_exist.fits']})
+    >>> with open('mast_relative_path.json', 'w') as file:
+    ...     json.dump(resp.json(), file, indent=4)  # doctest: +SKIP

astroquery/mast/tests/data/mast_relative_path.json

@@ -0,0 +1,10 @@
+{
+    "mast:HST/product/u9o40504m_c3m.fits": {
+        "status_code": 200,
+        "path": "/hst/public/u9o4/u9o40504m/u9o40504m_c3m.fits"
+    },
+    "mast:HST/product/does_not_exist.fits": {
+        "status_code": 404,
+        "path": null
+    }
+}

astroquery/mast/tests/test_mast.py

@@ -4,6 +4,7 @@
 import os
 import re
 from shutil import copyfile
+from unittest.mock import patch
 
 import pytest
 
@@ -16,7 +17,8 @@
 
 from astroquery.mast.services import _json_to_table
 from astroquery.utils.mocks import MockResponse
-from astroquery.exceptions import InvalidQueryError, InputWarning, MaxResultsWarning
+from astroquery.exceptions import (InvalidQueryError, InputWarning, MaxResultsWarning, NoResultsWarning,
+                                   RemoteServiceError)
 
 from astroquery import mast
 
@@ -48,6 +50,7 @@
               'Mast.HscMatches.Db.v3': 'matchid.json',
               'Mast.HscMatches.Db.v2': 'matchid.json',
               'Mast.HscSpectra.Db.All': 'spectra.json',
+              'mast_relative_path': 'mast_relative_path.json',
               'panstarrs': 'panstarrs.json',
               'panstarrs_columns': 'panstarrs_columns.json',
               'tess_cutout': 'astrocut_107.27_-70.0_5x5.zip',
@@ -142,6 +145,8 @@ def request_mockreturn(url, params={}):
         filename = data_path(DATA_FILES["Mast.Name.Lookup"])
     elif 'panstarrs' in url:
         filename = data_path(DATA_FILES['panstarrs_columns'])
+    elif 'path_lookup' in url:
+        filename = data_path(DATA_FILES['mast_relative_path'])
     with open(filename, 'rb') as infile:
         content = infile.read()
     return MockResponse(content)
@@ -678,6 +683,95 @@ def test_observations_download_file(patch_post, tmpdir):
     assert result == ('COMPLETE', None, None)
 
 
+@patch('boto3.client')
+def test_observations_get_cloud_uri(mock_client, patch_post):
+    pytest.importorskip("boto3")
+
+    mast_uri = 'mast:HST/product/u9o40504m_c3m.fits'
+    expected = 's3://stpubdata/hst/public/u9o4/u9o40504m/u9o40504m_c3m.fits'
+
+    # Error without cloud connection
+    with pytest.raises(RemoteServiceError):
+        mast.Observations.get_cloud_uri('mast:HST/product/u9o40504m_c3m.fits')
+
+    # Enable access to public AWS S3 bucket
+    mast.Observations.enable_cloud_dataset()
+
+    # Row input
+    product = Table()
+    product['dataURI'] = [mast_uri]
+    uri = mast.Observations.get_cloud_uri(product[0])
+    assert isinstance(uri, str)
+    assert uri == expected
+
+    # String input
+    uri = mast.Observations.get_cloud_uri(mast_uri)
+    assert uri == expected
+
+    mast.Observations.disable_cloud_dataset()
+
+
+@patch('boto3.client')
+def test_observations_get_cloud_uris(mock_client, patch_post):
+    pytest.importorskip("boto3")
+
+    mast_uri = 'mast:HST/product/u9o40504m_c3m.fits'
+    expected = 's3://stpubdata/hst/public/u9o4/u9o40504m/u9o40504m_c3m.fits'
+
+    # Error without cloud connection
+    with pytest.raises(RemoteServiceError):
+        mast.Observations.get_cloud_uris(['mast:HST/product/u9o40504m_c3m.fits'])
+
+    # Enable access to public AWS S3 bucket
+    mast.Observations.enable_cloud_dataset()
+
+    # Get the cloud URIs
+    # Table input
+    product = Table()
+    product['dataURI'] = [mast_uri]
+    uris = mast.Observations.get_cloud_uris([mast_uri])
+    assert isinstance(uris, list)
+    assert len(uris) == 1
+    assert uris[0] == expected
+
+    # List input
+    uris = mast.Observations.get_cloud_uris([mast_uri])
+    assert isinstance(uris, list)
+    assert len(uris) == 1
+    assert uris[0] == expected
+
+    # Warn if attempting to filter with list input
+    with pytest.warns(InputWarning, match='Filtering is not supported'):
+        mast.Observations.get_cloud_uris([mast_uri],
+                                         extension='png')
+
+    # Warn if not found
+    with pytest.warns(NoResultsWarning, match='Failed to retrieve MAST relative path'):
+        mast.Observations.get_cloud_uris(['mast:HST/product/does_not_exist.fits'])
+
+
+@patch('boto3.client')
+def test_observations_get_cloud_uris_query(mock_client, patch_post):
+    pytest.importorskip("boto3")
+
+    # enable access to public AWS S3 bucket
+    mast.Observations.enable_cloud_dataset()
+
+    # get uris with streamlined function
+    uris = mast.Observations.get_cloud_uris(target_name=234295610,
+                                            filter_products={'productSubGroupDescription': 'C3M'})
+    assert isinstance(uris, list)
+
+    # check that InvalidQueryError is thrown if neither data_products or **criteria are defined
+    with pytest.raises(InvalidQueryError):
+        mast.Observations.get_cloud_uris(filter_products={'productSubGroupDescription': 'C3M'})
+
+    # warn if no data products match filters
+    with pytest.warns(NoResultsWarning, match='No matching products'):
+        mast.Observations.get_cloud_uris(target_name=234295610,
+                                         filter_products={'productSubGroupDescription': 'LC'})
+
+
 ######################
 # CatalogClass tests #
 ######################

astroquery/mast/tests/test_mast_remote.py

@@ -805,6 +805,7 @@ def test_observations_get_cloud_uris(self, test_obs_id):
                                         extension='png')
 
     def test_observations_get_cloud_uris_list_input(self):
+        pytest.importorskip("boto3")
         uri_list = ['mast:HST/product/u24r0102t_c1f.fits',
                     'mast:PS1/product/rings.v3.skycell.1334.061.stk.r.unconv.exp.fits']
         expected = ['s3://stpubdata/hst/public/u24r/u24r0102t/u24r0102t_c1f.fits',
@@ -851,6 +852,8 @@ def test_observations_get_cloud_uris_query(self):
             Observations.get_cloud_uris(target_name=234295611)
 
     def test_observations_get_cloud_uris_no_duplicates(self, msa_product_table):
+        pytest.importorskip("boto3")
+
         # Get a product list with 6 duplicate JWST MSA config files
         products = msa_product_table
 

astroquery/mast/utils.py

@@ -185,7 +185,8 @@ def mast_relative_path(mast_uri):
     result = []
     for chunk in uri_list_chunks:
         response = _simple_request("https://mast.stsci.edu/api/v0.1/path_lookup/",
-                                   {"uri": chunk})
+                                   {"uri": [mast_uri[1] for mast_uri in chunk]})
+
         json_response = response.json()
 
         for uri in chunk: