diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 0a80a686c0..a1f97b9dbe 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -27,12 +27,6 @@ jobs: python: 3.x toxenv: codestyle - - name: docs build - os: ubuntu-latest - python: 3.8 - toxenv: build_docs - toxargs: -v - - name: oldest dependencies os: ubuntu-latest python: 3.7 @@ -41,20 +35,20 @@ jobs: - name: astropy dev with all dependencies with coverage os: ubuntu-latest - python: 3.9 - toxenv: py39-test-alldeps-devastropy-cov + python: '3.10' + toxenv: py310-test-alldeps-devastropy-cov toxargs: -v - - name: Python 3.7 with all optional dependencies (MacOS X) + - name: Python 3.8 with all optional dependencies (MacOS X) os: macos-latest - python: 3.7 - toxenv: py37-test-alldeps + python: 3.8 + toxenv: py38-test-alldeps toxargs: -v - - name: Python 3.8 with mandatory dependencies (Windows) + - name: Python 3.9 with mandatory dependencies (Windows) os: windows-latest - python: 3.8 - toxenv: py38-test + python: 3.9 + toxenv: py39-test toxargs: -v steps: diff --git a/.gitignore b/.gitignore index 254786eb63..7156719927 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ MANIFEST .ipynb_checkpoints .eggs pip-wheel-metadata +.hypothesis # Sphinx _build diff --git a/.mailmap b/.mailmap index 164bb4f7b3..9af20e3937 100644 --- a/.mailmap +++ b/.mailmap @@ -5,24 +5,32 @@ Adrian Damian Andrew O'Brien Austen Groener Austen Groener +Ayush Yadav Benjamin Alan Weaver +Benjamin Alan Weaver Brigitta Sipőcz Brigitta Sipőcz -Clara Brasseur +Caden Armstrong +Clara Brasseur +Clara Brasseur David Collom David Collom David Collom E. Madison Bray E. Madison Bray Edward Gomez +Elena Colomo Eric Koch +Erwan Pannier Fran Raga Fred Moolekamp Magnus Persson Hans Moritz Guenter Henrik Norman Henrik Norman Jaladh Singhal +Javier Ballester Javier Duran +Javier Duran Javier Duran Javier Duran Javier Duran @@ -33,6 +41,9 @@ Javier Espinosa Javier Espinosa <64952559+jespinosaar@users.noreply.github.com> Javier Espinosa Javier Espinosa +Javier Espinosa +Jennifer Medina +Jesus Juan Salgado Jonathan Gagne Jordan Mirocha Juan Carlos Segovia @@ -56,9 +67,15 @@ Nicholas Earl Oliver Oberdorf Oliver Oberdorf Pey Lian Lim <2090236+pllim@users.noreply.github.com> +Raul Gutierrez +Raul Gutierrez Simon Conseil Simon Conseil Simon Liedtke +Syed Gilani <59292422+syed-gilani@users.noreply.github.com> +Syed Gilani +Syed Gilani Tinuade Adeleke +Tim Galvin Volodymyr Savchenko Volodymyr Savchenko diff --git a/.readthedocs.yaml b/.readthedocs.yaml index a99592c422..f953c613c6 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,7 @@ python: path: . extra_requirements: - docs - - all_lt_39 + - all sphinx: fail_on_warning: true diff --git a/CHANGES.rst b/CHANGES.rst index 424981f8b5..fc38a7d48a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,121 @@ -0.4.5 (unreleased) +0.4.7 (unreleased) +================== + +New Tools and Services +---------------------- + + +Service fixes and enhancements +------------------------------ + + +Infrastructure, Utility and Other Changes and Additions +------------------------------------------------------- + + + +0.4.6 (2022-03-22) +================== + +Service fixes and enhancements +------------------------------ + +alma +^^^^ + +- Added ``verify_only`` option to check if data downloaded with correct file + size. [#2263] + +- Deprecated keywords and ``stage_data`` method has been removed. [#2309] + +- Deprecate broken functions from ``alma.utils``. [#2332] + +- Optional keyword arguments are now keyword only. [#2309] + +casda +^^^^^ + +- Simplify file names produced by ``download_files`` to avoid filename too + long errors. [#2308] + +esa.hubble +^^^^^^^^^^ + +- Changed ``query_target`` method to use TAP instead of AIO. [#2268] + + +- Added new method ``get_hap_hst_link`` and ``get_member_observations`` to + get related observations. [#2268] + +esa.xmm_newton +^^^^^^^^^^^^^^ + +- Add option to download proprietary data. [#2251] + +gaia +^^^^ + +- The ``query_object()`` and ``query_object_async()`` methods of + ``astroquery.gaia.Gaia`` no longer ignore their ``columns`` argument when + ``radius`` is specified. [#2249] + +- Enhanced methods ``launch_job`` and ``launch_job_async`` to avoid issues with + the name provided by the user for the output file when the results are + returned by the TAP in compressed format. [#2077] + +ipac.nexsci.nasa_exoplanet_archive +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Fixes to alias query, and regularize keyword removed from deprecated + ``query_star`` method. [#2264] + +mast +^^^^ + +- Adding moving target functionality to ``Tesscut`` [#2121] + +- Adding ``MastMissions`` class to provide mission-specific metadata query + functionalities. [#2095] + +- GALEX data is now available to download anonymously from the public + STScI S3 buckets. [#2261] + +- Adding the All-Sky PLATO Input Catalog ('plato') as a catalog option for + methods of ``Catalogs``. [#2279] + +- Optional keyword arguments are now keyword only. [#2317] + +sdss +^^^^ + +- Fix ``query_crossid`` for spectral data and DR17. [#2258, #2304] + +- Fix ``query_crossid`` to be able to query larger list of coordinates. [#2305] + +- Fix ``query_crossid`` for very old data releases (< DR10). [#2318] + + +Infrastructure, Utility and Other Changes and Additions +------------------------------------------------------- + +- Remove obsolete testing tools. [#2287] + +- Callback hooks are deleted before caching. Potentially all cached queries + prior to this PR will be rendered invalid. [#2295] + +utils.tap +^^^^^^^^^ + +- The modules that make use of the ``astroquery.utils.tap.model.job.Job`` class + (e.g. Gaia) no longer print messages about where the results of async queries + were written if the ``verbose`` setting is ``False``. [#2299] + +- New method, ``rename_table``, which allows the user to rename table and + column names. [#2077] + + + +0.4.5 (2021-12-24) ================== New Tools and Services @@ -7,7 +124,7 @@ New Tools and Services esa.jwst ^^^^^^^^^^ -- New module to provide access to eJWST Science Archive metadata and datasets. [#2140, #2238, #2243] +- New module to provide access to eJWST Science Archive metadata and datasets. [#2140, #2238] Service fixes and enhancements @@ -16,7 +133,12 @@ Service fixes and enhancements eso ^^^ -- Add option to retrieve_data from an earlier archive query [#1614] +- Add option to retrieve_data from an earlier archive query. [#1614] + +jplhorizons +^^^^^^^^^^^ + +- Fix result parsing issues by disabling caching of failed queries. [#2253] sdss ^^^^ @@ -29,6 +151,9 @@ Infrastructure, Utility and Other Changes and Additions - Adding ``--alma-site`` pytest option for testing to have a control over which specific site to test. [#2224] +- The function ``astroquery.utils.download_list_of_fitsfiles()`` has been + deprecated. [#2247] + utils.tap ^^^^^^^^^ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 7158f7d3ea..ac6ad0b023 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -6,6 +6,11 @@ Please see `astropy's contributing guildelines `__ for a general guide to the workflow involving git, etc. Everything below is astroquery-specific. +We strongly encourage draft pull requests to be opened early in development. +If you are thinking of contributing a new module, please open a pull request +as soon as you start developing code and mark it as a Draft PR on github. + + New Features ------------ We welcome any and all new features! If you have your own little query tool @@ -55,6 +60,17 @@ method is a wrapper around the `requests.request` function that provides important astroquery-specific utility, including caching, HTTP header generation, progressbars, and local writing-to-disk. +Dependencies +------------ +New contributions are generally not allowed to bring along additional dependencies. + +The astropy ecosystem tools should be used whenever possible. +For example, `astropy.table` should be used for table handling, +or `astropy.units` for unit and quantity +handling. + + + .. _astroquery API: docs/api.rst .. _template: docs/template.rst .. _requests: http://docs.python-requests.org/en/master/ diff --git a/LICENSE.rst b/LICENSE.rst index 8b43442a51..97acfd015c 100644 --- a/LICENSE.rst +++ b/LICENSE.rst @@ -1,4 +1,4 @@ -Copyright (c) 2011-2017 Astroquery Developers +Copyright (c) 2011-2022 Astroquery Developers All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/README.rst b/README.rst index 364aa857f2..e92ce67303 100644 --- a/README.rst +++ b/README.rst @@ -55,19 +55,23 @@ installed using `pip `_ or `anaconda `_. Running the tests requires `curl `_ to be installed. -The latest version of astroquery can be conda installed: +The latest version of astroquery can be pip installed (note the --pre for +picking up released developer versions): .. code-block:: bash - $ conda install -c conda-forge astroquery + $ pip install --pre astroquery -or pip installed: +To install all the mandatory and optional dependencies add the ``[all]`` +identifyer to the pip command above (or use ``[docs]`` or ``[test]`` for the +dependencies required to build the documentation or run the tests): .. code-block:: bash - $ pip install --pre astroquery + $ pip install --pre astroquery[all] + -and the 'bleeding edge' main version: +To install the 'bleeding edge' version: .. code-block:: bash @@ -88,10 +92,10 @@ Using astroquery ---------------- Importing astroquery on its own doesn't get you much: you need to import each -sub-module specifically. Check out the `docs`_ -to find a list of the tools available. The `API`_ -shows the standard suite of tools common to most modules, e.g. `query_object` -and `query_region`. +sub-module specifically. See the documentation for a list of `Available +Services `_. +The `API`_ shows the standard suite of tools common to most modules, e.g. +`query_object` and `query_region`. To report bugs and request features, please use the issue tracker. Code contributions are very welcome, though we encourage you to follow the `API`_ @@ -99,65 +103,6 @@ and `contributing guidelines `_ as much as possible. -List of Modules ---------------- - -The following modules have been completed using a common API: - - * `ALMA Archive `_ - * `Atomic Line List `_: A collection of more than 900,000 atomic transitions. - * `Besancon `_: Model of stellar population synthesis in the Galaxy. - * `CDS MOC Service `_: A collection of all-sky survey coverage maps. - * `CADC `_: Canadian Astronomy Data Centre. - * `CASDA `_: CSIRO ASKAP Science Data Archive. - * `ESASky `_: ESASky is a science driven discovery portal providing easy visualizations and full access to the entire sky as observed with ESA Space astronomy missions. - * `ESO Archive `_ - * `FIRST `_: Faint Images of the Radio Sky at Twenty-cm. 20-cm radio images of the extragalactic sky from the VLA. - * `Gaia `_: European Space Agency Gaia Archive. - * `ESA XMM `_: European Space Agency XMM-Newton Science Archive. - * `ESA Hubble `_: European Space Agency Hubble Science Archive. - * `ESA ISO `_: European Space Agency ISO Data Archive. - * `GAMA database `_ - * `Gemini `_: Gemini Archive. - * `HEASARC `_: NASA's High Energy Astrophysics Science Archive Research Center. - * `IBE `_: IRSA Image Server program interface (IBE) Query Tool provides access to the 2MASS, WISE, and PTF image archives. - * `IRSA `_: NASA/IPAC Infrared Science Archive. Science products for all of NASA's infrared and sub-mm missions. - * `IRSA dust `_: Galactic dust reddening and extinction maps from IRAS 100 um data. - * `MAGPIS `_: Multi-Array Galactic Plane Imaging Survey. 6 and 20-cm radio images of the Galactic plane from the VLA. - * `MAST `_: Barbara A. Mikulski Archive for Space Telescopes. - * `Minor Planet Center `_ - * `NASA ADS `_: SAO/NASA Astrophysics Data System. - * `NED `_: NASA/IPAC Extragalactic Database. Multiwavelength data from both surveys and publications. - * `NIST `_: National Institute of Standards and Technology (NIST) atomic lines database. - * `NRAO `_: Science data archive of the National Radio Astronomy Observatory. VLA, JVLA, VLBA and GBT data products. - * `NVAS archive `_ - * `Simbad `_: Basic data, cross-identifications, bibliography and measurements for astronomical objects outside the solar system. - * `Skyview `_: NASA SkyView service for imaging surveys. - * `Splatalogue `_: National Radio Astronomy Observatory (NRAO)-maintained (mostly) molecular radio and millimeter line list service. - * `UKIDSS `_: UKIRT Infrared Deep Sky Survey. JHK images of 7500 sq deg. in the northern sky. - * `Vamdc `_: VAMDC molecular line database. - * `Vizier `_: Set of 11,000+ published, multiwavelength catalogues hosted by the CDS. - * `VO Simple Cone Search `_ - * `xMatch `_: Cross-identify sources between very large data sets or between a user-uploaded list and a large catalogue. - -These others are functional, but do not follow a common or consistent API: - - * `Alfalfa `_: Arecibo Legacy Fast ALFA survey; extragalactic HI radio data. - * `CosmoSim `_: The CosmoSim database provides results from cosmological simulations performed within different projects: the MultiDark project, the BolshoiP project, and the CLUES project. - * `Exoplanet Orbit Database `_ - * `Fermi `_: Fermi gamma-ray telescope archive. - * `HITRAN `_: Access to the high-resolution transmission molecular absorption database. - * `JPL Horizons `_: JPL Solar System Dynamics Horizons Service. - * `JPL SBDB `_: JPL Solar System Dynamics Small-Body Database Browser Service. - * `Lamda `_: Leiden Atomic and Molecular Database; energy levels, radiative transitions, and collisional rates for astrophysically relevant atoms and molecules. - * `NASA Exoplanet Archive `_ - * `OAC API `_: Open Astronomy Catalog REST API Service. - * `Ogle `_: Optical Gravitational Lensing Experiment III; information on interstellar extinction towards the Galactic bulge. - * `Open Expolanet Catalog (OEC) `_ - * `SDSS `_: Sloan Digital Sky Survey data, including optical images, spectra, and spectral templates. - * `SHA `_: Spitzer Heritage Archive; infrared data products from the Spitzer Space Telescope. - - Citing Astroquery ----------------- @@ -188,7 +133,6 @@ Maintained by `Adam Ginsburg`_ and `Brigitta Sipocz .. _Download Stable ZIP: https://github.com/astropy/astroquery/zipball/stable .. _Download Stable TAR: https://github.com/astropy/astroquery/tarball/stable .. _View on Github: https://github.com/astropy/astroquery/ -.. _docs: http://astroquery.readthedocs.io .. _Documentation: http://astroquery.readthedocs.io .. _astropy.astroquery@gmail.com: mailto:astropy.astroquery@gmail.com .. _Adam Ginsburg: http://www.adamgginsburg.com diff --git a/astroquery/alfalfa/tests/test_alfalfa.py b/astroquery/alfalfa/tests/test_alfalfa.py index 81c8f5b112..6ee31f6031 100644 --- a/astroquery/alfalfa/tests/test_alfalfa.py +++ b/astroquery/alfalfa/tests/test_alfalfa.py @@ -5,7 +5,7 @@ from astropy import coordinates import pytest from ...utils import commons -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ... import alfalfa DATA_FILES = {'catalog': 'alfalfa_cat_small.txt', diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index 92d79713a5..0ccced8a26 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -8,18 +8,18 @@ import string import requests import warnings + from pkg_resources import resource_filename from bs4 import BeautifulSoup import pyvo - from urllib.parse import urljoin + from astropy.table import Table, Column, vstack from astroquery import log -from astropy.utils import deprecated from astropy.utils.console import ProgressBar -from astropy.utils.exceptions import AstropyDeprecationWarning from astropy import units as u from astropy.time import Time +from pyvo.dal.sia2 import SIA_PARAMETERS_DESC from ..exceptions import LoginError from ..utils import commons @@ -30,6 +30,7 @@ _gen_science_sql, _gen_spec_res_sql, ALMA_DATE_FORMAT from . import conf, auth_urls from astroquery.utils.commons import ASTROPY_LT_4_1 +from astroquery.exceptions import CorruptDataWarning __all__ = {'AlmaClass', 'ALMA_BANDS'} @@ -234,7 +235,7 @@ def tap(self): self._tap = pyvo.dal.tap.TAPService(baseurl=self.tap_url) return self._tap - def query_object_async(self, object_name, cache=None, public=True, + def query_object_async(self, object_name, *, public=True, science=True, payload=None, **kwargs): """ Query the archive for a source name. @@ -243,7 +244,6 @@ def query_object_async(self, object_name, cache=None, public=True, ---------- object_name : str The object name. Will be resolved by astropy.coord.SkyCoord - cache : deprecated public : bool True to return only public datasets, False to return private only, None to return both @@ -260,7 +260,7 @@ def query_object_async(self, object_name, cache=None, public=True, return self.query_async(public=public, science=science, payload=payload, **kwargs) - def query_region_async(self, coordinate, radius, cache=None, public=True, + def query_region_async(self, coordinate, radius, *, public=True, science=True, payload=None, **kwargs): """ Query the ALMA archive with a source name and radius @@ -271,8 +271,6 @@ def query_region_async(self, coordinate, radius, cache=None, public=True, the identifier or coordinates around which to query. radius : str / `~astropy.units.Quantity`, optional the radius of the region - cache : Deprecated - Cache the query? public : bool True to return only public datasets, False to return private only, None to return both @@ -297,10 +295,8 @@ def query_region_async(self, coordinate, radius, cache=None, public=True, return self.query_async(public=public, science=science, payload=payload, **kwargs) - def query_async(self, payload, cache=None, public=True, science=True, - legacy_columns=False, max_retries=None, - get_html_version=None, - get_query_payload=None, **kwargs): + def query_async(self, payload, *, public=True, science=True, + legacy_columns=False, get_query_payload=None, **kwargs): """ Perform a generic query with user-specified payload @@ -308,7 +304,6 @@ def query_async(self, payload, cache=None, public=True, science=True, ---------- payload : dictionary Please consult the `help` method - cache : deprecated public : bool True to return only public datasets, False to return private only, None to return both @@ -325,17 +320,6 @@ def query_async(self, payload, cache=None, public=True, science=True, Table with results. Columns are those in the ALMA ObsCore model (see ``help_tap``) unless ``legacy_columns`` argument is set to True. """ - local_args = dict(locals().items()) - - for arg in local_args: - # check if the deprecated attributes have been used - for dep in ['cache', 'max_retries', 'get_html_version']: - if arg[0] == dep and arg[1] is not None: - warnings.warn( - ("Argument '{}' has been deprecated " - "since version 4.0.1 and will be ignored".format(arg[0])), - AstropyDeprecationWarning) - del kwargs[arg[0]] if payload is None: payload = {} @@ -383,7 +367,7 @@ def query_async(self, payload, cache=None, public=True, science=True, return legacy_result return result - def query_sia(self, pos=None, band=None, time=None, pol=None, + def query_sia(self, *, pos=None, band=None, time=None, pol=None, field_of_view=None, spatial_resolution=None, spectral_resolving_power=None, exptime=None, timeres=None, publisher_did=None, @@ -397,6 +381,7 @@ def query_sia(self, pos=None, band=None, time=None, pol=None, Parameters ---------- + _SIA2_PARAMETERS Returns ------- @@ -424,6 +409,12 @@ def query_sia(self, pos=None, band=None, time=None, pol=None, maxrec=maxrec, **kwargs) + # SIA_PARAMETERS_DESC contains links that Sphinx can't resolve. + for var in ('POLARIZATION_STATES', 'CALIBRATION_LEVELS'): + SIA_PARAMETERS_DESC = SIA_PARAMETERS_DESC.replace(f'`pyvo.dam.obscore.{var}`', + f'pyvo.dam.obscore.{var}') + query_sia.__doc__ = query_sia.__doc__.replace('_SIA2_PARAMETERS', SIA_PARAMETERS_DESC) + def query_tap(self, query, maxrec=None): """ Send query to the ALMA TAP. Results in pyvo.dal.TapResult format. @@ -491,54 +482,7 @@ def _get_dataarchive_url(self): "on github.") return self.dataarchive_url - @deprecated(since="v0.4.1", alternative="get_data_info") - def stage_data(self, uids, expand_tarfiles=False, return_json=False): - """ - Obtain table of ALMA files - - DEPRECATED: Data is no longer staged. This method is deprecated and - kept here for backwards compatibility reasons but it's not fully - compatible with the original implementation. - - Parameters - ---------- - uids : list or str - A list of valid UIDs or a single UID. - UIDs should have the form: 'uid://A002/X391d0b/X7b' - expand_tarfiles : DEPRECATED - return_json : DEPRECATED - Note: The returned astropy table can be easily converted to json - through pandas: - output = StringIO() - stage_data(...).to_pandas().to_json(output) - table_json = output.getvalue() - - Returns - ------- - data_file_table : Table - A table containing 3 columns: the UID, the file URL (for future - downloading), and the file size - """ - - if return_json: - raise AttributeError( - 'return_json is deprecated. See method docs for a workaround') - table = Table() - res = self.get_data_info(uids, expand_tarfiles=expand_tarfiles) - p = re.compile(r'.*(uid__.*)\.asdm.*') - if res: - table['name'] = [u.split('/')[-1] for u in res['access_url']] - table['id'] = [p.search(x).group(1) if 'asdm' in x else 'None' - for x in table['name']] - table['type'] = res['content_type'] - table['size'] = res['content_length'] - table['permission'] = ['UNKNOWN'] * len(res) - table['mous_uid'] = [uids] * len(res) - table['URL'] = res['access_url'] - table['isProprietary'] = res['readable'] - return table - - def get_data_info(self, uids, expand_tarfiles=False, + def get_data_info(self, uids, *, expand_tarfiles=False, with_auxiliary=True, with_rawdata=True): """ @@ -676,8 +620,9 @@ def _HEADER_data_size(self, files): return data_sizes, totalsize.to(u.GB) - def download_files(self, files, savedir=None, cache=True, - continuation=True, skip_unauthorized=True,): + def download_files(self, files, *, savedir=None, cache=True, + continuation=True, skip_unauthorized=True, + verify_only=False): """ Given a list of file URLs, download them @@ -698,6 +643,10 @@ def download_files(self, files, savedir=None, cache=True, If you receive "unauthorized" responses for some of the download requests, skip over them. If this is False, an exception will be raised. + verify_only : bool + Option to go through the process of checking the files to see if + they're the right size, but not actually download them. This + option may be useful if a previous download run failed partway. """ if self.USERNAME: @@ -735,15 +684,34 @@ def download_files(self, files, savedir=None, cache=True, filename = os.path.join(savedir, filename) + if verify_only: + existing_file_length = os.stat(filename).st_size + if 'content-length' in check_filename.headers: + length = int(check_filename.headers['content-length']) + if length == 0: + warnings.warn('URL {0} has length=0'.format(url)) + elif existing_file_length == length: + log.info(f"Found cached file {filename} with expected size {existing_file_length}.") + elif existing_file_length < length: + log.info(f"Found cached file {filename} with size {existing_file_length} < expected " + f"size {length}. The download should be continued.") + elif existing_file_length > length: + warnings.warn(f"Found cached file {filename} with size {existing_file_length} > expected " + f"size {length}. The download is likely corrupted.", + CorruptDataWarning) + else: + warnings.warn(f"Could not verify {url} because it has no 'content-length'") + try: - self._download_file(file_link, - filename, - timeout=self.TIMEOUT, - auth=auth, - cache=cache, - method='GET', - head_safe=False, - continuation=continuation) + if not verify_only: + self._download_file(file_link, + filename, + timeout=self.TIMEOUT, + auth=auth, + cache=cache, + method='GET', + head_safe=False, + continuation=continuation) downloaded_files.append(filename) except requests.HTTPError as ex: @@ -788,7 +756,7 @@ def _parse_result(self, response, verbose=False): return response - def retrieve_data_from_uid(self, uids, cache=True): + def retrieve_data_from_uid(self, uids, *, cache=True): """ Stage & Download ALMA data. Will print out the expected file size before attempting the download. @@ -821,7 +789,7 @@ def retrieve_data_from_uid(self, uids, cache=True): downloaded_files = self.download_files(file_urls) return downloaded_files - def _get_auth_info(self, username, store_password=False, + def _get_auth_info(self, username, *, store_password=False, reenter_password=False): """ Get the auth info (user, password) for use in another function @@ -999,7 +967,7 @@ def cycle0_table(self): self._cycle0_table.rename_column('col2', 'uid') return self._cycle0_table - def get_files_from_tarballs(self, downloaded_files, regex=r'.*\.fits$', + def get_files_from_tarballs(self, downloaded_files, *, regex=r'.*\.fits$', path='cache_path', verbose=True): """ Given a list of successfully downloaded tarballs, extract files @@ -1049,7 +1017,7 @@ def get_files_from_tarballs(self, downloaded_files, regex=r'.*\.fits$', return filelist - def download_and_extract_files(self, urls, delete=True, regex=r'.*\.fits$', + def download_and_extract_files(self, urls, *, delete=True, regex=r'.*\.fits$', include_asdm=False, path='cache_path', verbose=True): """ @@ -1163,53 +1131,7 @@ def help(self, cache=True): print("Alma.query(payload=dict(project_code='2017.1.01355.L', " "source_name_alma='G008.67'))") - def _json_summary_to_table(self, data, base_url): - """ - Special tool to convert some JSON metadata to a table Obsolete as of - March 2020 - should be removed along with stage_data_prefeb2020 - """ - from ..utils import url_helpers - columns = {'mous_uid': [], 'URL': [], 'size': []} - for entry in data['node_data']: - # de_type can be useful (e.g., MOUS), but it is not necessarily - # specified - # file_name and file_key *must* be specified. - is_file = \ - (entry['file_name'] != 'null' and entry['file_key'] != 'null') - if is_file: - # "de_name": "ALMA+uid://A001/X122/X35e", - columns['mous_uid'].append(entry['de_name'][5:]) - if entry['file_size'] == 'null': - columns['size'].append(np.nan * u.Gbyte) - else: - columns['size'].append( - (int(entry['file_size']) * u.B).to(u.Gbyte)) - # example template for constructing url: - # https://almascience.eso.org/dataPortal/requests/keflavich/940238268/ALMA/ - # uid___A002_X9d6f4c_X154/2013.1.00546.S_uid___A002_X9d6f4c_X154.asdm.sdm.tar - # above is WRONG... except for ASDMs, when it's right - # should be: - # 2013.1.00546.S_uid___A002_X9d6f4c_X154.asdm.sdm.tar/2013.1.00546.S_uid___A002_X9d6f4c_X154.asdm.sdm.tar - # - # apparently ASDMs are different from others: - # templates: - # https://almascience.eso.org/dataPortal/requests/keflavich/946895898/ALMA/ - # 2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar/2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar - # uid___A002_X9ee74a_X26f0/2013.1.00308.S_uid___A002_X9ee74a_X26f0.asdm.sdm.tar - url = url_helpers.join(base_url, - entry['file_key'], - entry['file_name']) - if 'null' in url: - raise ValueError("The URL {0} was created containing " - "'null', which is invalid.".format(url)) - columns['URL'].append(url) - - columns['size'] = u.Quantity(columns['size'], u.Gbyte) - - tbl = Table([Column(name=k, data=v) for k, v in columns.items()]) - return tbl - - def get_project_metadata(self, projectid, cache=True): + def get_project_metadata(self, projectid, *, cache=True): """ Get the metadata - specifically, the project abstract - for a given project ID. """ diff --git a/astroquery/alma/tests/test_alma_remote.py b/astroquery/alma/tests/test_alma_remote.py index 9571957eb3..931d2f3041 100644 --- a/astroquery/alma/tests/test_alma_remote.py +++ b/astroquery/alma/tests/test_alma_remote.py @@ -3,6 +3,7 @@ import shutil import numpy as np import pytest +import warnings from datetime import datetime import os from urllib.parse import urlparse @@ -12,6 +13,7 @@ from astropy import coordinates from astropy import units as u +from astroquery.exceptions import CorruptDataWarning from astroquery.utils.commons import ASTROPY_LT_4_1 from .. import Alma @@ -73,13 +75,6 @@ def test_SgrAstar(self, temp_dir, alma): # "The Brick", g0.253, is in this one # assert b'2011.0.00217.S' in result_c['Project code'] # missing cycle 1 data - def test_docs_example(self, temp_dir, alma): - alma.cache_location = temp_dir - - rslt = alma.query(payload=dict(obs_creator_name='*Ginsburg*')) - - assert 'ADS/JAO.ALMA#2013.1.00269.S' in rslt['obs_publisher_did'] - def test_freq(self, alma): payload = {'frequency': '85..86'} result = alma.query(payload) @@ -133,104 +128,12 @@ def test_m83(self, temp_dir, alma): m83_data = alma.query_object('M83', science=True, legacy_columns=True) uids = np.unique(m83_data['Member ous id']) - link_list = alma.stage_data(uids) + link_list = alma.get_data_info(uids) # On Feb 8, 2016 there were 83 hits. This number should never go down. # Except it has. On May 18, 2016, there were 47. assert len(link_list) >= 47 - # test re-staging - # (has been replaced with warning) - # with pytest.raises(requests.HTTPError) as ex: - # link_list = alma.stage_data(uids) - # assert ex.value.args[0] == ('Received an error 405: this may indicate you have ' - # 'already staged the data. Try downloading the ' - # 'file URLs directly with download_files.') - - # log.warning doesn't actually make a warning - # link_list = alma.stage_data(uids) - # w = recwarn.pop() - # assert (str(w.message) == ('Error 405 received. If you have previously staged the ' - # 'same UIDs, the result returned is probably correct,' - # ' otherwise you may need to create a fresh astroquery.Alma instance.')) - - @pytest.mark.skipif("SKIP_SLOW", reason="Known issue") - def test_stage_data(self, temp_dir, alma): - alma.cache_location = temp_dir - - result_s = alma.query_object('Sgr A*', legacy_columns=True) - - if ASTROPY_LT_4_1: - assert b'2013.1.00857.S' in result_s['Project code'] - assert b'uid://A002/X40d164/X1b3' in result_s['Asdm uid'] - assert b'uid://A002/X391d0b/X23d' in result_s['Member ous id'] - match_val = b'uid://A002/X40d164/X1b3' - else: - assert '2013.1.00857.S' in result_s['Project code'] - assert 'uid://A002/X40d164/X1b3' in result_s['Asdm uid'] - assert 'uid://A002/X391d0b/X23d' in result_s['Member ous id'] - match_val = 'uid://A002/X40d164/X1b3' - - match = result_s['Asdm uid'] == match_val - uid = result_s['Member ous id'][match] - # this is temporary to switch back to ALMA servers - # del alma.dataarchive_url - # alma.archive_url = 'http://almascience.org' - result = alma.stage_data(uid) - - found = False - for url in result['URL']: - if 'uid___A002_X40d164_X1b3' in url: - found = True - break - assert found, 'URL to uid___A002_X40d164_X1b3 expected' - - def test_stage_data_listall(self, temp_dir, alma): - """ - test for expanded capability created in #1683 - """ - alma.cache_location = temp_dir - - uid = 'uid://A001/X12a3/Xe9' - result1 = alma.stage_data(uid, expand_tarfiles=False) - result2 = alma.stage_data(uid, expand_tarfiles=True) - - expected_names = [ - '2017.1.01185.S_uid___A002_Xd28a9e_X71b8.asdm.sdm.tar', - '2017.1.01185.S_uid___A002_Xd28a9e_X7b4d.asdm.sdm.tar', - '2017.1.01185.S_uid___A002_Xd29c1f_X1f74.asdm.sdm.tar', - '2017.1.01185.S_uid___A002_Xd29c1f_X5cf.asdm.sdm.tar'] - expected_names_with_aux = expected_names + \ - ['2017.1.01185.S_uid___A001_X12a3_Xe9_auxiliary.tar'] - for name in expected_names_with_aux: - assert name in result1['name'] - for res in result1: - p = re.compile(r'.*(uid__.*)\.asdm.*') - if res['name'] in expected_names: - assert 'application/x-tar' == res['type'] - assert res['id'] == p.search(res['name']).group(1) - else: - assert res['type'] in ['application/x-tar', 'application/x-votable+xml;content=datalink', 'text/plain'] - assert res['id'] == 'None' - assert 'UNKNOWN' == res['permission'] - assert res['mous_uid'] == uid - assert len(result2) > len(result1) - - def test_stage_data_json(self, temp_dir, alma): - """ - test for json returns - """ - alma.cache_location = temp_dir - - uid = 'uid://A001/X12a3/Xe9' - # this is temporary to switch back to ALMA servers - # alma.archive_url = 'http://almascience.org' - result = alma.stage_data(uid, return_json=False) - assert len(result) > 0 - with pytest.raises(AttributeError): - # this no longer works - alma.stage_data(uid, return_json=True) - def test_data_proprietary(self, alma): # public assert not alma.is_proprietary('uid://A001/X12a3/Xe9') @@ -272,7 +175,7 @@ def test_data_info(self, temp_dir, alma): file_url = url break assert file_url - alma.download_files([file_url], temp_dir) + alma.download_files([file_url], savedir=temp_dir) assert os.stat(os.path.join(temp_dir, file)).st_size # mock downloading an entire program @@ -280,8 +183,7 @@ def test_data_info(self, temp_dir, alma): alma.download_files = download_files_mock alma.retrieve_data_from_uid([uid]) - comparison = download_files_mock.mock_calls[0][1] == data_info_tar[ - 'access_url'] + comparison = download_files_mock.mock_calls[0][1] == data_info_tar['access_url'] assert comparison.all() def test_download_data(self, temp_dir, alma): @@ -298,7 +200,7 @@ def test_download_data(self, temp_dir, alma): alma._download_file = download_mock urls = [x['access_url'] for x in data_info if fitsre.match(x['access_url'])] - results = alma.download_files(urls, temp_dir) + results = alma.download_files(urls, savedir=temp_dir) alma._download_file.call_count == len(results) assert len(results) == len(urls) @@ -370,11 +272,11 @@ def test_doc_example(self, temp_dir, alma): assert X30.sum() == 4 # Jul 13, 2020 assert X31.sum() == 4 # Jul 13, 2020 - mous1 = alma.stage_data('uid://A001/X11f/X30') + mous1 = alma.get_data_info('uid://A001/X11f/X30') totalsize_mous1 = mous1['size'].sum() * u.Unit(mous1['size'].unit) assert (totalsize_mous1.to(u.B) > 1.9*u.GB) - mous = alma.stage_data('uid://A002/X3216af/X31') + mous = alma.get_data_info('uid://A002/X3216af/X31') totalsize_mous = mous['size'].sum() * u.Unit(mous['size'].unit) # More recent ALMA request responses do not include any information # about file size, so we have to allow for the possibility that all @@ -491,9 +393,9 @@ def test_cycle1(self, temp_dir, alma): # Need new Alma() instances each time a1 = alma() - uid_url_table_mous = a1.stage_data(result['Member ous id']) + uid_url_table_mous = a1.get_data_info(result['Member ous id']) a2 = alma() - uid_url_table_asdm = a2.stage_data(result['Asdm uid']) + uid_url_table_asdm = a2.get_data_info(result['Asdm uid']) # I believe the fixes as part of #495 have resulted in removal of a # redundancy in the table creation, so a 1-row table is OK here. # A 2-row table may not be OK any more, but that's what it used to @@ -540,8 +442,8 @@ def test_cycle0(self, temp_dir, alma): alma1 = alma() alma2 = alma() - uid_url_table_mous = alma1.stage_data(result['Member ous id']) - uid_url_table_asdm = alma2.stage_data(result['Asdm uid']) + uid_url_table_mous = alma1.get_data_info(result['Member ous id']) + uid_url_table_asdm = alma2.get_data_info(result['Asdm uid']) assert len(uid_url_table_asdm) == 1 assert len(uid_url_table_mous) == 32 @@ -604,32 +506,9 @@ def test_project_metadata(alma): @pytest.mark.remote_data -@pytest.mark.skip('Not working for now - Investigating') -def test_staging_postfeb2020(alma): - - tbl = alma.stage_data('uid://A001/X121/X4ba') - - assert 'mous_uid' in tbl.colnames - - assert '2013.1.00269.S_uid___A002_X9de499_X3d6c.asdm.sdm.tar' in tbl['name'] - - -@pytest.mark.remote_data -@pytest.mark.skip('Not working for now - Investigating') -def test_staging_uptofeb2020(alma): - tbl = alma.stage_data('uid://A001/X121/X4ba') - - assert 'mous_uid' in tbl.colnames - - names = [x.split("/")[-1] for x in tbl['URL']] - - assert '2013.1.00269.S_uid___A002_X9de499_X3d6c.asdm.sdm.tar' in names - - -@pytest.mark.remote_data -def test_staging_stacking(alma): - alma.stage_data(['uid://A001/X13d5/X1d', 'uid://A002/X3216af/X31', - 'uid://A001/X12a3/X240']) +def test_data_info_stacking(alma): + alma.get_data_info(['uid://A001/X13d5/X1d', 'uid://A002/X3216af/X31', + 'uid://A001/X12a3/X240']) @pytest.mark.remote_data @@ -652,6 +531,58 @@ def test_big_download_regression(alma): @pytest.mark.remote_data -def test_download_html_file(): +def test_download_html_file(alma): result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html']) assert result + + +@pytest.mark.remote_data +def test_verify_html_file(alma, caplog): + # first, make sure the file is not cached (in case this test gets called repeatedly) + # (we are hacking the file later in this test to trigger different failure modes so + # we need it fresh) + try: + result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'], verify_only=True) + local_filepath = result[0] + os.remove(local_filepath) + except FileNotFoundError: + pass + + caplog.clear() + + # download the file + result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html']) + assert 'member.uid___A001_X1284_X1353.qa2_report.html' in result[0] + + result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'], verify_only=True) + assert 'member.uid___A001_X1284_X1353.qa2_report.html' in result[0] + local_filepath = result[0] + existing_file_length = 66336 + assert f"Found cached file {local_filepath} with expected size {existing_file_length}." in caplog.text + + # manipulate the file + with open(local_filepath, 'ab') as fh: + fh.write(b"Extra Text") + + caplog.clear() + length = 66336 + existing_file_length = length + 10 + with pytest.warns(expected_warning=CorruptDataWarning, + match=f"Found cached file {local_filepath} with size {existing_file_length} > expected size {length}. The download is likely corrupted."): + result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'], verify_only=True) + assert 'member.uid___A001_X1284_X1353.qa2_report.html' in result[0] + + # manipulate the file: make it small + with open(local_filepath, 'wb') as fh: + fh.write(b"Empty Text") + + caplog.clear() + result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'], verify_only=True) + assert 'member.uid___A001_X1284_X1353.qa2_report.html' in result[0] + length = 66336 + existing_file_length = 10 + assert f"Found cached file {local_filepath} with size {existing_file_length} < expected size {length}. The download should be continued." in caplog.text + + # cleanup: we don't want `test_download_html_file` to fail if this test is re-run + if os.path.exists(local_filepath): + os.remove(local_filepath) diff --git a/astroquery/alma/tests/test_alma_utils.py b/astroquery/alma/tests/test_alma_utils.py index c5df6101ef..39546dbb6c 100644 --- a/astroquery/alma/tests/test_alma_utils.py +++ b/astroquery/alma/tests/test_alma_utils.py @@ -14,38 +14,6 @@ from .. import utils -@pytest.mark.skipif('not pyregion_OK') -def test_pyregion_subset(): - header = dict(naxis=2, crpix1=15, crpix2=15, crval1=0.1, crval2=0.1, - cdelt1=-1. / 3600, cdelt2=1. / 3600., ctype1='GLON-CAR', - ctype2='GLAT-CAR') - mywcs = wcs.WCS(header) - # circle with radius 10" at 0.1, 0.1 - shape = Shape('circle', (0.1, 0.1, 10. / 3600.)) - shape.coord_format = 'galactic' - shape.coord_list = (0.1, 0.1, 10. / 3600.) - shape.attr = ([], {}) - data = np.ones([40, 40]) - - # The following line raises a DeprecationWarning from pyregion, ignore it - with warnings.catch_warnings(): - warnings.filterwarnings('ignore') - (xlo, xhi, ylo, yhi), d = utils.pyregion_subset(shape, data, mywcs) - - # sticky note over check-engine light solution... but this problem is too - # large in scope to address here. See - # https://github.com/astropy/astropy/pull/3992 - assert d.sum() >= 313 & d.sum() <= 315 # VERY approximately pi - np.testing.assert_almost_equal(xlo, - data.shape[0] / 2 - mywcs.wcs.crpix[0] - 1) - np.testing.assert_almost_equal(xhi, - data.shape[0] - mywcs.wcs.crpix[0] - 1) - np.testing.assert_almost_equal(ylo, - data.shape[1] / 2 - mywcs.wcs.crpix[1] - 1) - np.testing.assert_almost_equal(yhi, - data.shape[1] - mywcs.wcs.crpix[1] - 1) - - frq_sup_str = ('[86.26..88.14GHz,976.56kHz, XX YY] U ' '[88.15..90.03GHz,976.56kHz, XX YY] U ' '[98.19..100.07GHz,976.56kHz, XX YY] U ' @@ -63,28 +31,3 @@ def test_parse_frequency_support(frq_sup_str=frq_sup_str, result=franges): def approximate_primary_beam_sizes(frq_sup_str=frq_sup_str, beamsizes=beamsizes): assert np.all(utils.approximate_primary_beam_sizes(frq_sup_str) == beamsizes) - - -@pytest.mark.remote_data -@pytest.mark.skipif('not pyregion_OK') -@pytest.mark.skip('To be fixed later') -def test_make_finder_chart(): - import matplotlib - matplotlib.use('agg') - if matplotlib.get_backend() != 'agg': - pytest.xfail("Matplotlib backend was incorrectly set to {0}, could " - "not run finder chart test.".format(matplotlib.get_backend())) - - result = utils.make_finder_chart('Eta Carinae', 3 * u.arcmin, - 'Eta Carinae') - image, catalog, hit_mask_public, hit_mask_private = result - - assert len(catalog) >= 6 # down to 6 on Nov 17, 2016 - assert 3 in [int(x) for x in hit_mask_public] - # Feb 8 2016: apparently the 60s integration hasn't actually been released yet... - if 3 in hit_mask_public: - assert hit_mask_public[3][256, 256] >= 30.23 - elif b'3' in hit_mask_public: - assert hit_mask_public[b'3'][256, 256] >= 30.23 - else: - raise ValueError("hit_mask keys are not of any known type") diff --git a/astroquery/alma/utils.py b/astroquery/alma/utils.py index 1e299ea519..8b6b92aaff 100644 --- a/astroquery/alma/utils.py +++ b/astroquery/alma/utils.py @@ -10,12 +10,18 @@ from astroquery import log from astropy import units as u from astropy.io import fits +from astropy.utils import deprecated + from astroquery.utils.commons import ASTROPY_LT_4_1 from astroquery.skyview import SkyView from astroquery.alma import Alma +__all__ = ['parse_frequency_support', 'footprint_to_reg', 'approximate_primary_beam_sizes'] + + +@deprecated('0.4.6', 'this function has been deprecated and will be removed in the next release.') def pyregion_subset(region, data, mywcs): """ Return a subset of an image (``data``) given a region. @@ -153,6 +159,7 @@ def footprint_to_reg(footprint): return reglist +@deprecated('0.4.6', 'this function has been deprecated and will be removed in the next release.') def add_meta_to_reg(reg, meta): reg.meta = meta return reg @@ -181,6 +188,7 @@ def approximate_primary_beam_sizes(frequency_support_str, return u.Quantity(beam_sizes) +@deprecated('0.4.6', 'this function has been deprecated and will be removed in the next release.') def make_finder_chart(target, radius, save_prefix, service=SkyView.get_images, service_kwargs={'survey': ['2MASS-K'], 'pixels': 500}, alma_kwargs={'public': False, 'science': False}, @@ -229,6 +237,7 @@ def make_finder_chart(target, radius, save_prefix, service=SkyView.get_images, **kwargs) +@deprecated('0.4.6', 'this function has been deprecated and will be removed in the next release.') def make_finder_chart_from_image(image, target, radius, save_prefix, alma_kwargs={'public': False, 'science': False, @@ -273,6 +282,7 @@ def make_finder_chart_from_image(image, target, radius, save_prefix, **kwargs) +@deprecated('0.4.6', 'this function has been deprecated and will be removed in the next release.') def make_finder_chart_from_image_and_catalog(image, catalog, save_prefix, alma_kwargs={'public': False, 'science': False}, diff --git a/astroquery/besancon/tests/test_besancon.py b/astroquery/besancon/tests/test_besancon.py index afd4c548a2..0254864cbf 100644 --- a/astroquery/besancon/tests/test_besancon.py +++ b/astroquery/besancon/tests/test_besancon.py @@ -6,7 +6,7 @@ from astropy.io.ascii.tests.common import assert_equal from ... import besancon from ...utils import commons -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse # SKIP - don't run tests because Besancon folks don't want them (based on # the fact that your@email.net is now rejected) diff --git a/astroquery/cadc/core.py b/astroquery/cadc/core.py index 10b5d283c1..20f719d15b 100644 --- a/astroquery/cadc/core.py +++ b/astroquery/cadc/core.py @@ -525,8 +525,10 @@ def get_data_urls(self, query_result, include_auxiliaries=False): urlencode({'ID': pid_sublist, 'REQUEST': 'downloads-only'}, True))) for service_def in datalink: - if service_def.semantics == 'http://www.openadc.org/caom2#pkg': - # pkg is an alternative for downloading multiple + if service_def.semantics in ['http://www.opencadc.org/caom2#pkg', '#package']: + # TODO http://www.openadc.org/caom2#pkg has been replaced + # by "package". Removed it after CADC rolls out the change + # package is an alternative for downloading multiple # data files in a tar file as an alternative to separate # downloads. It doesn't make much sense in this case so # filter it out. diff --git a/astroquery/cadc/tests/test_cadctap.py b/astroquery/cadc/tests/test_cadctap.py index 91da50c889..256dfe2b66 100644 --- a/astroquery/cadc/tests/test_cadctap.py +++ b/astroquery/cadc/tests/test_cadctap.py @@ -230,10 +230,13 @@ class Result: file3.semantics = '#preview' file3.access_url = 'https://get.your.data/previewpath' # add the package file that should be filtered out + package_file_old = Mock() + package_file_old.semantics = 'http://www.opencadc.org/caom2#pkg' package_file = Mock() - package_file.semantics = 'http://www.openadc.org/caom2#pkg' - result = [file1, file2, file3, package_file] - with patch('pyvo.dal.adhoc.DatalinkResults.from_result_url') as dl_results_mock: + package_file.semantics = '#package' + result = [file1, file2, file3, package_file_old, package_file] + with patch('pyvo.dal.adhoc.DatalinkResults.from_result_url') as \ + dl_results_mock: dl_results_mock.return_value = result cadc = Cadc() cadc._request = get # mock the request diff --git a/astroquery/cadc/tests/test_cadctap_remote.py b/astroquery/cadc/tests/test_cadctap_remote.py index ea8c041218..b2b03b55ed 100644 --- a/astroquery/cadc/tests/test_cadctap_remote.py +++ b/astroquery/cadc/tests/test_cadctap_remote.py @@ -206,6 +206,7 @@ def test_authsession(self): @pytest.mark.skipif(one_test, reason='One test mode') @pytest.mark.skipif(not pyvo_OK, reason='not pyvo_OK') + @pytest.mark.xfail(reason='#2325') def test_get_images(self): cadc = Cadc() coords = '08h45m07.5s +54d18m00s' @@ -253,6 +254,7 @@ def test_get_images_against_AS(self): @pytest.mark.skipif(one_test, reason='One test mode') @pytest.mark.skipif(not pyvo_OK, reason='not pyvo_OK') + @pytest.mark.xfail(reason='#2325') def test_get_images_async(self): cadc = Cadc() coords = '01h45m07.5s +23d18m00s' @@ -313,6 +315,7 @@ def test_list_tables(self): reason='Requires real CADC certificate (CADC_CERT ' 'environment variable)') @pytest.mark.skipif(not pyvo_OK, reason='not pyvo_OK') + @pytest.mark.xfail(reason='#2325') def test_list_jobs(self): cadc = Cadc() cadc.login(certificate_file=os.environ['CADC_CERT']) diff --git a/astroquery/casda/core.py b/astroquery/casda/core.py index d1a072cb5d..223f9fcf45 100644 --- a/astroquery/casda/core.py +++ b/astroquery/casda/core.py @@ -2,7 +2,8 @@ # 1. standard library imports from io import BytesIO -from urllib.parse import unquote +import os +from urllib.parse import unquote, urlparse import time from xml.etree import ElementTree from datetime import datetime, timezone @@ -239,9 +240,15 @@ def download_files(self, urls, savedir=''): # for each url in list, download file and checksum filenames = [] for url in urls: - fn = self._request('GET', url, save=True, savedir=savedir, timeout=self.TIMEOUT, cache=False) - if fn: - filenames.append(fn) + parseResult = urlparse(url) + local_filename = unquote(os.path.basename(parseResult.path)) + if os.name == 'nt': + # Windows doesn't allow special characters in filenames like + # ":" so replace them with an underscore + local_filename = local_filename.replace(':', '_') + local_filepath = os.path.join(savedir or self.cache_location or '.', local_filename) + self._download_file(url, local_filepath, timeout=self.TIMEOUT, cache=False) + filenames.append(local_filepath) return filenames diff --git a/astroquery/casda/tests/test_casda.py b/astroquery/casda/tests/test_casda.py index da7775b7a4..fe74075f5f 100644 --- a/astroquery/casda/tests/test_casda.py +++ b/astroquery/casda/tests/test_casda.py @@ -280,3 +280,19 @@ def test_stage_data(patch_get): urls = casda.stage_data(table, verbose=True) assert urls == ['http://casda.csiro.au/download/web/111-000-111-000/askap_img.fits.checksum', 'http://casda.csiro.au/download/web/111-000-111-000/askap_img.fits'] + + +def test_download_file(patch_get): + urls = ['https://ingest.pawsey.org/bucket_name/path/askap_img.fits?security=stuff', + 'http://casda.csiro.au/download/web/111-000-111-000/askap_img.fits.checksum', + 'https://ingest.pawsey.org.au/casda-prd-as110-01/dc52217/primary_images/RACS-DR1_0000%2B18A.fits?security=stuff'] + casda = Casda('user', 'password') + + # skip the actual downloading of the file + download_mock = MagicMock() + casda._download_file = download_mock + + filenames = casda.download_files(urls) + assert filenames[0].endswith('askap_img.fits') + assert filenames[1].endswith('askap_img.fits.checksum') + assert filenames[2].endswith('RACS-DR1_0000+18A.fits') diff --git a/astroquery/cds/tests/test_mocserver.py b/astroquery/cds/tests/test_mocserver.py index 27194a43bf..53762d89dc 100644 --- a/astroquery/cds/tests/test_mocserver.py +++ b/astroquery/cds/tests/test_mocserver.py @@ -21,7 +21,7 @@ HAS_REGIONS = False from ..core import cds -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse DATA_FILES = { diff --git a/astroquery/esa/hubble/core.py b/astroquery/esa/hubble/core.py index 7b413d922e..1530774786 100644 --- a/astroquery/esa/hubble/core.py +++ b/astroquery/esa/hubble/core.py @@ -16,19 +16,12 @@ """ -from astroquery.utils import commons from astropy import units from astropy.coordinates import SkyCoord from astropy.coordinates import Angle -from astropy.units import Quantity from astroquery.utils.tap.core import TapPlus -from astroquery.utils.tap.model import modelutils from astroquery.query import BaseQuery -from astropy.table import Table -from io import BytesIO import shutil -import os -import json from . import conf from astroquery import log @@ -119,14 +112,105 @@ def download_product(self, observation_id, *, calibration_level=None, shutil.move(response, filename) + def get_member_observations(self, observation_id): + """ + Returns the related members of simple and composite observations + + Parameters + ---------- + observation_id : str + Observation identifier. + + Returns + ------- + A list of strings with the observation_id of the associated + observations + """ + observation_type = self.get_observation_type(observation_id) + + if 'Composite' in observation_type: + oids = self._select_related_members(observation_id) + elif 'Simple' in observation_type: + oids = self._select_related_composite(observation_id) + else: + raise ValueError("Invalid observation id") + return oids + + def get_hap_hst_link(self, observation_id): + """ + Returns the related members of hap and hst observations + + Parameters + ---------- + observation_id : string + id of the observation to be downloaded, mandatory + The identifier of the observation we want to retrieve, regardless + of whether it is simple or composite. + + Returns + ------- + A list of strings with the observation_id of the associated + observations + """ + observation_type = self.get_observation_type(observation_id) + if 'Composite' in observation_type: + raise ValueError("HAP-HST link is only available for simple observations. Input observation is Composite.") + elif 'HAP' in observation_type: + oids = self._select_related_members(observation_id) + elif 'HST' in observation_type: + query = f"select observation_id from ehst.observation where obs_type='HAP Simple' and members like '%{observation_id}%'" + job = self.query_hst_tap(query=query) + oids = job["observation_id"].pformat(show_name=False) + else: + raise ValueError("Invalid observation id") + return oids + + def get_observation_type(self, observation_id): + """ + Returns the type of an observation + + Parameters + ---------- + observation_id : string + id of the observation to be downloaded, mandatory + The identifier of the observation we want to retrieve, regardless + of whether it is simple or composite. + + Returns + ------- + String with the observation type + """ + if observation_id is None: + raise ValueError("Please input an observation id") + + query = f"select obs_type from ehst.observation where observation_id='{observation_id}'" + job = self.query_hst_tap(query=query) + if any(job["obs_type"]): + obs_type = self._get_decoded_string(string=job["obs_type"][0]) + else: + raise ValueError("Invalid Observation ID") + return obs_type + + def _select_related_members(self, observation_id): + query = f"select members from ehst.observation where observation_id='{observation_id}'" + job = self.query_hst_tap(query=query) + oids = self._get_decoded_string(string=job["members"][0]).replace("caom:HST/", "").split(" ") + return oids + + def _select_related_composite(self, observation_id): + query = f"select observation_id from ehst.observation where members like '%{observation_id}%'" + job = self.query_hst_tap(query=query) + oids = job["observation_id"].pformat(show_name=False) + return oids + def __validate_product_type(self, product_type): - if(product_type not in self.product_types): + if (product_type not in self.product_types): raise ValueError("This product_type is not allowed") def _get_product_filename(self, product_type, filename): - if(product_type == "PRODUCT"): + if (product_type == "PRODUCT"): return filename - elif(product_type == "SCIENCE_PRODUCT"): + elif (product_type == "SCIENCE_PRODUCT"): log.info("This is a SCIENCE_PRODUCT, the filename will be " "renamed to " + filename + ".fits.gz") return filename + ".fits.gz" @@ -263,27 +347,27 @@ def cone_search(self, coordinates, radius, filename=None, radius_in_grades = radius.to(units.deg).value ra = coord.ra.deg dec = coord.dec.deg - query = "select o.observation_id, "\ - "o.start_time, o.end_time, o.start_time_mjd, "\ - "o.end_time_mjd, o.exposure_duration, o.release_date, "\ - "o.run_id, o.program_id, o.set_id, o.collection, "\ - "o.members_number, o.instrument_configuration, "\ - "o.instrument_name, o.obs_type, o.target_moving, "\ - "o.target_name, o.target_description, o.proposal_id, "\ - "o.pi_name, prop.title, pl.metadata_provenance, "\ - "pl.data_product_type, pl.software_version, pos.ra, "\ - "pos.dec, pos.gal_lat, pos.gal_lon, pos.ecl_lat, "\ - "pos.ecl_lon, pos.fov_size, en.wave_central, "\ - "en.wave_bandwidth, en.wave_max, en.wave_min, "\ - "en.filter from ehst.observation o join ehst.proposal "\ - "prop on o.proposal_id=prop.proposal_id join ehst.plane "\ - "pl on pl.observation_id=o.observation_id join "\ - "ehst.position pos on pos.plane_id = pl.plane_id join "\ - "ehst.energy en on en.plane_id=pl.plane_id where "\ - "pl.main_science_plane='true' and 1=CONTAINS(POINT('ICRS', "\ - "pos.ra, pos.dec),CIRCLE('ICRS', {0}, {1}, {2})) order "\ - "by prop.proposal_id desc".format(str(ra), str(dec), - str(radius_in_grades)) + query = ("select o.observation_id, " + "o.start_time, o.end_time, o.start_time_mjd, " + "o.end_time_mjd, o.exposure_duration, o.release_date, " + "o.run_id, o.program_id, o.set_id, o.collection, " + "o.members_number, o.instrument_configuration, " + "o.instrument_name, o.obs_type, o.target_moving, " + "o.target_name, o.target_description, o.proposal_id, " + "o.pi_name, prop.title, pl.metadata_provenance, " + "pl.data_product_type, pl.software_version, pos.ra, " + "pos.dec, pos.gal_lat, pos.gal_lon, pos.ecl_lat, " + "pos.ecl_lon, pos.fov_size, en.wave_central, " + "en.wave_bandwidth, en.wave_max, en.wave_min, " + "en.filter from ehst.observation o join ehst.proposal " + "prop on o.proposal_id=prop.proposal_id join ehst.plane " + "pl on pl.observation_id=o.observation_id join " + "ehst.position pos on pos.plane_id = pl.plane_id join " + "ehst.energy en on en.plane_id=pl.plane_id where " + "pl.main_science_plane='true' and 1=CONTAINS(POINT('ICRS', " + f"pos.ra, pos.dec),CIRCLE('ICRS', {str(ra)}, {str(dec)}, {str(radius_in_grades)})) order " + "by prop.proposal_id desc") + print("type: " + str(type(query))) if verbose: log.info(query) table = self.query_hst_tap(query=query, async_job=async_job, @@ -380,19 +464,20 @@ def cone_search_criteria(self, radius, target=None, raise TypeError("Please use only target or coordinates as" "parameter.") if target: - ra, dec = self._query_tap_target(target) + coord = self._query_tap_target(target) else: coord = self._getCoordInput(coordinates) - ra = coord.ra.deg - dec = coord.dec.deg + + ra = coord.ra.deg + dec = coord.dec.deg if type(radius) == int or type(radius) == float: radius_in_grades = Angle(radius, units.arcmin).deg else: radius_in_grades = radius.to(units.deg).value - cone_query = "1=CONTAINS(POINT('ICRS', pos.ra, pos.dec),"\ - "CIRCLE('ICRS', {0}, {1}, {2}))".\ - format(str(ra), str(dec), str(radius_in_grades)) + cone_query = "1=CONTAINS(POINT('ICRS', pos.ra, pos.dec)," \ + "CIRCLE('ICRS', {0}, {1}, {2}))". \ + format(str(ra), str(dec), str(radius_in_grades)) query = "{}{})".format(crit_query, cone_query) if verbose: log.info(query) @@ -415,15 +500,15 @@ def _query_tap_target(self, target): target_result = target_response.json()['data'][0] ra = target_result['RA_DEGREES'] dec = target_result['DEC_DEGREES'] - return ra, dec + return SkyCoord(ra=ra, dec=dec, unit="deg") except KeyError as e: raise ValueError("This target cannot be resolved") def query_metadata(self, output_format='votable', verbose=False): return - def query_target(self, name, filename=None, output_format='votable', - verbose=False): + def query_target(self, name, *, filename=None, output_format='votable', + verbose=False, async_job=False, radius=7): """ It executes a query over EHST and download the xml with the results. @@ -439,34 +524,22 @@ def query_target(self, name, filename=None, output_format='votable', verbose : bool optional, default 'False' Flag to display information about the process + async_job : bool, optional, default 'False' + executes the query (job) in asynchronous/synchronous mode (default + synchronous) + radius : int + optional, default 7 + radius in arcmin (int, float) or quantity of the cone_search Returns ------- Table with the result of the query. It downloads metadata as a file. """ + coordinates = self._query_tap_target(name) + table = self.cone_search(coordinates, radius, filename=filename, output_format=output_format, + verbose=verbose, async_job=async_job) - params = {"RESOURCE_CLASS": "OBSERVATION", - "USERNAME": "ehst-astroquery", - "SELECTED_FIELDS": "OBSERVATION", - "QUERY": "(TARGET.TARGET_NAME=='" + name + "')", - "RETURN_TYPE": str(output_format)} - response = self._request('GET', self.metadata_url, save=True, - cache=True, - params=params) - - if verbose: - log.info(self.metadata_url + "?RESOURCE_CLASS=OBSERVATION&" - "SELECTED_FIELDS=OBSERVATION&QUERY=(TARGET.TARGET_NAME" - "=='" + name + "')&USERNAME=ehst-astroquery&" - "RETURN_TYPE=" + str(output_format)) - log.info(self.copying_string.format(filename)) - if filename is None: - filename = "target.xml" - - shutil.move(response, filename) - - return modelutils.read_results_table_from_file(filename, - str(output_format)) + return table def query_hst_tap(self, query, async_job=False, output_file=None, output_format="votable", verbose=False): @@ -495,13 +568,12 @@ def query_hst_tap(self, query, async_job=False, output_file=None, job = self._tap.launch_job_async(query=query, output_file=output_file, output_format=output_format, - verbose=False, - dump_to_file=output_file - is not None) + verbose=verbose, + dump_to_file=output_file is not None) else: job = self._tap.launch_job(query=query, output_file=output_file, output_format=output_format, - verbose=False, + verbose=verbose, dump_to_file=output_file is not None) table = job.get_results() return table @@ -582,9 +654,9 @@ def query_criteria(self, calibration_level=None, parameters.append("(o.instrument_configuration LIKE '%{}%')" .format("%' OR o.instrument_configuration " "LIKE '%".join(filters))) - query = "select o.*, p.calibration_level, p.data_product_type, "\ - "pos.ra, pos.dec from ehst.observation AS o JOIN "\ - "ehst.plane as p on o.observation_uuid=p.observation_uuid "\ + query = "select o.*, p.calibration_level, p.data_product_type, " \ + "pos.ra, pos.dec from ehst.observation AS o JOIN " \ + "ehst.plane as p on o.observation_uuid=p.observation_uuid " \ "JOIN ehst.position as pos on p.plane_id = pos.plane_id" if parameters: query += " where({})".format(" AND ".join(parameters)) @@ -600,7 +672,7 @@ def query_criteria(self, calibration_level=None, def __get_calibration_level(self, calibration_level): condition = "" - if(calibration_level is not None): + if (calibration_level is not None): if isinstance(calibration_level, str): condition = calibration_level elif isinstance(calibration_level, int): @@ -696,5 +768,11 @@ def _getCoordInput(self, value): else: return value + def _get_decoded_string(self, string): + try: + return string.decode('utf-8') + except (UnicodeDecodeError, AttributeError): + return string + ESAHubble = ESAHubbleClass() diff --git a/astroquery/esa/hubble/tests/test_esa_hubble.py b/astroquery/esa/hubble/tests/test_esa_hubble.py index 51ab6a92ed..107bff87be 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble.py @@ -11,13 +11,15 @@ """ +import numpy as np import pytest import os +from unittest.mock import patch from requests.models import Response from astroquery.esa.hubble import ESAHubbleClass from astroquery.esa.hubble.tests.dummy_tap_handler import DummyHubbleTapHandler -from astroquery.utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from astropy import coordinates from unittest.mock import MagicMock from astropy.table.table import Table @@ -64,6 +66,14 @@ def ehst_cone_search(request): return mp +class MockResponse: + observation_id = 'test' + + @staticmethod + def pformat(): + return True + + class TestESAHubble: def get_dummy_tap_handler(self): @@ -122,12 +132,14 @@ def test_get_postcard(self): filename="X0MC5101T.vot", verbose=True) - def test_query_target(self): - parameters = {'name': "m31", - 'verbose': True} + @patch.object(ESAHubbleClass, 'cone_search') + @patch.object(ESAHubbleClass, '_query_tap_target') + def test_query_target(self, mock_query_tap_target, mock_cone_search): + mock_query_tap_target.return_value = 10, 10 + mock_cone_search.return_value = "test" ehst = ESAHubbleClass(self.get_dummy_tap_handler()) - ehst.query_target(name=parameters['name'], - verbose=parameters['verbose']) + table = ehst.query_target(name="test") + assert table == "test" def test_cone_search(self): coords = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", @@ -178,7 +190,7 @@ def test_cone_search_coords(self): parameters['verbose']) with pytest.raises(ValueError) as err: ehst._getCoordInput(1234) - assert "Coordinates must be either a string or "\ + assert "Coordinates must be either a string or " \ "astropy.coordinates" in err.value.args[0] def test_query_hst_tap(self): @@ -349,32 +361,32 @@ def test_cone_search_criteria(self): 'filename': "output_test_query_by_criteria.vot.gz", 'output_format': "votable", 'verbose': True} - test_query = "select o.*, p.calibration_level, p.data_product_type, "\ - "pos.ra, pos.dec from ehst.observation AS o JOIN "\ - "ehst.plane as p on o.observation_uuid=p.observation_"\ - "uuid JOIN ehst.position as pos on p.plane_id = "\ - "pos.plane_id where((o.collection LIKE '%HST%') AND "\ - "(o.instrument_name LIKE '%WFPC2%') AND "\ - "(o.instrument_configuration LIKE '%F606W%') AND "\ - "1=CONTAINS(POINT('ICRS', pos.ra, pos.dec),"\ - "CIRCLE('ICRS', 10.6847083, 41.26875, "\ + test_query = "select o.*, p.calibration_level, p.data_product_type, " \ + "pos.ra, pos.dec from ehst.observation AS o JOIN " \ + "ehst.plane as p on o.observation_uuid=p.observation_" \ + "uuid JOIN ehst.position as pos on p.plane_id = " \ + "pos.plane_id where((o.collection LIKE '%HST%') AND " \ + "(o.instrument_name LIKE '%WFPC2%') AND " \ + "(o.instrument_configuration LIKE '%F606W%') AND " \ + "1=CONTAINS(POINT('ICRS', pos.ra, pos.dec)," \ + "CIRCLE('ICRS', 10.6847083, 41.26875, " \ "0.11666666666666667)))" parameters3 = {'query': test_query, 'output_file': "output_test_query_by_criteria.vot.gz", 'output_format': "votable", 'verbose': False} ehst = ESAHubbleClass(self.get_dummy_tap_handler()) - query_criteria_query = "select o.*, p.calibration_level, "\ - "p.data_product_type, pos.ra, pos.dec from "\ - "ehst.observation AS o JOIN ehst.plane as p "\ - "on o.observation_uuid=p.observation_uuid "\ - "JOIN ehst.position as pos on p.plane_id = "\ - "pos.plane_id where((o.collection LIKE "\ - "'%HST%') AND (o.instrument_name LIKE "\ - "'%WFPC2%') AND (o.instrument_configuration "\ + query_criteria_query = "select o.*, p.calibration_level, " \ + "p.data_product_type, pos.ra, pos.dec from " \ + "ehst.observation AS o JOIN ehst.plane as p " \ + "on o.observation_uuid=p.observation_uuid " \ + "JOIN ehst.position as pos on p.plane_id = " \ + "pos.plane_id where((o.collection LIKE " \ + "'%HST%') AND (o.instrument_name LIKE " \ + "'%WFPC2%') AND (o.instrument_configuration " \ "LIKE '%F606W%'))" ehst.query_criteria = MagicMock(return_value=query_criteria_query) - target = {'RA_DEGREES': '10.6847083', 'DEC_DEGREES': '41.26875'} + target = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') ehst._query_tap_target = MagicMock(return_value=target) ehst.cone_search_criteria(target=parameters1['target'], radius=parameters1['radius'], @@ -419,14 +431,14 @@ def test_cone_search_criteria(self): output_format=parameters1 ['output_format'], verbose=parameters1['verbose']) - assert "Please use only target or coordinates as"\ - "parameter." in err.value.args[0] + assert "Please use only target or coordinates as" \ + "parameter." in err.value.args[0] def test_query_criteria_no_params(self): ehst = ESAHubbleClass(self.get_dummy_tap_handler()) ehst.query_criteria(async_job=False, output_file="output_test_query_" - "by_criteria.vot.gz", + "by_criteria.vot.gz", output_format="votable", verbose=True) parameters = {'query': "select o.*, p.calibration_level, " @@ -445,8 +457,175 @@ def test_empty_list(self): ehst.query_criteria(instrument_name=[1], async_job=False, output_file="output_test_query_" - "by_criteria.vot.gz", + "by_criteria.vot.gz", output_format="votable", verbose=True) - assert "One of the lists is empty or there are "\ + assert "One of the lists is empty or there are " \ "elements that are not strings" in err.value.args[0] + + def test__get_decoded_string(self): + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + dummy = '\x74\x65\x73\x74' + decoded_string = ehst._get_decoded_string(dummy) + assert decoded_string == 'test' + + def test__get_decoded_string_unicodedecodeerror(self): + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + dummy = '\xd0\x91' + decoded_string = ehst._get_decoded_string(dummy) + assert decoded_string == dummy + + def test__get_decoded_string_attributeerror(self): + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + dummy = True + decoded_string = ehst._get_decoded_string(dummy) + assert decoded_string == dummy + + @patch.object(ESAHubbleClass, 'query_hst_tap') + def test__select_related_composite(self, mock_query): + arr = {'a': np.array([1, 4], dtype=np.int32), + 'b': [2.0, 5.0], + 'observation_id': ['x', 'y']} + data_table = Table(arr) + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + mock_query.return_value = data_table + dummy_obs_id = "1234" + oids = ehst._select_related_composite(dummy_obs_id) + assert oids == ['x', 'y'] + + @patch.object(ESAHubbleClass, 'query_hst_tap') + def test__select_related_members(self, mock_query): + arr = {'a': np.array([1, 4], dtype=np.int32), + 'b': [2.0, 5.0], + 'members': ['caom:HST/test', 'y']} + data_table = Table(arr) + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + mock_query.return_value = data_table + dummy_obs_id = "1234" + oids = ehst._select_related_members(dummy_obs_id) + assert oids == ['test'] + + @patch.object(ESAHubbleClass, 'query_hst_tap') + def test_get_observation_type(self, mock_query): + arr = {'a': np.array([1, 4], dtype=np.int32), + 'b': [2.0, 5.0], + 'obs_type': ['HST Test', 'y']} + data_table = Table(arr) + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + mock_query.return_value = data_table + dummy_obs_id = "1234" + oids = ehst.get_observation_type(dummy_obs_id) + assert oids == 'HST Test' + + def test_get_observation_type_obs_id_none_valueerror(self): + with pytest.raises(ValueError): + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + dummy_obs_id = None + ehst.get_observation_type(dummy_obs_id) + + @patch.object(ESAHubbleClass, 'query_hst_tap') + def test_get_observation_type_invalid_obs_id_valueerror(self, mock_query): + with pytest.raises(ValueError): + arr = {'a': np.array([], dtype=np.int32), + 'b': [], + 'obs_type': []} + data_table = Table(arr) + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + mock_query.return_value = data_table + dummy_obs_id = '1234' + ehst.get_observation_type(dummy_obs_id) + + @patch.object(ESAHubbleClass, 'query_hst_tap') + @patch.object(ESAHubbleClass, 'get_observation_type') + def test_get_hst_link(self, mock_observation_type, mock_query): + mock_observation_type.return_value = "HST" + arr = {'a': np.array([1], dtype=np.int32), + 'b': [2.0], + 'observation_id': ['1234']} + data_table = Table(arr) + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + mock_query.return_value = data_table + dummy_obs_id = "1234" + oids = ehst.get_hap_hst_link(dummy_obs_id) + assert oids == ['1234'] + + @patch.object(ESAHubbleClass, 'get_observation_type') + @patch.object(ESAHubbleClass, '_select_related_members') + def test_get_hap_link(self, mock_select_related_members, mock_observation_type): + mock_select_related_members.return_value = 'test' + mock_observation_type.return_value = "HAP" + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + dummy_obs_id = "1234" + oids = ehst.get_hap_hst_link(dummy_obs_id) + assert oids == 'test' + + @patch.object(ESAHubbleClass, 'get_observation_type') + def test_get_hap_hst_link_invalid_id_valueerror(self, mock_observation_type): + with pytest.raises(ValueError): + mock_observation_type.return_value = "valueerror" + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + dummy_obs_id = "1234" + ehst.get_hap_hst_link(dummy_obs_id) + + @patch.object(ESAHubbleClass, 'get_observation_type') + def test_get_hap_hst_link_compositeerror(self, mock_observation_type): + with pytest.raises(ValueError): + mock_observation_type.return_value = "HAP Composite" + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + dummy_obs_id = "1234" + ehst.get_hap_hst_link(dummy_obs_id) + + @patch.object(ESAHubbleClass, '_select_related_members') + @patch.object(ESAHubbleClass, 'get_observation_type') + def test_get_member_observations_composite(self, mock_observation_type, mock_select_related_members): + mock_observation_type.return_value = "Composite" + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + mock_select_related_members.return_value = 'test' + dummy_obs_id = "1234" + oids = ehst.get_member_observations(dummy_obs_id) + assert oids == 'test' + + @patch.object(ESAHubbleClass, '_select_related_composite') + @patch.object(ESAHubbleClass, 'get_observation_type') + def test_get_member_observations_simple(self, mock_observation_type, mock_select_related_composite): + mock_observation_type.return_value = "Simple" + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + mock_select_related_composite.return_value = 'test' + dummy_obs_id = "1234" + oids = ehst.get_member_observations(dummy_obs_id) + assert oids == 'test' + + @patch.object(ESAHubbleClass, 'get_observation_type') + def test_get_member_observations_invalid_id_valueerror(self, mock_observation_type): + with pytest.raises(ValueError): + mock_observation_type.return_value = "valueerror" + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + dummy_obs_id = "1234" + ehst.get_member_observations(dummy_obs_id) + + @patch.object(ESAHubbleClass, 'query_criteria') + @patch.object(ESAHubbleClass, '_query_tap_target') + @patch.object(ESAHubbleClass, 'query_hst_tap') + def test_cone_search_criteria_only_target(self, mock_query_hst_tap, mock__query_tap_target, mock_query_criteria): + mock_query_criteria.return_value = "Simple query" + mock__query_tap_target.return_value = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') + mock_query_hst_tap.return_value = "table" + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + oids = ehst.cone_search_criteria(target="m11", radius=1) + assert oids == 'table' + + @patch.object(ESAHubbleClass, 'query_criteria') + @patch.object(ESAHubbleClass, 'query_hst_tap') + def test_cone_search_criteria_only_coordinates(self, mock_query_hst_tap, mock_query_criteria): + mock_query_criteria.return_value = "Simple query" + mock_query_hst_tap.return_value = "table" + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + oids = ehst.cone_search_criteria(coordinates="00h42m44.51s +41d16m08.45s", radius=1) + assert oids == 'table' + + @patch.object(ESAHubbleClass, 'query_criteria') + def test_cone_search_criteria_typeerror(self, mock_query_criteria): + mock_query_criteria.return_value = "Simple query" + with pytest.raises(TypeError): + ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst.cone_search_criteria(coordinates="00h42m44.51s +41d16m08.45s", target="m11", radius=1) diff --git a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py index 605621e10d..79c385c521 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py @@ -13,21 +13,12 @@ """ import tempfile -import unittest import os import pytest -from astropy.tests.helper import remote_data -from requests.models import Response from astroquery.esa.hubble import ESAHubble -from astroquery.utils.testing_tools import MockResponse from astropy import coordinates -from unittest.mock import MagicMock -from astropy.table.table import Table -import shutil import random -from PIL import Image - esa_hubble = ESAHubble() @@ -49,15 +40,14 @@ def remove_last_job(): @pytest.mark.remote_data class TestEsaHubbleRemoteData: - obs_query = "select top 2050 o.observation_id from ehst.observation o" top_obs_query = "select top 100 o.observation_id from ehst.observation o" - hst_query = "select top 50 o.observation_id from ehst.observation "\ + hst_query = "select top 50 o.observation_id from ehst.observation " \ "o where o.collection='HST'" - top_artifact_query = "select top 50 a.artifact_id, a.observation_id "\ + top_artifact_query = "select top 50 a.artifact_id, a.observation_id " \ " from ehst.artifact a" temp_folder = create_temp_folder() @@ -84,12 +74,6 @@ def test_get_artifact(self): esa_hubble.get_artifact(artifact_id, temp_file) assert os.path.exists(temp_file) - def test_query_target(self): - temp_file = self.temp_folder.name + "/" + "m31_query.xml" - table = esa_hubble.query_target("m31", temp_file) - assert os.path.exists(temp_file) - assert 'OBSERVATION_ID' in table.columns - def test_cone_search(self): esa_hubble = ESAHubble() c = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') @@ -99,3 +83,40 @@ def test_cone_search(self): assert 'observation_id' in table.columns assert len(table) > 0 remove_last_job() + + # tests for get_related_members + def test_hst_composite_to_hst_simple(self): + esa_hubble = ESAHubble() + result = esa_hubble.get_member_observations('jdrz0c010') + assert result == ['jdrz0cjxq', 'jdrz0cjyq'] + + def test_hst_simple_to_hst_composite(self): + esa_hubble = ESAHubble() + result = esa_hubble.get_member_observations('jdrz0cjxq') + assert result == ['jdrz0c010'] + + def test_hap_composite_to_hap_simple(self): + esa_hubble = ESAHubble() + result = esa_hubble.get_member_observations('hst_15446_4v_acs_wfc_f606w_jdrz4v') + assert result == ['hst_15446_4v_acs_wfc_f606w_jdrz4vkv', 'hst_15446_4v_acs_wfc_f606w_jdrz4vkw'] + + def test_hap_simple_to_hap_composite(self): + esa_hubble = ESAHubble() + result = esa_hubble.get_member_observations('hst_16316_71_acs_sbc_f150lp_jec071i9') + assert result == ['hst_16316_71_acs_sbc_f150lp_jec071'] + + def test_hap_simple_to_hst_simple(self): + esa_hubble = ESAHubble() + result = esa_hubble.get_hap_hst_link('hst_16316_71_acs_sbc_f150lp_jec071i9') + assert result == ['jec071i9q'] + + def test_hst_simple_to_hap_simple(self): + esa_hubble = ESAHubble() + result = esa_hubble.get_hap_hst_link('jec071i9q') + assert result == ['hst_16316_71_acs_sbc_f150lp_jec071i9'] + + def test_query_target(self): + temp_file = self.temp_folder.name + "/" + "m31_query.xml" + table = esa_hubble.query_target(name="m3", filename=temp_file) + assert os.path.exists(temp_file) + assert 'observation_id' in table.columns diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index d67a51cc50..a9a4bd59c5 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -647,17 +647,6 @@ def login(self, *, user=None, password=None, credentials_file=None, if token: self.set_token(token=token) - def login_gui(self, *, verbose=False): - """Performs a login using a GUI dialog - TAP+ only - - Parameters - ---------- - verbose : bool, optional, default 'False' - flag to display information about the process - """ - return self.__jwsttap.login_gui(verbose) - def logout(self, *, verbose=False): """Performs a logout TAP+ only diff --git a/astroquery/esa/jwst/data_access.py b/astroquery/esa/jwst/data_access.py index 310a7133d9..f39bd35d01 100644 --- a/astroquery/esa/jwst/data_access.py +++ b/astroquery/esa/jwst/data_access.py @@ -10,6 +10,7 @@ """ from astropy.utils import data +from . import conf __all__ = ['JwstDataHandler'] @@ -17,7 +18,7 @@ class JwstDataHandler: def __init__(self, base_url=None): if base_url is None: - self.base_url = "http://jwstdummydata.com" + self.base_url = conf.JWST_DATA_SERVER else: self.base_url = base_url diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index 523fd2f5c6..54b127d16e 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -1002,14 +1002,6 @@ def test_login(self): tap.login(user='test_user', password='test_password') dummyTapHandler.check_call('login', parameters) - def test_login_gui(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) - parameters = {} - parameters['verbose'] = False - tap.login_gui() - dummyTapHandler.check_call('login_gui', parameters) - def test_logout(self): dummyTapHandler = DummyTapHandler() tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) diff --git a/astroquery/esa/xmm_newton/__init__.py b/astroquery/esa/xmm_newton/__init__.py index 0dbaceb6fa..5f5c44d99a 100644 --- a/astroquery/esa/xmm_newton/__init__.py +++ b/astroquery/esa/xmm_newton/__init__.py @@ -17,13 +17,13 @@ class Conf(_config.ConfigNamespace): """ Configuration parameters for `astroquery.esa.xmm_newton`. """ - DATA_ACTION = _config.ConfigItem("http://nxsa.esac.esa.int/" + DATA_ACTION = _config.ConfigItem("https://nxsa.esac.esa.int/" "nxsa-sl/servlet/data-action?", "Main url for retriving XSA files") - DATA_ACTION_AIO = _config.ConfigItem("http://nxsa.esac.esa.int/" + DATA_ACTION_AIO = _config.ConfigItem("https://nxsa.esac.esa.int/" "nxsa-sl/servlet/data-action-aio?", "Main url for retriving XSA files") - METADATA_ACTION = _config.ConfigItem("http://nxsa.esac.esa.int/" + METADATA_ACTION = _config.ConfigItem("https://nxsa.esac.esa.int/" "nxsa-sl/servlet/" "metadata-action?", "Main url for retriving XSA metadata") diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index 23cb392ef4..7d96d38834 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -12,13 +12,15 @@ """ import re +from getpass import getpass from ...utils.tap.core import TapPlus -from ...query import BaseQuery +from ...query import BaseQuery, QueryWithLogin import shutil import cgi from pathlib import Path import tarfile import os +import configparser from astropy.io import fits from . import conf @@ -26,12 +28,10 @@ from astropy.coordinates import SkyCoord from ...exceptions import LoginError - __all__ = ['XMMNewton', 'XMMNewtonClass'] class XMMNewtonClass(BaseQuery): - data_url = conf.DATA_ACTION data_aio_url = conf.DATA_ACTION_AIO metadata_url = conf.METADATA_ACTION @@ -39,16 +39,16 @@ class XMMNewtonClass(BaseQuery): def __init__(self, tap_handler=None): super(XMMNewtonClass, self).__init__() + self.configuration = configparser.ConfigParser() if tap_handler is None: - self._tap = TapPlus(url="http://nxsa.esac.esa.int" - "/tap-server/tap/") + self._tap = TapPlus(url="https://nxsa.esac.esa.int/tap-server/tap") else: self._tap = tap_handler self._rmf_ftp = str("http://sasdev-xmm.esac.esa.int/pub/ccf/constituents/extras/responses/") def download_data(self, observation_id, *, filename=None, verbose=False, - cache=True, **kwargs): + cache=True, prop=False, credentials_file=None, **kwargs): """ Download data from XMM-Newton @@ -63,6 +63,13 @@ def download_data(self, observation_id, *, filename=None, verbose=False, verbose : bool optional, default 'False' flag to display information about the process + prop: boolean + optional, default 'False' + flag to download proprietary data, the method will then ask the user to + input their username and password either manually or using the credentials_file + credentials_file: string + optional, default None + path to where the users config.ini file is stored with their username and password level : string level to download, optional, by default everything is downloaded values: ODF, PPS @@ -94,48 +101,44 @@ def download_data(self, observation_id, *, filename=None, verbose=False, file format, optional, by default all formats values: ASC, ASZ, FTZ, HTM, IND, PDF, PNG - Returns ------- None if not verbose. It downloads the observation indicated If verbose returns the filename """ - if filename is not None: - filename = os.path.splitext(filename)[0] - link = self.data_aio_url + "obsno=" + observation_id + # create url to access the aio + link = self._create_link(observation_id, **kwargs) - link = link + "".join("&{0}={1}".format(key, val) - for key, val in kwargs.items()) + # If the user wants to access proprietary data, ask them for their credentials + if prop: + username, password = self._get_username_and_password(credentials_file) + link = f"{link}&AIOUSER={username}&AIOPWD={password}" if verbose: log.info(link) - # we can cache this HEAD request - the _download_file one will check - # the file size and will never cache - response = self._request('HEAD', link, save=False, cache=cache) - - # Get original extension - if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']: - _, params = cgi.parse_header(response.headers['Content-Disposition']) - else: - if response.status_code == 401: - error = "Data protected by proprietary rights. Please check your credentials" - raise LoginError(error) - response.raise_for_status() - + # get response of created url + params = self._request_link(link, cache) r_filename = params["filename"] suffixes = Path(r_filename).suffixes - if filename is None: - filename = observation_id - - filename += "".join(suffixes) - - self._download_file(link, filename, head_safe=True, cache=cache) + # get desired filename + filename = self._create_filename(filename, observation_id, suffixes) + """ + If prop we change the log level so that it is above 20, this is to stop a log.debug (line 431) in query.py. + This debug reveals the url being sent which in turn reveals the users username and password + """ + if prop: + previouslevel = log.getEffectiveLevel() + log.setLevel(21) + self._download_file(link, filename, head_safe=True, cache=cache) + log.setLevel(previouslevel) + else: + self._download_file(link, filename, head_safe=True, cache=cache) if verbose: - log.info("Wrote {0} to {1}".format(link, filename)) + log.info(f"Wrote {link} to {filename}") def get_postcard(self, observation_id, *, image_type="OBS_EPIC", filename=None, verbose=False): @@ -186,12 +189,12 @@ def get_postcard(self, observation_id, *, image_type="OBS_EPIC", else: filename = observation_id + ".png" - log.info("Copying file to {0}...".format(filename)) + log.info(f"Copying file to {filename}...") shutil.move(local_filepath, filename) if verbose: - log.info("Wrote {0} to {1}".format(link, filename)) + log.info(f"Wrote {link} to {filename}") return filename @@ -273,14 +276,54 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): break if columns is None: - raise ValueError("table name specified is not found in " - "XSA TAP service") + raise ValueError("table name specified is not found in XSA TAP service") if only_names: return [c.name for c in columns] else: return columns + def _create_link(self, observation_id, **kwargs): + link = f"{self.data_aio_url}obsno={observation_id}" + link = link + "".join("&{0}={1}".format(key, val) + for key, val in kwargs.items()) + return link + + def _request_link(self, link, cache): + # we can cache this HEAD request - the _download_file one will check + # the file size and will never cache + response = self._request('HEAD', link, save=False, cache=cache) + # Get original extension + if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']: + _, params = cgi.parse_header(response.headers['Content-Disposition']) + elif response.status_code == 401: + error = "Data protected by proprietary rights. Please check your credentials" + raise LoginError(error) + elif 'Content-Type' not in response.headers: + error = "Incorrect credentials" + raise LoginError(error) + response.raise_for_status() + return params + + def _get_username_and_password(self, credentials_file): + if credentials_file is not None: + self.configuration.read(credentials_file) + xmm_username = self.configuration.get("xmm_newton", "username") + password = self.configuration.get("xmm_newton", "password") + else: + xmm_username = input("Username: ") + password, password_from_keyring = QueryWithLogin._get_password(self, service_name="xmm_newton", + username=xmm_username, reenter=False) + return xmm_username, password + + def _create_filename(self, filename, observation_id, suffixes): + if filename is not None: + filename = os.path.splitext(filename)[0] + else: + filename = observation_id + filename += "".join(suffixes) + return filename + def _parse_filename(self, filename): """Parses the file's name of a product @@ -572,9 +615,9 @@ def get_epic_metadata(self, *, target_name=None, Tables containing the metadata of the target """ if not target_name and not coordinates: - raise Exception("Input parameters needed, " - "please provide the name " - "or the coordinates of the target") + raise ValueError("Input parameters needed, " + "please provide the name " + "or the coordinates of the target") epic_source = {"table": "xsa.v_epic_source", "column": "epic_source_equatorial_spoint"} @@ -592,7 +635,7 @@ def get_epic_metadata(self, *, target_name=None, c = SkyCoord.from_name(target_name, parse=True) if type(c) is not SkyCoord: - raise Exception("The coordinates must be an " + raise TypeError("The coordinates must be an " "astroquery.coordinates.SkyCoord object") if not radius: radius = 0.1 @@ -600,29 +643,29 @@ def get_epic_metadata(self, *, target_name=None, query_fmt = ("select {} from {} " "where 1=contains({}, circle('ICRS', {}, {}, {}));") epic_source_table = self.query_xsa_tap(query_fmt.format(cols, - epic_source["table"], - epic_source["column"], - c.ra.degree, - c.dec.degree, - radius)) + epic_source["table"], + epic_source["column"], + c.ra.degree, + c.dec.degree, + radius)) cat_4xmm_table = self.query_xsa_tap(query_fmt.format(cols, - cat_4xmm["table"], - cat_4xmm["column"], - c.ra.degree, - c.dec.degree, - radius)) + cat_4xmm["table"], + cat_4xmm["column"], + c.ra.degree, + c.dec.degree, + radius)) stack_4xmm_table = self.query_xsa_tap(query_fmt.format(cols, - stack_4xmm["table"], - stack_4xmm["column"], - c.ra.degree, - c.dec.degree, - radius)) + stack_4xmm["table"], + stack_4xmm["column"], + c.ra.degree, + c.dec.degree, + radius)) slew_source_table = self.query_xsa_tap(query_fmt.format(cols, - slew_source["table"], - slew_source["column"], - c.ra.degree, - c.dec.degree, - radius)) + slew_source["table"], + slew_source["column"], + c.ra.degree, + c.dec.degree, + radius)) return epic_source_table, cat_4xmm_table, stack_4xmm_table, slew_source_table def get_epic_lightcurve(self, filename, source_number, *, diff --git a/astroquery/esa/xmm_newton/tests/data/dummy_config.ini b/astroquery/esa/xmm_newton/tests/data/dummy_config.ini new file mode 100644 index 0000000000..7983bd2ae9 --- /dev/null +++ b/astroquery/esa/xmm_newton/tests/data/dummy_config.ini @@ -0,0 +1,3 @@ +[xmm_newton] +username=test +password=test \ No newline at end of file diff --git a/astroquery/esa/xmm_newton/tests/setup_package.py b/astroquery/esa/xmm_newton/tests/setup_package.py index ae93358d3e..a3829e4510 100644 --- a/astroquery/esa/xmm_newton/tests/setup_package.py +++ b/astroquery/esa/xmm_newton/tests/setup_package.py @@ -19,6 +19,7 @@ def get_package_data(): paths = [os.path.join('data', '*.tar'), os.path.join('data', '*.xml'), + os.path.join('data', '*.ini') ] # etc, add other extensions # you can also enlist files individually by names # finally construct and return a dict for the sub module diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index d487ae6a68..e08ff4e1f0 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -1,41 +1,50 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ - @author: Elena Colomo @contact: ecolomo@esa.int - European Space Astronomy Centre (ESAC) European Space Agency (ESA) - Created on 4 Sept. 2019 """ +from unittest.mock import patch import pytest - -import sys import tarfile import os import errno import shutil -from astropy.coordinates import SkyCoord -from astropy.utils.diff import report_diff_values -from astroquery.utils.tap.core import TapPlus from ..core import XMMNewtonClass from ..tests.dummy_tap_handler import DummyXMMNewtonTapHandler from ..tests.dummy_handler import DummyHandler -from fileinput import filename -from tarfile import is_tarfile +from astroquery.exceptions import LoginError -class TestXMMNewton(): +def data_path(filename): + data_dir = os.path.join(os.path.dirname(__file__), 'data') + return os.path.join(data_dir, filename) + +class mockResponse(): + headers = {'Date': 'Wed, 24 Nov 2021 13:43:50 GMT', + 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips', + 'Content-Disposition': 'inline; filename="0560181401.tar.gz"', + 'Content-Type': 'application/x-gzip', + 'Content-Length': '6590874', 'Connection': 'close'} + status_code = 400 + + @staticmethod + def raise_for_status(): + pass + + +class TestXMMNewton(): def get_dummy_tap_handler(self): - parameterst = {'query': "select top 10 * from v_public_observations", - 'output_file': "test2.vot", - 'output_format': "votable", - 'verbose': False} - dummyTapHandler = DummyXMMNewtonTapHandler("launch_job", parameterst) + parameters = {'query': "select top 10 * from v_public_observations", + 'output_file': "test2.vot", + 'output_format': "votable", + 'verbose': False} + dummyTapHandler = DummyXMMNewtonTapHandler("launch_job", parameters) return dummyTapHandler def test_query_xsa_tap(self): @@ -72,6 +81,11 @@ def test_get_columns(self): xsa.get_columns("table", only_names=True, verbose=True) dummyTapHandler.check_call("get_columns", parameters2) + def test_get_columns_valueerror(self): + with pytest.raises(ValueError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + xsa.get_columns("", only_names=True, verbose=True) + def test_dummy_handler(self): parameters2 = {'table_name': "table", 'only_names': True, @@ -235,7 +249,7 @@ def _create_tar(self, tarname, files): os.makedirs(os.path.join(ob_name, ftype)) except OSError as exc: if exc.errno == errno.EEXIST and \ - os.path.isdir(os.path.join(ob_name, ftype)): + os.path.isdir(os.path.join(ob_name, ftype)): pass else: raise @@ -255,7 +269,7 @@ def _create_tar_lightcurves(self, tarname, files): os.makedirs(os.path.join(ob_name, ftype)) except OSError as exc: if exc.errno == errno.EEXIST and \ - os.path.isdir(os.path.join(ob_name, ftype)): + os.path.isdir(os.path.join(ob_name, ftype)): pass else: raise @@ -377,7 +391,7 @@ def test_get_epic_images(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) res = xsa.get_epic_images(_tarname, band=[], instrument=[], get_detmask=True, get_exposure_map=True) - assert len(res) == 6 # Number of different bands + assert len(res) == 6 # Number of different bands assert len(res[1]) == 9 # Number of different inst within band 1 assert len(res[2]) == 9 # Number of different inst within band 2 assert len(res[3]) == 9 # Number of different inst within band 3 @@ -510,3 +524,61 @@ def test_get_epic_lightcurve_invalid_source_number(self, capsys): % (_tarname, _invalid_source_number, _default_instrument)) os.remove(_tarname) + + def test_create_link(self): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + link = xsa._create_link("0560181401") + assert link == "https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401" + + @patch('astroquery.query.BaseQuery._request') + def test_request_link(self, mock_request): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + mock_request.return_value = mockResponse + params = xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + assert params == {'filename': '0560181401.tar.gz'} + + @patch('astroquery.query.BaseQuery._request') + def test_request_link_protected(self, mock_request): + with pytest.raises(LoginError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + + @patch('astroquery.query.BaseQuery._request') + def test_request_link_incorrect_credentials(self, mock_request): + with pytest.raises(LoginError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + dummyclass.status_code = 10 + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + + @patch('astroquery.query.BaseQuery._request') + def test_request_link_with_statuscode_401(self, mock_request): + with pytest.raises(LoginError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + dummyclass.status_code = 401 + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + + def test_get_username_and_password(self): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + file = data_path("dummy_config.ini") + username, password = xsa._get_username_and_password(file) + assert username == "test" + assert password == "test" + + def test_create_filename_None(self): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + filename = xsa._create_filename(None, "0560181401", ['.tar', '.gz']) + assert filename == "0560181401.tar.gz" + + def test_create_filename_Not_None(self): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + filename = xsa._create_filename("Test", "0560181401", ['.tar', '.gz']) + assert filename == "Test.tar.gz" diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py index c0464a87b2..15b9a50ba8 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py @@ -1,18 +1,15 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ - @author: Elena Colomo @contact: ecolomo@esa.int - European Space Astronomy Centre (ESAC) European Space Agency (ESA) - Created on 4 Sept. 2019 """ import pytest +from astroquery.exceptions import LoginError -import sys import tarfile import os import errno @@ -22,9 +19,97 @@ from astroquery.utils.tap.core import TapPlus from ..core import XMMNewtonClass +from ..tests.dummy_tap_handler import DummyXMMNewtonTapHandler class TestXMMNewtonRemote(): + _files = { + "0405320501": { + "pps": [ + "P0405320501M1S002EXPMAP1000.FTZ", + "P0405320501M1S002IMAGE_4000.FTZ", + "P0405320501M2S003EXPMAP2000.FTZ", + "P0405320501M2S003IMAGE_5000.FTZ", + "P0405320501PNS001EXPMAP3000.FTZ", + "P0405320501PNS001IMAGE_8000.FTZ", + "P0405320501M1S002EXPMAP2000.FTZ", + "P0405320501M1S002IMAGE_5000.FTZ", + "P0405320501M2S003EXPMAP3000.FTZ", + "P0405320501M2S003IMAGE_8000.FTZ", + "P0405320501PNS001EXPMAP4000.FTZ", + "P0405320501PNX000DETMSK1000.FTZ", + "P0405320501M1S002EXPMAP3000.FTZ", + "P0405320501M1S002IMAGE_8000.FTZ", + "P0405320501M2S003EXPMAP4000.FTZ", + "P0405320501M2X000DETMSK1000.FTZ", + "P0405320501PNS001EXPMAP5000.FTZ", + "P0405320501PNX000DETMSK2000.FTZ", + "P0405320501M1S002EXPMAP4000.FTZ", + "P0405320501M1X000DETMSK1000.FTZ", + "P0405320501M2S003EXPMAP5000.FTZ", + "P0405320501M2X000DETMSK2000.FTZ", + "P0405320501PNS001EXPMAP8000.FTZ", + "P0405320501PNX000DETMSK3000.FTZ", + "P0405320501M1S002EXPMAP5000.FTZ", + "P0405320501M1X000DETMSK2000.FTZ", + "P0405320501M2S003EXPMAP8000.FTZ", + "P0405320501M2X000DETMSK3000.FTZ", + "P0405320501PNS001IMAGE_1000.FTZ", + "P0405320501PNX000DETMSK4000.FTZ", + "P0405320501M1S002EXPMAP8000.FTZ", + "P0405320501M1X000DETMSK3000.FTZ", + "P0405320501M2S003IMAGE_1000.FTZ", + "P0405320501M2X000DETMSK4000.FTZ", + "P0405320501PNS001IMAGE_2000.FTZ", + "P0405320501PNX000DETMSK5000.FTZ", + "P0405320501M1S002IMAGE_1000.FTZ", + "P0405320501M1X000DETMSK4000.FTZ", + "P0405320501M2S003IMAGE_2000.FTZ", + "P0405320501M2X000DETMSK5000.FTZ", + "P0405320501PNS001IMAGE_3000.FTZ", + "P0405320501M1S002IMAGE_2000.FTZ", + "P0405320501M1X000DETMSK5000.FTZ", + "P0405320501M2S003IMAGE_3000.FTZ", + "P0405320501PNS001EXPMAP1000.FTZ", + "P0405320501PNS001IMAGE_4000.FTZ", + "P0405320501M1S002IMAGE_3000.FTZ", + "P0405320501M2S003EXPMAP1000.FTZ", + "P0405320501M2S003IMAGE_4000.FTZ", + "P0405320501PNS001EXPMAP2000.FTZ", + "P0405320501PNS001IMAGE_5000.FTZ", + "P0405320501PNU001IMAGE_5000.FTZ", + "P0405320501PNX001IMAGE_5000.FTZ" + ] + } + } + + def get_dummy_tap_handler(self): + parameters = {'query': "select top 10 * from v_public_observations", + 'output_file': "test2.vot", + 'output_format': "votable", + 'verbose': False} + dummyTapHandler = DummyXMMNewtonTapHandler("launch_job", parameters) + return dummyTapHandler + + def _create_tar(self, tarname, files): + with tarfile.open(tarname, "w") as tar: + for ob_name, ob in self._files.items(): + for ftype, ftype_val in ob.items(): + for f in ftype_val: + try: + os.makedirs(os.path.join(ob_name, ftype)) + except OSError as exc: + if exc.errno == errno.EEXIST and \ + os.path.isdir(os.path.join(ob_name, ftype)): + pass + else: + raise + _file = open(os.path.join(ob_name, ftype, f), "w") + _file.close() + tar.add(os.path.join(ob_name, ftype, f)) + os.remove(os.path.join(ob_name, ftype, f)) + shutil.rmtree(os.path.join(ob_name, ftype)) + shutil.rmtree(ob_name) @pytest.mark.remote_data def test_download_data(self): @@ -65,23 +150,6 @@ def test_get_postcard_filename(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) xsa.get_postcard(**parameters) - @pytest.mark.remote_data - def test_get_epic_spectra(self): - _tarname = "tarfile.tar" - _source_number = 83 - _instruments = ["M1", "M1_arf", "M1_bkg", "M1_rmf", - "M2", "M2_arf", "M2_bkg", "M2_rmf", - "PN", "PN_arf", "PN_bkg", "PN_rmf"] - self._create_tar(_tarname, self._files) - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - res = xsa.get_epic_spectra(_tarname, _source_number, - instrument=[]) - assert len(res) == 8 - # Removing files created in this test - for ob_name in self._files: - shutil.rmtree(ob_name) - os.remove(_tarname) - @pytest.mark.remote_data def test_get_epic_metadata(self): tap_url = "http://nxsadev.esac.esa.int/tap-server/tap/" @@ -125,3 +193,44 @@ def test_get_epic_metadata(self): c.dec.degree, radius)) assert report_diff_values(slew_source, table) + + @pytest.mark.remote_data + @pytest.mark.xfail(raises=LoginError) + def test_download_proprietary_data_incorrect_credentials(self): + parameters = {'observation_id': "0762470101", + 'prop': 'True', + 'credentials_file': "astroquery/esa/xmm_newton/tests/data/dummy_config.ini", + 'level': "PPS", + 'name': 'OBSMLI', + 'filename': 'single', + 'instname': 'OM', + 'extension': 'FTZ', + 'verbose': False} + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + xsa.download_data(**parameters) + + @pytest.mark.remote_data + @pytest.mark.xfail(raises=LoginError) + def test_download_proprietary_data_without_credentials(self): + parameters = {'observation_id': "0883780101", + 'level': "PPS", + 'name': 'OBSMLI', + 'filename': 'single', + 'instname': 'OM', + 'extension': 'FTZ', + 'verbose': False} + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + xsa.download_data(**parameters) + + @pytest.mark.remote_data + def test_get_epic_spectra(self): + _tarname = "tarfile.tar" + _source_number = 83 + _instruments = ["M1", "M1_arf", "M1_bkg", "M1_rmf", + "M2", "M2_arf", "M2_bkg", "M2_rmf", + "PN", "PN_arf", "PN_bkg", "PN_rmf"] + self._create_tar(_tarname, self._files) + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + res = xsa.get_epic_spectra(_tarname, _source_number, + instrument=[]) + assert len(res) == 0 diff --git a/astroquery/eso/tests/test_eso.py b/astroquery/eso/tests/test_eso.py index 5ee246c7e2..a78cfecdbc 100644 --- a/astroquery/eso/tests/test_eso.py +++ b/astroquery/eso/tests/test_eso.py @@ -1,6 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import os -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ...eso import Eso diff --git a/astroquery/exceptions.py b/astroquery/exceptions.py index d8522457ed..27651416d8 100644 --- a/astroquery/exceptions.py +++ b/astroquery/exceptions.py @@ -8,7 +8,7 @@ __all__ = ['TimeoutError', 'InvalidQueryError', 'RemoteServiceError', 'TableParseError', 'LoginError', 'ResolverError', 'NoResultsWarning', 'LargeQueryWarning', 'InputWarning', - 'AuthenticationWarning', 'MaxResultsWarning'] + 'AuthenticationWarning', 'MaxResultsWarning', 'CorruptDataWarning'] class TimeoutError(Exception): @@ -98,6 +98,14 @@ class MaxResultsWarning(AstropyWarning): pass +class CorruptDataWarning(AstropyWarning): + """ + Astroquery warning class to be issued when there is a sign that the + (partially) downloaded data are corrupt. + """ + pass + + class EmptyResponseError(ValueError): """ Astroquery error class to be raised when the query returns an empty result diff --git a/astroquery/fermi/tests/test_fermi.py b/astroquery/fermi/tests/test_fermi.py index 70031db4b8..0485a7e354 100644 --- a/astroquery/fermi/tests/test_fermi.py +++ b/astroquery/fermi/tests/test_fermi.py @@ -4,7 +4,7 @@ import requests import pytest import astropy.coordinates as coord -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ... import fermi DATA_FILES = {'async': "query_result_m31.html", diff --git a/astroquery/gaia/core.py b/astroquery/gaia/core.py index 7170f72214..6453dfae0e 100644 --- a/astroquery/gaia/core.py +++ b/astroquery/gaia/core.py @@ -11,8 +11,7 @@ European Space Agency (ESA) Created on 30 jun. 2016 - - +Modified on 18 Ene. 2022 by mhsarmiento """ from requests import HTTPError @@ -32,6 +31,8 @@ from astropy.io import fits from astropy.table import Table from astropy import units as u +import warnings +from astroquery.exceptions import InputWarning class GaiaClass(TapPlus): @@ -77,9 +78,9 @@ def login(self, user=None, password=None, credentials_file=None, verbose=False): """Performs a login. User and password arguments can be used or a file that contains - user name and password - (2 lines: one for user name and the following one for the password). - If no arguments are provided, a prompt asking for user name and + username and password + (2 lines: one for username and the following one for the password). + If no arguments are provided, a prompt asking for username and password will appear. Parameters @@ -172,13 +173,18 @@ def load_data(self, ids, data_release=None, data_structure='INDIVIDUAL', retriev By default, it takes the current default one. data_structure: str, optional, default 'INDIVIDUAL' it can be 'INDIVIDUAL', 'COMBINED', 'RAW': - 'INDIVIDUAL' means... - 'COMBINED' means... - 'RAW' means... - retrieval_type : str, optional, default 'ALL' - retrieval type identifier. It can be either 'epoch_photometry' - for compatibility reasons or 'ALL' to retrieve all data from - the list of sources. + 'INDIVIDUAL' means products are provided in separate files for each sourceId. All files are zipped in a single + bundle, even if only one source/file is considered + 'COMBINED' means products are provided in a single file concatenating the data of all sourceIds together. + How this is organised depends on the chosen format + 'RAW' means products are provided following a Data Model similar to that used in the MDB, meaning in + particular that parameters stored as arrays will remain as such. Like in the COMBINED structure, a single + file is provided for the data of all sourceIds together, but in this case there will be always be one + row per sourceId + retrieval_type : str, optional, default 'ALL' to retrieve all data from the list of sources + retrieval type identifier. For GAIA DR2 possible values are ['EPOCH_PHOTOMETRY'] + For future GAIA DR3 (Once published), possible values will be ['EPOC_PHOTOMETRY', 'RVS', 'XP_CONTINUOUS', + 'XP_SAMPLED', 'MCMC_GSPPHOT' or 'MCMC_MSC'] valid_data : bool, optional, default True By default, the epoch photometry service returns only valid data, that is, all data rows where flux is not null and @@ -193,7 +199,7 @@ def load_data(self, ids, data_release=None, data_structure='INDIVIDUAL', retriev By default, this value will be set to False. If it is set to 'true' the Datalink items tags will not be checked. format : str, optional, default 'votable' - loading format + loading format. Other available formats are 'csv', 'ecsv','json','votable_plain' and 'fits' output_file : string, optional, default None file where the results are saved. If it is not provided, the http response contents are returned. @@ -206,6 +212,7 @@ def load_data(self, ids, data_release=None, data_structure='INDIVIDUAL', retriev ------- A table object """ + if retrieval_type is None: raise ValueError("Missing mandatory argument 'retrieval_type'") @@ -364,12 +371,11 @@ def __query_object(self, coordinate, radius=None, width=None, height=None, ---------- coordinate : astropy.coordinate, mandatory coordinates center point - radius : astropy.units, required if no 'width' nor 'height' are - provided + radius : astropy.units if no 'width' nor 'height' are provided radius (deg) - width : astropy.units, required if no 'radius' is provided + width : astropy.units if no 'radius' is provided box width - height : astropy.units, required if no 'radius' is provided + height : astropy.units if no 'radius' is provided box height async_job : bool, optional, default 'False' executes the query (job) in asynchronous/synchronous mode (default @@ -386,8 +392,8 @@ def __query_object(self, coordinate, radius=None, width=None, height=None, coord = self.__getCoordInput(coordinate, "coordinate") job = None if radius is not None: - job = self.__cone_search(coord, radius, - async_job=async_job, verbose=verbose) + job = self.__cone_search(coord, radius, async_job=async_job, + verbose=verbose, columns=columns) else: raHours, dec = commons.coord_to_radec(coord) ra = raHours * 15.0 # Converts to degrees @@ -426,7 +432,8 @@ def __query_object(self, coordinate, radius=None, width=None, height=None, dist ASC """.format(**{'row_limit': "TOP {0}".format(self.ROW_LIMIT) if self.ROW_LIMIT > 0 else "", 'ra_column': self.MAIN_GAIA_TABLE_RA, 'dec_column': self.MAIN_GAIA_TABLE_DEC, - 'columns': columns, 'table_name': self.MAIN_GAIA_TABLE or conf.MAIN_GAIA_TABLE, 'ra': ra, 'dec': dec, + 'columns': columns, 'table_name': self.MAIN_GAIA_TABLE or conf.MAIN_GAIA_TABLE, + 'ra': ra, 'dec': dec, 'width': widthDeg.value, 'height': heightDeg.value}) if async_job: job = self.launch_job_async(query, verbose=verbose) @@ -443,11 +450,11 @@ def query_object(self, coordinate, radius=None, width=None, height=None, ---------- coordinate : astropy.coordinates, mandatory coordinates center point - radius : astropy.units, required if no 'width'/'height' are provided + radius : astropy.units if no 'width'/'height' are provided radius (deg) - width : astropy.units, required if no 'radius' is provided + width : astropy.units if no 'radius' is provided box width - height : astropy.units, required if no 'radius' is provided + height : astropy.units if no 'radius' is provided box height verbose : bool, optional, default 'False' flag to display information about the process @@ -469,11 +476,11 @@ def query_object_async(self, coordinate, radius=None, width=None, ---------- coordinate : astropy.coordinates, mandatory coordinates center point - radius : astropy.units, required if no 'width'/'height' are provided + radius : astropy.units if no 'width'/'height' are provided radius - width : astropy.units, required if no 'radius' is provided + width : astropy.units if no 'radius' is provided box width - height : astropy.units, required if no 'radius' is provided + height : astropy.units if no 'radius' is provided box height verbose : bool, optional, default 'False' flag to display information about the process @@ -503,8 +510,7 @@ def __cone_search(self, coordinate, radius, table_name=None, coordinates center point radius : astropy.units, mandatory radius - table_name : str, optional, default main gaia table - table name doing the cone search against + table_name : str, optional, default main gaia table name doing the cone search against ra_column_name : str, optional, default ra column in main gaia table ra column doing the cone search against dec_column_name : str, optional, default dec column in main gaia table @@ -531,6 +537,7 @@ def __cone_search(self, coordinate, radius, table_name=None, ------- A Job object """ + radiusDeg = None coord = self.__getCoordInput(coordinate, "coordinate") raHours, dec = commons.coord_to_radec(coord) ra = raHours * 15.0 # Converts to degrees @@ -563,7 +570,8 @@ def __cone_search(self, coordinate, radius, table_name=None, """.format(**{'ra_column': ra_column_name, 'row_limit': "TOP {0}".format(self.ROW_LIMIT) if self.ROW_LIMIT > 0 else "", 'dec_column': dec_column_name, 'columns': columns, 'ra': ra, 'dec': dec, - 'radius': radiusDeg, 'table_name': table_name or self.MAIN_GAIA_TABLE or conf.MAIN_GAIA_TABLE}) + 'radius': radiusDeg, + 'table_name': table_name or self.MAIN_GAIA_TABLE or conf.MAIN_GAIA_TABLE}) if async_job: return self.launch_job_async(query=query, @@ -596,8 +604,7 @@ def cone_search(self, coordinate, radius=None, coordinates center point radius : astropy.units, mandatory radius - table_name : str, optional, default main gaia table - table name doing the cone search against + table_name : str, optional, default main gaia table name doing the cone search against ra_column_name : str, optional, default ra column in main gaia table ra column doing the cone search against dec_column_name : str, optional, default dec column in main gaia table @@ -646,8 +653,7 @@ def cone_search_async(self, coordinate, radius=None, coordinates center point radius : astropy.units, mandatory radius - table_name : str, optional, default main gaia table - table name doing the cone search against + table_name : str, optional, default main gaia table name doing the cone search against ra_column_name : str, optional, default ra column in main gaia table ra column doing the cone search against dec_column_name : str, optional, default dec column in main gaia table @@ -750,7 +756,7 @@ def cross_match(self, full_qualified_table_name_a=None, radius=1.0, background=False, verbose=False): - """Performs a cross match between the specified tables + """Performs a cross-match between the specified tables The result is a join table (stored in the user storage area) with the identifies of both tables and the distance. TAP+ only @@ -820,12 +826,14 @@ def launch_job(self, query, name=None, output_file=None, ---------- query : str, mandatory query to be executed + name : str, optional, default None + custom name defined by the user for the job that is going to be created output_file : str, optional, default None file name where the results are saved if dumpToFile is True. If this parameter is not provided, the jobid is used instead output_format : str, optional, default 'votable' results format. Available formats are: 'votable', 'votable_plain', - 'fits', 'csv' and 'json', default is 'votable'. + 'fits', 'csv', 'ecsv' and 'json', default is 'votable'. Returned results for 'votable' and 'fits' formats are compressed gzip files. verbose : bool, optional, default 'False' @@ -842,6 +850,7 @@ def launch_job(self, query, name=None, output_file=None, ------- A Job object """ + return TapPlus.launch_job(self, query=query, name=name, output_file=output_file, output_format=output_format, @@ -861,13 +870,15 @@ def launch_job_async(self, query, name=None, output_file=None, ---------- query : str, mandatory query to be executed + name : str, optional, default None + custom name defined by the user for the job that is going to be created output_file : str, optional, default None file name where the results are saved if dumpToFile is True. If this parameter is not provided, the jobid is used instead output_format : str, optional, default 'votable' results format. Available formats are: 'votable', 'votable_plain', 'fits', 'csv' and 'json', default is 'votable'. - Returned results for 'votable' and 'fits' formats are compressed + Returned results for 'votable' 'ecsv' and 'fits' format are compressed gzip files. verbose : bool, optional, default 'False' flag to display information about the process diff --git a/astroquery/gaia/tests/data/job_2.vot.gz b/astroquery/gaia/tests/data/job_2.vot.gz new file mode 100755 index 0000000000..fc69e0d7d4 Binary files /dev/null and b/astroquery/gaia/tests/data/job_2.vot.gz differ diff --git a/astroquery/gaia/tests/test_gaia_remote.py b/astroquery/gaia/tests/test_gaia_remote.py index 98f34f100f..dcebe328b9 100644 --- a/astroquery/gaia/tests/test_gaia_remote.py +++ b/astroquery/gaia/tests/test_gaia_remote.py @@ -5,6 +5,15 @@ from .. import GaiaClass +@pytest.mark.remote_data +def test_query_object_columns_with_radius(): + # Regression test: `columns` were ignored if `radius` was provided [#2025] + Gaia = GaiaClass() + sc = SkyCoord(ra=0*u.deg, dec=0*u.deg) + table = Gaia.query_object_async(sc, radius=10*u.arcsec, columns=['ra']) + assert table.colnames == ['ra', 'dist'] + + @pytest.mark.remote_data def test_query_object_row_limit(): Gaia = GaiaClass() diff --git a/astroquery/gaia/tests/test_gaiatap.py b/astroquery/gaia/tests/test_gaiatap.py index 77abccb2ee..079019c8fe 100644 --- a/astroquery/gaia/tests/test_gaiatap.py +++ b/astroquery/gaia/tests/test_gaiatap.py @@ -16,7 +16,10 @@ """ import unittest import os +from unittest.mock import patch + import pytest +from requests import HTTPError from astroquery.gaia import conf from astroquery.gaia.core import GaiaClass @@ -40,22 +43,22 @@ def data_path(filename): class TestTap(unittest.TestCase): def test_query_object(self): - connHandler = DummyConnHandler() - tapplus = TapPlus("http://test:1111/tap", connhandler=connHandler) - tap = GaiaClass(connHandler, tapplus) + conn_handler = DummyConnHandler() + tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap = GaiaClass(conn_handler, tapplus) # Launch response: we use default response because the query contains # decimals - responseLaunchJob = DummyResponse() - responseLaunchJob.set_status_code(200) - responseLaunchJob.set_message("OK") - jobDataFile = data_path('job_1.vot') - jobData = utils.read_file_content(jobDataFile) - responseLaunchJob.set_data(method='POST', - context=None, - body=jobData, - headers=None) + response_launch_job = DummyResponse() + response_launch_job.set_status_code(200) + response_launch_job.set_message("OK") + job_data_file = data_path('job_1.vot') + job_data = utils.read_file_content(job_data_file) + response_launch_job.set_data(method='POST', + context=None, + body=job_data, + headers=None) # The query contains decimals: force default response - connHandler.set_default_response(responseLaunchJob) + conn_handler.set_default_response(response_launch_job) sc = SkyCoord(ra=29.0, dec=15.0, unit=(u.degree, u.degree), frame='icrs') with pytest.raises(ValueError) as err: @@ -121,45 +124,45 @@ def test_query_object(self): np.int32) def test_query_object_async(self): - connHandler = DummyConnHandler() - tapplus = TapPlus("http://test:1111/tap", connhandler=connHandler) - tap = GaiaClass(connHandler, tapplus) + conn_handler = DummyConnHandler() + tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap = GaiaClass(conn_handler, tapplus) jobid = '12345' # Launch response - responseLaunchJob = DummyResponse() - responseLaunchJob.set_status_code(303) - responseLaunchJob.set_message("OK") + response_launch_job = DummyResponse() + response_launch_job.set_status_code(303) + response_launch_job.set_message("OK") # list of list (httplib implementation for headers in response) - launchResponseHeaders = [ + launch_response_headers = [ ['location', 'http://test:1111/tap/async/' + jobid] - ] - responseLaunchJob.set_data(method='POST', - context=None, - body=None, - headers=launchResponseHeaders) - connHandler.set_default_response(responseLaunchJob) + ] + response_launch_job.set_data(method='POST', + context=None, + body=None, + headers=launch_response_headers) + conn_handler.set_default_response(response_launch_job) # Phase response - responsePhase = DummyResponse() - responsePhase.set_status_code(200) - responsePhase.set_message("OK") - responsePhase.set_data(method='GET', - context=None, - body="COMPLETED", - headers=None) + response_phase = DummyResponse() + response_phase.set_status_code(200) + response_phase.set_message("OK") + response_phase.set_data(method='GET', + context=None, + body="COMPLETED", + headers=None) req = "async/" + jobid + "/phase" - connHandler.set_response(req, responsePhase) + conn_handler.set_response(req, response_phase) # Results response - responseResultsJob = DummyResponse() - responseResultsJob.set_status_code(200) - responseResultsJob.set_message("OK") - jobDataFile = data_path('job_1.vot') - jobData = utils.read_file_content(jobDataFile) - responseResultsJob.set_data(method='GET', - context=None, - body=jobData, - headers=None) + response_results_job = DummyResponse() + response_results_job.set_status_code(200) + response_results_job.set_message("OK") + job_data_file = data_path('job_1.vot') + job_data = utils.read_file_content(job_data_file) + response_results_job.set_data(method='GET', + context=None, + body=job_data, + headers=None) req = "async/" + jobid + "/results/result" - connHandler.set_response(req, responseResultsJob) + conn_handler.set_response(req, response_results_job) sc = SkyCoord(ra=29.0, dec=15.0, unit=(u.degree, u.degree), frame='icrs') width = Quantity(12, u.deg) @@ -216,25 +219,25 @@ def test_query_object_async(self): np.int32) def test_cone_search_sync(self): - connHandler = DummyConnHandler() - tapplus = TapPlus("http://test:1111/tap", connhandler=connHandler) - tap = GaiaClass(connHandler, tapplus) + conn_handler = DummyConnHandler() + tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap = GaiaClass(conn_handler, tapplus) # Launch response: we use default response because the query contains # decimals - responseLaunchJob = DummyResponse() - responseLaunchJob.set_status_code(200) - responseLaunchJob.set_message("OK") - jobDataFile = data_path('job_1.vot') - jobData = utils.read_file_content(jobDataFile) - responseLaunchJob.set_data(method='POST', - context=None, - body=jobData, - headers=None) + response_launch_job = DummyResponse() + response_launch_job.set_status_code(200) + response_launch_job.set_message("OK") + job_data_file = data_path('job_1.vot') + job_data = utils.read_file_content(job_data_file) + response_launch_job.set_data(method='POST', + context=None, + body=job_data, + headers=None) ra = 19.0 dec = 20.0 sc = SkyCoord(ra=ra, dec=dec, unit=(u.degree, u.degree), frame='icrs') radius = Quantity(1.0, u.deg) - connHandler.set_default_response(responseLaunchJob) + conn_handler.set_default_response(response_launch_job) job = tap.cone_search(sc, radius) assert job is not None, "Expected a valid job" assert job.async_ is False, "Expected a synchronous job" @@ -269,49 +272,49 @@ def test_cone_search_sync(self): np.int32) def test_cone_search_async(self): - connHandler = DummyConnHandler() - tapplus = TapPlus("http://test:1111/tap", connhandler=connHandler) - tap = GaiaClass(connHandler, tapplus) + conn_handler = DummyConnHandler() + tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap = GaiaClass(conn_handler, tapplus) jobid = '12345' # Launch response - responseLaunchJob = DummyResponse() - responseLaunchJob.set_status_code(303) - responseLaunchJob.set_message("OK") + response_launch_job = DummyResponse() + response_launch_job.set_status_code(303) + response_launch_job.set_message("OK") # list of list (httplib implementation for headers in response) - launchResponseHeaders = [ + launch_response_headers = [ ['location', 'http://test:1111/tap/async/' + jobid] - ] - responseLaunchJob.set_data(method='POST', - context=None, - body=None, - headers=launchResponseHeaders) + ] + response_launch_job.set_data(method='POST', + context=None, + body=None, + headers=launch_response_headers) ra = 19 dec = 20 sc = SkyCoord(ra=ra, dec=dec, unit=(u.degree, u.degree), frame='icrs') radius = Quantity(1.0, u.deg) - connHandler.set_default_response(responseLaunchJob) + conn_handler.set_default_response(response_launch_job) # Phase response - responsePhase = DummyResponse() - responsePhase.set_status_code(200) - responsePhase.set_message("OK") - responsePhase.set_data(method='GET', - context=None, - body="COMPLETED", - headers=None) + response_phase = DummyResponse() + response_phase.set_status_code(200) + response_phase.set_message("OK") + response_phase.set_data(method='GET', + context=None, + body="COMPLETED", + headers=None) req = "async/" + jobid + "/phase" - connHandler.set_response(req, responsePhase) + conn_handler.set_response(req, response_phase) # Results response - responseResultsJob = DummyResponse() - responseResultsJob.set_status_code(200) - responseResultsJob.set_message("OK") - jobDataFile = data_path('job_1.vot') - jobData = utils.read_file_content(jobDataFile) - responseResultsJob.set_data(method='GET', - context=None, - body=jobData, - headers=None) + response_results_job = DummyResponse() + response_results_job.set_status_code(200) + response_results_job.set_message("OK") + job_data_file = data_path('job_1.vot') + job_data = utils.read_file_content(job_data_file) + response_results_job.set_data(method='GET', + context=None, + body=job_data, + headers=None) req = "async/" + jobid + "/results/result" - connHandler.set_response(req, responseResultsJob) + conn_handler.set_response(req, response_results_job) job = tap.cone_search_async(sc, radius) assert job is not None, "Expected a valid job" assert job.async_ is True, "Expected an asynchronous job" @@ -360,25 +363,25 @@ def test_cone_search_async(self): # Cleanup. conf.reset('MAIN_GAIA_TABLE') - def __check_results_column(self, results, columnName, description, unit, - dataType): - c = results[columnName] + def __check_results_column(self, results, column_name, description, unit, + data_type): + c = results[column_name] assert c.description == description, \ "Wrong description for results column '%s'. " % \ "Expected: '%s', found '%s'" % \ - (columnName, description, c.description) + (column_name, description, c.description) assert c.unit == unit, \ "Wrong unit for results column '%s'. " % \ "Expected: '%s', found '%s'" % \ - (columnName, unit, c.unit) - assert c.dtype == dataType, \ + (column_name, unit, c.unit) + assert c.dtype == data_type, \ "Wrong dataType for results column '%s'. " % \ "Expected: '%s', found '%s'" % \ - (columnName, dataType, c.dtype) + (column_name, data_type, c.dtype) def test_load_data(self): - dummyHandler = DummyTapHandler() - tap = GaiaClass(dummyHandler, dummyHandler) + dummy_handler = DummyTapHandler() + tap = GaiaClass(dummy_handler, dummy_handler) ids = "1,2,3,4" retrieval_type = "epoch_photometry" @@ -410,81 +413,79 @@ def test_load_data(self): parameters = {} parameters['params_dict'] = params_dict # Output file name contains a timestamp: cannot be verified - of = dummyHandler._DummyTapHandler__parameters['output_file'] + of = dummy_handler._DummyTapHandler__parameters['output_file'] parameters['output_file'] = of parameters['verbose'] = verbose - dummyHandler.check_call('load_data', parameters) + dummy_handler.check_call('load_data', parameters) def test_get_datalinks(self): - dummyHandler = DummyTapHandler() - tap = GaiaClass(dummyHandler, dummyHandler) + dummy_handler = DummyTapHandler() + tap = GaiaClass(dummy_handler, dummy_handler) ids = ["1", "2", "3", "4"] verbose = True parameters = {} parameters['ids'] = ids parameters['verbose'] = verbose tap.get_datalinks(ids, verbose) - dummyHandler.check_call('get_datalinks', parameters) - tap.get_datalinks(ids, verbose) - dummyHandler.check_call('get_datalinks', parameters) + dummy_handler.check_call('get_datalinks', parameters) def test_xmatch(self): - connHandler = DummyConnHandler() - tapplus = TapPlus("http://test:1111/tap", connhandler=connHandler) - tap = GaiaClass(connHandler, tapplus) + conn_handler = DummyConnHandler() + tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap = GaiaClass(conn_handler, tapplus) jobid = '12345' # Launch response - responseLaunchJob = DummyResponse() - responseLaunchJob.set_status_code(303) - responseLaunchJob.set_message("OK") + response_launch_job = DummyResponse() + response_launch_job.set_status_code(303) + response_launch_job.set_message("OK") # list of list (httplib implementation for headers in response) - launchResponseHeaders = [ + launch_response_headers = [ ['location', 'http://test:1111/tap/async/' + jobid] - ] - responseLaunchJob.set_data(method='POST', - context=None, - body=None, - headers=launchResponseHeaders) - connHandler.set_default_response(responseLaunchJob) + ] + response_launch_job.set_data(method='POST', + context=None, + body=None, + headers=launch_response_headers) + conn_handler.set_default_response(response_launch_job) # Phase response - responsePhase = DummyResponse() - responsePhase.set_status_code(200) - responsePhase.set_message("OK") - responsePhase.set_data(method='GET', - context=None, - body="COMPLETED", - headers=None) + response_phase = DummyResponse() + response_phase.set_status_code(200) + response_phase.set_message("OK") + response_phase.set_data(method='GET', + context=None, + body="COMPLETED", + headers=None) req = "async/" + jobid + "/phase" - connHandler.set_response(req, responsePhase) + conn_handler.set_response(req, response_phase) # Results response - responseResultsJob = DummyResponse() - responseResultsJob.set_status_code(200) - responseResultsJob.set_message("OK") - jobDataFile = data_path('job_1.vot') - jobData = utils.read_file_content(jobDataFile) - responseResultsJob.set_data(method='GET', - context=None, - body=jobData, - headers=None) + response_results_job = DummyResponse() + response_results_job.set_status_code(200) + response_results_job.set_message("OK") + job_data_file = data_path('job_1.vot') + job_data = utils.read_file_content(job_data_file) + response_results_job.set_data(method='GET', + context=None, + body=job_data, + headers=None) req = "async/" + jobid + "/results/result" - connHandler.set_response(req, responseResultsJob) + conn_handler.set_response(req, response_results_job) query = ("SELECT crossmatch_positional(", "'schemaA','tableA','schemaB','tableB',1.0,'results')", "FROM dual;") - dTmp = {"q": query} - dTmpEncoded = connHandler.url_encode(dTmp) - p = dTmpEncoded.find("=") - q = dTmpEncoded[p+1:] - dictTmp = { + d_tmp = {"q": query} + d_tmp_encoded = conn_handler.url_encode(d_tmp) + p = d_tmp_encoded.find("=") + q = d_tmp_encoded[p + 1:] + dict_tmp = { "REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": "votable", "tapclient": str(TAP_CLIENT_ID), "PHASE": "RUN", "QUERY": str(q)} - sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) - jobRequest = "sync?" + sortedKey - connHandler.set_response(jobRequest, responseLaunchJob) + sorted_key = taputils.taputil_create_sorted_dict_key(dict_tmp) + job_request = "sync?" + sorted_key + conn_handler.set_response(job_request, response_launch_job) # check parameters # missing table A with pytest.raises(ValueError) as err: @@ -498,7 +499,7 @@ def test_xmatch(self): full_qualified_table_name_b='schemaB.tableB', results_table_name='results') assert "Not found schema name in full qualified table A: 'tableA'" \ - in err.value.args[0] + in err.value.args[0] # missing table B with pytest.raises(ValueError) as err: tap.cross_match(full_qualified_table_name_a='schemaA.tableA', @@ -511,7 +512,7 @@ def test_xmatch(self): full_qualified_table_name_b='tableB', results_table_name='results') assert "Not found schema name in full qualified table B: 'tableB'" \ - in err.value.args[0] + in err.value.args[0] # missing results table with pytest.raises(ValueError) as err: tap.cross_match(full_qualified_table_name_a='schemaA.tableA', @@ -524,21 +525,21 @@ def test_xmatch(self): full_qualified_table_name_b='schemaB.tableB', results_table_name='schema.results') assert "Please, do not specify schema for 'results_table_name'" \ - in err.value.args[0] + in err.value.args[0] # radius < 0.1 with pytest.raises(ValueError) as err: tap.cross_match(full_qualified_table_name_a='schemaA.tableA', full_qualified_table_name_b='schemaB.tableB', results_table_name='results', radius=0.01) assert "Invalid radius value. Found 0.01, valid range is: 0.1 to 10.0" \ - in err.value.args[0] + in err.value.args[0] # radius > 10.0 with pytest.raises(ValueError) as err: tap.cross_match(full_qualified_table_name_a='schemaA.tableA', full_qualified_table_name_b='schemaB.tableB', results_table_name='results', radius=10.1) assert "Invalid radius value. Found 10.1, valid range is: 0.1 to 10.0" \ - in err.value.args[0] + in err.value.args[0] # check default parameters parameters = {} query = "SELECT crossmatch_positional(\ @@ -565,16 +566,50 @@ def test_xmatch(self): ('COMPLETED', job.get_phase()) assert job.failed is False, "Wrong job status (set Failed = True)" job = tap.cross_match( - full_qualified_table_name_a='schemaA.tableA', - full_qualified_table_name_b='schemaB.tableB', - results_table_name='results', - background=True) + full_qualified_table_name_a='schemaA.tableA', + full_qualified_table_name_b='schemaB.tableB', + results_table_name='results', + background=True) assert job.async_ is True, "Expected an asynchronous job" assert job.get_phase() == 'EXECUTING', \ "Wrong job phase. Expected: %s, found %s" % \ ('EXECUTING', job.get_phase()) assert job.failed is False, "Wrong job status (set Failed = True)" + @patch.object(TapPlus, 'login') + def test_login(self, mock_login): + conn_handler = DummyConnHandler() + tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap = GaiaClass(conn_handler, tapplus) + tap.login("user", "password") + assert (mock_login.call_count == 2) + mock_login.side_effect = HTTPError("Login error") + tap.login("user", "password") + assert (mock_login.call_count == 3) + + @patch.object(TapPlus, 'login_gui') + @patch.object(TapPlus, 'login') + def test_login_gui(self, mock_login_gui, mock_login): + conn_handler = DummyConnHandler() + tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap = GaiaClass(conn_handler, tapplus) + tap.login_gui() + assert (mock_login_gui.call_count == 1) + mock_login_gui.side_effect = HTTPError("Login error") + tap.login("user", "password") + assert (mock_login.call_count == 1) + + @patch.object(TapPlus, 'logout') + def test_logout(self, mock_logout): + conn_handler = DummyConnHandler() + tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap = GaiaClass(conn_handler, tapplus) + tap.logout() + assert (mock_logout.call_count == 2) + mock_logout.side_effect = HTTPError("Login error") + tap.logout() + assert (mock_logout.call_count == 3) + if __name__ == "__main__": # import sys;sys.argv = ['', 'Test.testName'] diff --git a/astroquery/heasarc/core.py b/astroquery/heasarc/core.py index 703aafc867..e5cabd9d33 100644 --- a/astroquery/heasarc/core.py +++ b/astroquery/heasarc/core.py @@ -17,7 +17,6 @@ def Table_read(*args, **kwargs): - # why does if commons.ASTROPY_LT_5_0 not work on Windows? if commons.ASTROPY_LT_5_1: return Table.read(*args, **kwargs) else: diff --git a/astroquery/heasarc/tests/parametrization.py b/astroquery/heasarc/tests/parametrization.py new file mode 100644 index 0000000000..e40de7655c --- /dev/null +++ b/astroquery/heasarc/tests/parametrization.py @@ -0,0 +1,123 @@ +import os +import glob +import hashlib +import requests +import pytest +from ... import log + +""" +This is an attempt to reduce code duplication between remote and local tests. + +if there is test data: + runs as usual, except all tests have two versions with the same code: remote and local +else + runs remote test patched so that the test data is stored in a temporary directory. + advice is given to copy the newly generated test data into the repository +""" + + +class MockResponse: + def __init__(self, text): + self.text = text + self.content = text.encode() + + +def data_path(filename, output=False): + if output: + data_dir = os.path.join( + os.getenv("TMPDIR", "/tmp"), "astroquery-heasarc-saved-data" + ) + os.makedirs(data_dir, exist_ok=True) + else: + data_dir = os.path.join(os.path.dirname(__file__), "data") + + return os.path.join(data_dir, filename + ".dat") + + +def fileid_for_request(url, params): + return hashlib.md5(str((url, sorted(params.items()))).encode()).hexdigest()[:8] + + +def filename_for_request(url, params, output=False): + fileid = fileid_for_request(url, params) + return data_path(fileid, output=output) + + +def get_mockreturn(session, method, url, params=None, timeout=10, **kwargs): + + filename = filename_for_request(url, params) + try: + content = open(filename, "rt").read() + except FileNotFoundError: + log.error( + f'no stored mock data in {filename} for url="{url}" and params="{params}", ' + f'perhaps you need to clean test data and regenerate it? ' + f'It will be regenerated automatically if cleaned, try `rm -fv astroquery/heasarc/tests/data/* ./build`' + ) + raise + + return MockResponse(content) + + +def save_response_of_get(session, method, url, params=None, timeout=10, **kwargs): + # _original_request is a monkeypatch-added attribute in patch_get + text = requests.Session._original_request( + session, method, url, params=params, timeout=timeout + ).text + + filename = filename_for_request(url, params, output=True) + + with open(filename, "wt") as f: + log.info(f'saving output to {filename} for url="{url}" and params="{params}"') + log.warning( + f"you may want to run `cp -fv {os.path.dirname(filename)}/* astroquery/heasarc/tests/data/; rm -rfv build`" + "`git add astroquery/heasarc/tests/data/*`." + ) + f.write(text) + + return MockResponse(text) + + +@pytest.fixture(autouse=True) +def patch_get(request): + """ + If the mode is not remote, patch `requests.Session` to either return saved local data or run save data new local data + """ + mode = request.param + mp = request.getfixturevalue("monkeypatch") + + if mode != "remote": + requests.Session._original_request = requests.Session.request + mp.setattr( + requests.Session, + "request", + {"save": save_response_of_get, "local": get_mockreturn}[mode], + ) + + mp.assume_fileid_for_request = lambda patched_fileid_for_request: \ + mp.setattr('astroquery.heasarc.tests.parametrization.fileid_for_request', patched_fileid_for_request) + + mp.reset_default_fileid_for_request = lambda: \ + mp.delattr('astroquery.heasarc.tests.parametrization.fileid_for_request') + + return mp + + +def have_mock_data(): + return len(glob.glob(data_path("*"))) > 0 + + +parametrization_local_save_remote = pytest.mark.parametrize( + "patch_get", [ + pytest.param("local", marks=[ + pytest.mark.skipif(not have_mock_data(), + reason="No test data found. If remote_data is allowed, we'll generate some.")]), + pytest.param("save", marks=[ + pytest.mark.remote_data, + pytest.mark.skipif(have_mock_data(), + reason="found some test data: please delete them to save again.")]), + pytest.param("remote", marks=[ + pytest.mark.remote_data, + pytest.mark.skipif(not have_mock_data(), + reason="No test data found, [save] will run remote tests and save data.")])], + indirect=True) diff --git a/astroquery/image_cutouts/first/tests/test_first.py b/astroquery/image_cutouts/first/tests/test_first.py index 2fcf1ef4f7..4dff16d3ba 100644 --- a/astroquery/image_cutouts/first/tests/test_first.py +++ b/astroquery/image_cutouts/first/tests/test_first.py @@ -6,7 +6,7 @@ import astropy.units as u from ....utils import commons -from ....utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ... import first DATA_FILES = {'image': 'image.fits'} diff --git a/astroquery/imcce/tests/test_miriade.py b/astroquery/imcce/tests/test_miriade.py index d9bd3a4179..ac9049502b 100644 --- a/astroquery/imcce/tests/test_miriade.py +++ b/astroquery/imcce/tests/test_miriade.py @@ -4,7 +4,7 @@ import os import astropy.units as u -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from .. import Miriade, MiriadeClass diff --git a/astroquery/imcce/tests/test_skybot.py b/astroquery/imcce/tests/test_skybot.py index d8c0cdddb0..9ed6c3b2d4 100644 --- a/astroquery/imcce/tests/test_skybot.py +++ b/astroquery/imcce/tests/test_skybot.py @@ -3,7 +3,7 @@ import pytest import os -from astroquery.utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse import astropy.units as u from astropy.time import Time from astropy.coordinates import SkyCoord, Angle diff --git a/astroquery/ipac/irsa/ibe/tests/test_ibe.py b/astroquery/ipac/irsa/ibe/tests/test_ibe.py index c9fce5b559..5ba03e39b6 100644 --- a/astroquery/ipac/irsa/ibe/tests/test_ibe.py +++ b/astroquery/ipac/irsa/ibe/tests/test_ibe.py @@ -8,7 +8,7 @@ from astropy.coordinates import SkyCoord import astropy.units as u -from astroquery.utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from astroquery.ipac.irsa.ibe import Ibe DATA_FILES = { diff --git a/astroquery/ipac/irsa/irsa_dust/core.py b/astroquery/ipac/irsa/irsa_dust/core.py index 74a6d9125e..719424ee85 100644 --- a/astroquery/ipac/irsa/irsa_dust/core.py +++ b/astroquery/ipac/irsa/irsa_dust/core.py @@ -123,7 +123,7 @@ def get_images_async(self, coordinate, radius=None, image_type=None, to remote server. Defaults to `False`. Returns - -------- + ------- list : list A list of context-managers that yield readable file-like objects. """ @@ -146,7 +146,7 @@ def get_image_list(self, coordinate, radius=None, image_type=None, of URLs to the Irsa-Dust images. Parameters - ----------- + ---------- coordinate : str Can be either the name of an object or a coordinate string If a name, must be resolvable by NED, SIMBAD, 2MASS, or SWAS. @@ -169,7 +169,7 @@ def get_image_list(self, coordinate, radius=None, image_type=None, to remote server. Defaults to `False`. Returns - -------- + ------- url_list : list A list of URLs to the FITS images corresponding to the queried object. @@ -203,7 +203,7 @@ def get_extinction_table(self, coordinate, radius=None, timeout=TIMEOUT, server. Defaults to `~astroquery.ipac.irsa.irsa_dust.IrsaDustClass.TIMEOUT`. Returns - -------- + ------- table : `~astropy.table.Table` """ readable_obj = self.get_extinction_table_async( @@ -289,7 +289,7 @@ def get_query_table(self, coordinate, radius=None, Defaults to `~astroquery.ipac.irsa.irsa_dust.IrsaDustClass.DUST_SERVICE_URL`. Returns - -------- + ------- table : `~astropy.table.Table` Table representing the query results, (all or as per specified). """ diff --git a/astroquery/ipac/irsa/sha/tests/test_sha.py b/astroquery/ipac/irsa/sha/tests/test_sha.py index 5faeabe12c..f47bef0737 100644 --- a/astroquery/ipac/irsa/sha/tests/test_sha.py +++ b/astroquery/ipac/irsa/sha/tests/test_sha.py @@ -4,7 +4,7 @@ import requests from astroquery.ipac.irsa import sha -from astroquery.utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse DATA_FILES = {'img': 'img.fits', 'nid_t': 'nid_t.txt', diff --git a/astroquery/ipac/irsa/tests/test_irsa.py b/astroquery/ipac/irsa/tests/test_irsa.py index abc7064168..0633fdce9a 100644 --- a/astroquery/ipac/irsa/tests/test_irsa.py +++ b/astroquery/ipac/irsa/tests/test_irsa.py @@ -9,7 +9,7 @@ import astropy.coordinates as coord import astropy.units as u -from astroquery.utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from astroquery.utils import commons from astroquery.ipac.irsa import Irsa, conf from astroquery.ipac import irsa diff --git a/astroquery/ipac/ned/core.py b/astroquery/ipac/ned/core.py index 82e8537cdb..b943788540 100644 --- a/astroquery/ipac/ned/core.py +++ b/astroquery/ipac/ned/core.py @@ -385,7 +385,7 @@ def get_images_async(self, object_name, get_query_payload=False, request. Defaults to `False` Returns - -------- + ------- A list of context-managers that yield readable file-like objects """ @@ -438,7 +438,7 @@ def get_spectra_async(self, object_name, get_query_payload=False, request. Defaults to `False` Returns - -------- + ------- A list of context-managers that yield readable file-like objects """ diff --git a/astroquery/ipac/ned/tests/test_ned.py b/astroquery/ipac/ned/tests/test_ned.py index 7328e52a83..0b21597687 100644 --- a/astroquery/ipac/ned/tests/test_ned.py +++ b/astroquery/ipac/ned/tests/test_ned.py @@ -8,7 +8,7 @@ import astropy.coordinates as coord import astropy.units as u from astroquery.exceptions import RemoteServiceError -from astroquery.utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from astroquery.ipac import ned from astroquery.utils import commons diff --git a/astroquery/ipac/nexsci/nasa_exoplanet_archive/__init__.py b/astroquery/ipac/nexsci/nasa_exoplanet_archive/__init__.py index 9a8ad1779f..6dfc0326f1 100644 --- a/astroquery/ipac/nexsci/nasa_exoplanet_archive/__init__.py +++ b/astroquery/ipac/nexsci/nasa_exoplanet_archive/__init__.py @@ -21,6 +21,9 @@ class Conf(_config.ConfigNamespace): url_tap = _config.ConfigItem( "https://exoplanetarchive.ipac.caltech.edu/TAP/", "URL for the NASA Exoplanet Archive TAP") + url_aliaslookup = _config.ConfigItem( + "https://exoplanetarchive.ipac.caltech.edu/cgi-bin/Lookup/nph-aliaslookup.py?objname=", + "URL for the NASA Exoplanet Archive aliaslookup") timeout = _config.ConfigItem( 600, "Time limit for requests from the NASA Exoplanet Archive servers") cache = _config.ConfigItem(False, "Should the requests be cached?") diff --git a/astroquery/ipac/nexsci/nasa_exoplanet_archive/core.py b/astroquery/ipac/nexsci/nasa_exoplanet_archive/core.py index feb2307433..9f64b8e7b2 100644 --- a/astroquery/ipac/nexsci/nasa_exoplanet_archive/core.py +++ b/astroquery/ipac/nexsci/nasa_exoplanet_archive/core.py @@ -5,6 +5,8 @@ import io import re import warnings +import requests +import json # Import various astropy modules import astropy.coordinates as coord @@ -105,6 +107,8 @@ def get_access_url(service='tap'): url = conf.url_tap elif service == 'api': url = conf.url_api + elif service == 'aliaslookup': + url = conf.url_aliaslookup return url @@ -138,7 +142,7 @@ class NasaExoplanetArchiveClass(BaseQuery): """ # When module us imported, __init__.py runs and loads a configuration object, - # setting the configuration parameters con.url, conf.timeout and conf.cache + # setting the configuration parameters conf.url, conf.timeout and conf.cache URL_API = conf.url_api URL_TAP = conf.url_tap TIMEOUT = conf.timeout @@ -377,7 +381,7 @@ def query_aliases(self, object_name, *, cache=None): Parameters ---------- object_name : str - The name of a planet or star to regularize using the ``aliastable`` table. + The name of a planet or star to regularize using the ``aliaslookup`` service. cache : bool, optional Should the request result be cached? This can be useful for large repeated queries, but since the data in the archive is updated regularly, this defaults to ``False``. @@ -387,24 +391,47 @@ def query_aliases(self, object_name, *, cache=None): response : list A list of aliases found for the object name. The default name will be listed first. """ - return list( - self.query_criteria( - "aliastable", objname=object_name.strip(), cache=cache, format="csv" - )["aliasdis"] - ) + data = self._request_query_aliases(object_name) + + try: + objname_split = object_name.split() + if len(objname_split) > 1 and len(objname_split[-1]) == 1 and objname_split[-1].isalpha(): + pl_letter = object_name.split()[-1] + else: + pl_letter = '' + + default_objname = [data['system']['system_info']['alias_set']['default_name']] + other_objnames = list(set(data['system']['objects']['stellar_set']['stars'][default_objname[0]]['alias_set']['aliases']) - set(default_objname)) + other_objnames.sort() + aliases = default_objname + other_objnames + + if pl_letter: + aliases = [a + ' ' + pl_letter for a in aliases] + + except KeyError: + aliases = [] + warnings.warn("No aliases found for name: '{0}'".format(object_name), NoResultsWarning) + + return aliases @class_or_instance def _regularize_object_name(self, object_name): - """Regularize the name of a planet or planet host using the ``aliastable`` table""" + """Regularize the name of a planet or planet host using the ``aliaslookup`` service""" try: aliases = self.query_aliases(object_name, cache=False) - except RemoteServiceError: + except KeyError: aliases = [] if aliases: return aliases[0] warnings.warn("No aliases found for name: '{0}'".format(object_name), NoResultsWarning) return object_name + def _request_query_aliases(self, object_name): + """Service request for query_aliases()""" + url = requests.get(get_access_url('aliaslookup')+object_name) + response = json.loads(url.text) + return response + # Look for response errors. This might need to be updated for TAP def _handle_error(self, text): """ @@ -653,7 +680,7 @@ def _request_to_sql(self, request_payload): @deprecated(since="v0.4.1", alternative="query_object") @deprecated_renamed_argument(["show_progress", "table_path"], [None, None], "v0.4.1", arg_in_kwargs=True) - def query_planet(self, planet_name, cache=None, regularize=True, **criteria): + def query_planet(self, planet_name, cache=None, **criteria): """ Search the ``exoplanets`` table for a confirmed planet @@ -665,14 +692,10 @@ def query_planet(self, planet_name, cache=None, regularize=True, **criteria): cache : bool, optional Should the request result be cached? This can be useful for large repeated queries, but since the data in the archive is updated regularly, this defaults to ``False``. - regularize : bool, optional - If ``True``, the ``aliastable`` will be used to regularize the target name. **criteria Any other filtering criteria to apply. Values provided using the ``where`` keyword will be ignored. """ - if regularize: - planet_name = self._regularize_object_name(planet_name) criteria = self._handle_all_columns_argument(**criteria) criteria["where"] = "pl_name='{0}'".format(planet_name.strip()) return self.query_criteria("exoplanets", cache=cache, **criteria) @@ -680,7 +703,7 @@ def query_planet(self, planet_name, cache=None, regularize=True, **criteria): @deprecated(since="v0.4.1", alternative="query_object") @deprecated_renamed_argument(["show_progress", "table_path"], [None, None], "v0.4.1", arg_in_kwargs=True) - def query_star(self, host_name, cache=None, regularize=True, **criteria): + def query_star(self, host_name, cache=None, **criteria): """ Search the ``exoplanets`` table for a confirmed planet host @@ -692,14 +715,10 @@ def query_star(self, host_name, cache=None, regularize=True, **criteria): cache : bool, optional Should the request result be cached? This can be useful for large repeated queries, but since the data in the archive is updated regularly, this defaults to ``False``. - regularize : bool, optional - If ``True``, the ``aliastable`` will be used to regularize the target name. **criteria Any other filtering criteria to apply. Values provided using the ``where`` keyword will be ignored. """ - if regularize: - host_name = self._regularize_object_name(host_name) criteria = self._handle_all_columns_argument(**criteria) criteria["where"] = "pl_hostname='{0}'".format(host_name.strip()) return self.query_criteria("exoplanets", cache=cache, **criteria) diff --git a/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/data/bpic_aliaslookup.json b/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/data/bpic_aliaslookup.json new file mode 100644 index 0000000000..de453d39ad --- /dev/null +++ b/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/data/bpic_aliaslookup.json @@ -0,0 +1,99 @@ +{ + "manifest": { + "requested_name": "bet pic", + "resolved_name": "bet Pic", + "lookup_status": "OK", + "compilation_date": "2022-01-28 14:31:40.254838", + "system_snapshot": { + "number_of_stars": 1, + "number_of_planets": 2 + } + }, + "system": { + "system_info": { + "id": "bet Pic", + "alias_set": { + "item_count": 1, + "default_name": "bet Pic", + "aliases": [ + "bet Pic" + ] + } + }, + "objects": { + "stellar_set": { + "item_count": 1, + "stars": { + "bet Pic": { + "alias_set": { + "item_count": 14, + "aliases": [ + "TIC 270577175", + "Gaia DR2 4792774797545105664", + "GJ 219", + "HR 2020", + "bet Pic", + "HD 39060", + "HIP 27321", + "SAO 234134", + "CD-51 1620", + "CPD-51 774", + "IRAS 05460-5104", + "TYC 8099-01392-1", + "2MASS J05471708-5103594", + "WISE J054717.10-510358.4" + ] + } + } + } + }, + "planet_set": { + "item_count": 2, + "planets": { + "bet Pic b": { + "alias_set": { + "item_count": 14, + "aliases": [ + "GJ 219 b", + "HR 2020 b", + "bet Pic b", + "HD 39060 b", + "HIP 27321 b", + "SAO 234134 b", + "CD-51 1620 b", + "CPD-51 774 b", + "IRAS 05460-5104 b", + "TYC 8099-01392-1 b", + "2MASS J05471708-5103594 b", + "WISE J054717.10-510358.4 b", + "TIC 270577175 b", + "Gaia DR2 4792774797545105664 b" + ] + } + }, + "bet Pic c": { + "alias_set": { + "item_count": 14, + "aliases": [ + "GJ 219 c", + "HR 2020 c", + "bet Pic c", + "HD 39060 c", + "HIP 27321 c", + "SAO 234134 c", + "CD-51 1620 c", + "CPD-51 774 c", + "IRAS 05460-5104 c", + "TYC 8099-1392-1 c", + "2MASS J05471708-5103594 c", + "WISE J054717.10-510358.4 c", + "TIC 270577175 c", + "Gaia DR2 4792774797545105664 c" + ] + } + } + } + } + } + } +} \ No newline at end of file diff --git a/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/test_nasa_exoplanet_archive.py b/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/test_nasa_exoplanet_archive.py index 119388c3d6..d240a0d5df 100644 --- a/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/test_nasa_exoplanet_archive.py +++ b/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/test_nasa_exoplanet_archive.py @@ -13,8 +13,8 @@ from astropy.utils.exceptions import AstropyDeprecationWarning from astroquery.exceptions import NoResultsWarning -from astroquery.utils.testing_tools import MockResponse -from astroquery.ipac.nexsci.nasa_exoplanet_archive.core import NasaExoplanetArchiveClass, conf, InvalidTableError +from astroquery.utils.mocks import MockResponse +from astroquery.ipac.nexsci.nasa_exoplanet_archive.core import NasaExoplanetArchiveClass, conf, InvalidTableError, get_access_url try: from unittest.mock import Mock, patch, PropertyMock except ImportError: @@ -119,16 +119,53 @@ def patch_get(request): # pragma: nocover return mp -def test_regularize_object_name(patch_get): - NasaExoplanetArchiveMock = NasaExoplanetArchiveClass() +# aliaslookup file in data/ +LOOKUP_DATA_FILE = 'bpic_aliaslookup.json' + + +def data_path(filename): + data_dir = os.path.join(os.path.dirname(__file__), 'data') + return os.path.join(data_dir, filename) + + +# monkeypatch replacement request function +def query_aliases_mock(self, *args, **kwargs): + with open(data_path(LOOKUP_DATA_FILE), 'rb') as f: + response = json.load(f) + return response + + +# use a pytest fixture to create a dummy 'requests.get' function, +# that mocks(monkeypatches) the actual 'requests.get' function: +@pytest.fixture +def query_aliases_request(request): + mp = request.getfixturevalue("monkeypatch") + mp.setattr(NasaExoplanetArchiveClass, '_request_query_aliases', query_aliases_mock) + return mp + + +def test_query_aliases(query_aliases_request): + nasa_exoplanet_archive = NasaExoplanetArchiveClass() + result = nasa_exoplanet_archive.query_aliases(object_name='bet Pic') + assert len(result) > 10 + assert 'GJ 219' in result + assert 'bet Pic' in result + assert '2MASS J05471708-5103594' in result + + +def test_query_aliases_planet(query_aliases_request): + nasa_exoplanet_archive = NasaExoplanetArchiveClass() + result = nasa_exoplanet_archive.query_aliases('bet Pic b') + assert len(result) > 10 + assert 'GJ 219 b' in result + assert 'bet Pic b' in result + assert '2MASS J05471708-5103594 b' in result - NasaExoplanetArchiveMock._tap_tables = ['list'] - assert NasaExoplanetArchiveMock._regularize_object_name("kepler 2") == "HAT-P-7" - assert NasaExoplanetArchiveMock._regularize_object_name("kepler 1 b") == "TrES-2 b" - with pytest.warns(NoResultsWarning) as warning: - NasaExoplanetArchiveMock._regularize_object_name("not a planet") - assert "No aliases found for name: 'not a planet'" == str(warning[0].message) +def test_get_access_url(): + assert get_access_url('tap') == conf.url_tap + assert get_access_url('api') == conf.url_api + assert get_access_url('aliaslookup') == conf.url_aliaslookup def test_backwards_compat(patch_get): diff --git a/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/test_nasa_exoplanet_archive_remote.py b/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/test_nasa_exoplanet_archive_remote.py index 7988c02d7e..c7cd1ddbe0 100644 --- a/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/test_nasa_exoplanet_archive_remote.py +++ b/astroquery/ipac/nexsci/nasa_exoplanet_archive/tests/test_nasa_exoplanet_archive_remote.py @@ -159,7 +159,7 @@ def test_query_region(): def test_query_aliases(): name = "bet Pic" aliases = NasaExoplanetArchive.query_aliases(name) - assert len(aliases) == 12 + assert len(aliases) > 10 assert "HD 39060" in aliases diff --git a/astroquery/jplhorizons/core.py b/astroquery/jplhorizons/core.py index fbef822aa4..272e7f97fa 100644 --- a/astroquery/jplhorizons/core.py +++ b/astroquery/jplhorizons/core.py @@ -1287,11 +1287,20 @@ def _parse_result(self, response, verbose=None): data : `astropy.Table` """ + self.last_response = response if self.query_type not in ['ephemerides', 'elements', 'vectors']: return None else: - data = self._parse_horizons(response.text) - + try: + data = self._parse_horizons(response.text) + except Exception as ex: + try: + self._last_query.remove_cache_file(self.cache_location) + except OSError: + # this is allowed: if `cache` was set to False, this + # won't be needed + pass + raise return data diff --git a/astroquery/jplhorizons/tests/data/README b/astroquery/jplhorizons/tests/data/README index 75b8a933c2..12cf5764f9 100644 --- a/astroquery/jplhorizons/tests/data/README +++ b/astroquery/jplhorizons/tests/data/README @@ -10,4 +10,7 @@ ceres_vectors.txt https://ssd.jpl.nasa.gov/api/horizons.api?format=text&EPHEM_TYPE=VECTORS&OUT_UNITS=AU-D&COMMAND=%22Ceres%3B%22&CENTER=%27500%4010%27&CSV_FORMAT=%22YES%22&REF_PLANE=ECLIPTIC&REF_SYSTEM=ICRF&TP_TYPE=ABSOLUTE&VEC_LABELS=YES&VEC_CORR=%22NONE%22&VEC_DELTA_T=NO&OBJ_DATA=YES&TLIST=2451544.5 no_H.txt -https://ssd.jpl.nasa.gov/api/horizons.api?format=text&EPHEM_TYPE=OBSERVER&QUANTITIES=%271%2C2%2C3%2C4%2C5%2C6%2C7%2C8%2C9%2C10%2C11%2C12%2C13%2C14%2C15%2C16%2C17%2C18%2C19%2C20%2C21%2C22%2C23%2C24%2C25%2C26%2C27%2C28%2C29%2C30%2C31%2C32%2C33%2C34%2C35%2C36%2C37%2C38%2C39%2C40%2C41%2C42%2C43%27&COMMAND=%221935+UZ%3B%22&SOLAR_ELONG=%220%2C180%22&LHA_CUTOFF=0&CSV_FORMAT=YES&CAL_FORMAT=BOTH&ANG_FORMAT=DEG&APPARENT=AIRLESS&REF_SYSTEM=ICRF&EXTRA_PREC=NO&CENTER=%27500%40399%27&TLIST=2459480.5004416634&SKIP_DAYLT=NO \ No newline at end of file +https://ssd.jpl.nasa.gov/api/horizons.api?format=text&EPHEM_TYPE=OBSERVER&QUANTITIES=%271%2C2%2C3%2C4%2C5%2C6%2C7%2C8%2C9%2C10%2C11%2C12%2C13%2C14%2C15%2C16%2C17%2C18%2C19%2C20%2C21%2C22%2C23%2C24%2C25%2C26%2C27%2C28%2C29%2C30%2C31%2C32%2C33%2C34%2C35%2C36%2C37%2C38%2C39%2C40%2C41%2C42%2C43%27&COMMAND=%221935+UZ%3B%22&SOLAR_ELONG=%220%2C180%22&LHA_CUTOFF=0&CSV_FORMAT=YES&CAL_FORMAT=BOTH&ANG_FORMAT=DEG&APPARENT=AIRLESS&REF_SYSTEM=ICRF&EXTRA_PREC=NO&CENTER=%27500%40399%27&TLIST=2459480.5004416634&SKIP_DAYLT=NO + +tlist_error.txt +https://ssd.jpl.nasa.gov/api/horizons.api?format=text&COMMAND=%27499%27&OBJ_DATA=%27YES%27&MAKE_EPHEM=%27YES%27&EPHEM_TYPE=%27OBSERVER%27&CENTER=%27500@399%27&QUANTITIES=%271,9,20,23,24,29%27&TLIST=[] diff --git a/astroquery/jplhorizons/tests/data/tlist_error.txt b/astroquery/jplhorizons/tests/data/tlist_error.txt new file mode 100644 index 0000000000..cdbc9b32b4 --- /dev/null +++ b/astroquery/jplhorizons/tests/data/tlist_error.txt @@ -0,0 +1,5 @@ +API VERSION: 1.1 +API SOURCE: NASA/JPL Horizons API + +BATVAR: no TLIST values found (or missing quotes) +WLDINI: error loading execution-control file. diff --git a/astroquery/jplhorizons/tests/test_jplhorizons.py b/astroquery/jplhorizons/tests/test_jplhorizons.py index b8b03f29c3..af1296e7fd 100644 --- a/astroquery/jplhorizons/tests/test_jplhorizons.py +++ b/astroquery/jplhorizons/tests/test_jplhorizons.py @@ -1,5 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +from multiprocessing import Value import pytest import os from collections import OrderedDict @@ -8,14 +9,17 @@ from astropy.tests.helper import assert_quantity_allclose from astropy.utils.exceptions import AstropyDeprecationWarning -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse +from ...query import AstroQuery +from ...exceptions import TableParseError from ... import jplhorizons # files in data/ for different query types DATA_FILES = {'ephemerides': 'ceres_ephemerides.txt', 'elements': 'ceres_elements.txt', 'vectors': 'ceres_vectors.txt', - '"1935 UZ"': 'no_H.txt'} + '"1935 UZ"': 'no_H.txt', + '"tlist_error"': 'tlist_error.txt'} def data_path(filename): @@ -55,6 +59,13 @@ def patch_request(request): # --------------------------------- actual test functions +def test_parse_result(patch_request): + q = jplhorizons.Horizons(id='tlist_error') + # need _last_query to be defined + q._last_query = AstroQuery('GET', 'http://dummy') + with pytest.raises(ValueError): + res = q.ephemerides() + def test_ephemerides_query(patch_request): # check values of Ceres for a given epoch diff --git a/astroquery/jplsbdb/tests/test_jplsbdb.py b/astroquery/jplsbdb/tests/test_jplsbdb.py index 0a10f2fcc8..fb44aa4c55 100644 --- a/astroquery/jplsbdb/tests/test_jplsbdb.py +++ b/astroquery/jplsbdb/tests/test_jplsbdb.py @@ -3,7 +3,7 @@ import pytest import os -from astroquery.utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse import astropy.units as u from astropy.tests.helper import assert_quantity_allclose from .. import SBDB, SBDBClass diff --git a/astroquery/jplspec/core.py b/astroquery/jplspec/core.py index bf79ecc9c5..45bb9b66ef 100644 --- a/astroquery/jplspec/core.py +++ b/astroquery/jplspec/core.py @@ -191,12 +191,12 @@ def get_species_table(self, catfile='catdir.cat'): | (I6,X, A13, I6, 7F7.4, I2) Parameters - ----------- + ---------- catfile : str, name of file, default 'catdir.cat' The catalog file, installed locally along with the package Returns - -------- + ------- Table: `~astropy.table.Table` | TAG : The species tag or molecular identifier. | NAME : An ASCII name for the species. diff --git a/astroquery/linelists/cdms/core.py b/astroquery/linelists/cdms/core.py index e330cbdb92..8988f0f446 100644 --- a/astroquery/linelists/cdms/core.py +++ b/astroquery/linelists/cdms/core.py @@ -254,12 +254,12 @@ def get_species_table(self, catfile='catdir.cat'): The table is derived from https://cdms.astro.uni-koeln.de/classic/entries/partition_function.html Parameters - ----------- + ---------- catfile : str, name of file, default 'catdir.cat' The catalog file, installed locally along with the package Returns - -------- + ------- Table: `~astropy.table.Table` | tag : The species tag or molecular identifier. | molecule : An ASCII name for the species. diff --git a/astroquery/magpis/tests/test_magpis.py b/astroquery/magpis/tests/test_magpis.py index abebc179e9..c42dfaa2ce 100644 --- a/astroquery/magpis/tests/test_magpis.py +++ b/astroquery/magpis/tests/test_magpis.py @@ -7,7 +7,7 @@ import astropy.units as u from ...utils import commons -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ... import magpis DATA_FILES = {'image': 'image.fits'} diff --git a/astroquery/mast/__init__.py b/astroquery/mast/__init__.py index c08d016160..0a43e4dbc2 100644 --- a/astroquery/mast/__init__.py +++ b/astroquery/mast/__init__.py @@ -33,11 +33,13 @@ class Conf(_config.ConfigNamespace): from .cutouts import TesscutClass, Tesscut, ZcutClass, Zcut from .observations import Observations, ObservationsClass, MastClass, Mast from .collections import Catalogs, CatalogsClass +from .missions import MastMissions, MastMissionsClass from .core import MastQueryWithLogin from . import utils __all__ = ['Observations', 'ObservationsClass', 'Catalogs', 'CatalogsClass', + 'MastMissions', 'MastMissionsClass', 'Mast', 'MastClass', 'Tesscut', 'TesscutClass', 'Zcut', 'ZcutClass', diff --git a/astroquery/mast/cloud.py b/astroquery/mast/cloud.py index 2992f68208..1707fbaddb 100644 --- a/astroquery/mast/cloud.py +++ b/astroquery/mast/cloud.py @@ -53,7 +53,7 @@ def __init__(self, provider="AWS", profile=None, verbose=False): import boto3 import botocore - self.supported_missions = ["mast:hst/product", "mast:tess/product", "mast:kepler"] + self.supported_missions = ["mast:hst/product", "mast:tess/product", "mast:kepler", "mast:galex"] self.boto3 = boto3 self.botocore = botocore @@ -116,7 +116,10 @@ def get_cloud_uri(self, data_product, include_bucket=True, full_url=False): if path is None: raise InvalidQueryError("Malformed data uri {}".format(data_product['dataURI'])) - path = path.lstrip("/") + if 'galex' in path: + path = path.lstrip("/mast/") + else: + path = path.lstrip("/") try: s3_client.head_object(Bucket=self.pubdata_bucket, Key=path) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 39d66dbeff..6675609adc 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -42,14 +42,15 @@ def __init__(self): services = {"panstarrs": {"path": "panstarrs/{data_release}/{table}.json", "args": {"data_release": "dr2", "table": "mean"}}} + self._service_api_connection.set_service_params(services, "catalogs", True) self.catalog_limit = None self._current_connection = None - def _parse_result(self, response, verbose=False): + def _parse_result(self, response, *, verbose=False): - results_table = self._current_connection._parse_result(response, verbose) + results_table = self._current_connection._parse_result(response, verbose=verbose) if len(results_table) == self.catalog_limit: warnings.warn("Maximum catalog results returned, may not include all sources within radius.", @@ -58,7 +59,7 @@ def _parse_result(self, response, verbose=False): return results_table @class_or_instance - def query_region_async(self, coordinates, radius=0.2*u.deg, catalog="Hsc", + def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", version=None, pagesize=None, page=None, **kwargs): """ Given a sky position and radius, returns a list of catalog entries. @@ -141,10 +142,17 @@ def query_region_async(self, coordinates, radius=0.2*u.deg, catalog="Hsc", if version == 1: service = "Mast.Catalogs.GaiaDR1.Cone" else: - if version not in (2, None): + if version not in (None, 2): warnings.warn("Invalid Gaia version number, defaulting to DR2.", InputWarning) service = "Mast.Catalogs.GaiaDR2.Cone" + elif catalog.lower() == 'plato': + if version in (None, 1): + service = "Mast.Catalogs.Plato.Cone" + else: + warnings.warn("Invalid PLATO catalog version number, defaulting to DR1.", InputWarning) + service = "Mast.Catalogs.Plato.Cone" + else: service = "Mast.Catalogs." + catalog + ".Cone" self.catalog_limit = None @@ -153,10 +161,10 @@ def query_region_async(self, coordinates, radius=0.2*u.deg, catalog="Hsc", for prop, value in kwargs.items(): params[prop] = value - return self._current_connection.service_request_async(service, params, pagesize, page) + return self._current_connection.service_request_async(service, params, pagesize=pagesize, page=page) @class_or_instance - def query_object_async(self, objectname, radius=0.2*u.deg, catalog="Hsc", + def query_object_async(self, objectname, *, radius=0.2*u.deg, catalog="Hsc", pagesize=None, page=None, version=None, **kwargs): """ Given an object name, returns a list of catalog entries. @@ -196,11 +204,16 @@ def query_object_async(self, objectname, radius=0.2*u.deg, catalog="Hsc", coordinates = utils.resolve_object(objectname) - return self.query_region_async(coordinates, radius, catalog, - version=version, pagesize=pagesize, page=page, **kwargs) + return self.query_region_async(coordinates, + radius=radius, + catalog=catalog, + version=version, + pagesize=pagesize, + page=page, + **kwargs) @class_or_instance - def query_criteria_async(self, catalog, pagesize=None, page=None, **criteria): + def query_criteria_async(self, catalog, *, pagesize=None, page=None, **criteria): """ Given an set of filters, returns a list of catalog entries. See column documentation for specific catalogs `here `__. @@ -302,7 +315,7 @@ def query_criteria_async(self, catalog, pagesize=None, page=None, **criteria): return self._current_connection.service_request_async(service, params, pagesize=pagesize, page=page) @class_or_instance - def query_hsc_matchid_async(self, match, version=3, pagesize=None, page=None): + def query_hsc_matchid_async(self, match, *, version=3, pagesize=None, page=None): """ Returns all the matches for a given Hubble Source Catalog MatchID. @@ -320,7 +333,7 @@ def query_hsc_matchid_async(self, match, version=3, pagesize=None, page=None): one sepcific page of results. Returns - -------- + ------- response : list of `~requests.Response` """ @@ -339,10 +352,10 @@ def query_hsc_matchid_async(self, match, version=3, pagesize=None, page=None): params = {"input": match} - return self._current_connection.service_request_async(service, params, pagesize, page) + return self._current_connection.service_request_async(service, params, pagesize=pagesize, page=page) @class_or_instance - def get_hsc_spectra_async(self, pagesize=None, page=None): + def get_hsc_spectra_async(self, *, pagesize=None, page=None): """ Returns all Hubble Source Catalog spectra. @@ -356,7 +369,7 @@ def get_hsc_spectra_async(self, pagesize=None, page=None): one sepcific page of results. Returns - -------- + ------- response : list of `~requests.Response` """ @@ -367,7 +380,7 @@ def get_hsc_spectra_async(self, pagesize=None, page=None): return self._current_connection.service_request_async(service, params, pagesize, page) - def download_hsc_spectra(self, spectra, download_dir=None, cache=True, curl_flag=False): + def download_hsc_spectra(self, spectra, *, download_dir=None, cache=True, curl_flag=False): """ Download one or more Hubble Source Catalog spectra. @@ -387,7 +400,7 @@ def download_hsc_spectra(self, spectra, download_dir=None, cache=True, curl_flag will be downloaded that can be used to download the data files at a later time. Returns - -------- + ------- response : list of `~requests.Response` """ diff --git a/astroquery/mast/cutouts.py b/astroquery/mast/cutouts.py index 29df098a1a..dd32b411f6 100644 --- a/astroquery/mast/cutouts.py +++ b/astroquery/mast/cutouts.py @@ -104,10 +104,13 @@ def __init__(self): super().__init__() services = {"sector": {"path": "sector"}, - "astrocut": {"path": "astrocut"}} + "astrocut": {"path": "astrocut"}, + "mt_sector": {"path": "moving_target/sector"}, + "mt_astrocut": {"path": "moving_target/astrocut"} + } self._service_api_connection.set_service_params(services, "tesscut") - def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None): + def get_sectors(self, *, coordinates=None, radius=0*u.deg, objectname=None, moving_target=False, mt_type=None): """ Get a list of the TESS data sectors whose footprints intersect with the given search area. @@ -117,36 +120,73 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None): coordinates : str or `astropy.coordinates` object, optional The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - One and only one of coordinates and objectname must be supplied. + + NOTE: If moving_target or objectname is supplied, this argument cannot be used. radius : str, float, or `~astropy.units.Quantity` object, optional Default 0 degrees. If supplied as a float degrees is the assumed unit. The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from `astropy.units` may also be used. + + NOTE: If moving_target is supplied, this argument is ignored. objectname : str, optional The target around which to search, by name (objectname="M104") - or TIC ID (objectname="TIC 141914082"). - One and only one of coordinates and objectname must be supplied. + or TIC ID (objectname="TIC 141914082"). If moving_target is True, input must be the name or ID (as understood by the + `JPL ephemerides service `__) + of a moving target such as an asteroid or comet. + + NOTE: If coordinates is supplied, this argument cannot be used. + moving_target : bool, optional + Indicate whether the object is a moving target or not. Default is set to False, in other words, not a moving target. + + NOTE: If coordinates is supplied, this argument cannot be used. + mt_type : str, optional + The moving target type, valid inputs are majorbody and smallbody. If not supplied + first majorbody is tried and then smallbody if a matching majorbody is not found. + + NOTE: If moving_target is supplied, this argument is ignored. Returns ------- response : `~astropy.table.Table` - Sector/camera/chip information for given coordinates/raduis. + Sector/camera/chip information for given coordinates/objectname/moving_target. """ - # Get Skycoord object for coordinates/object - coordinates = parse_input_location(coordinates, objectname) + if moving_target: - # If radius is just a number we assume degrees - radius = Angle(radius, u.deg) + # Check that objectname has been passed in and coordinates + # is not + if coordinates: + raise InvalidQueryError("Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname.") - params = {"ra": coordinates.ra.deg, - "dec": coordinates.dec.deg, - "radius": radius.deg} + if not objectname: + raise InvalidQueryError("Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet.") - response = self._service_api_connection.service_request_async("sector", params) - response.raise_for_status() # Raise any errors + params = {"obj_id": objectname} + + # Add optional parameter is present + if mt_type: + params["obj_type"] = mt_type + + response = self._service_api_connection.service_request_async("mt_sector", params) + + else: + + # Get Skycoord object for coordinates/object + coordinates = parse_input_location(coordinates, objectname) + + # If radius is just a number we assume degrees + radius = Angle(radius, u.deg) + + params = {"ra": coordinates.ra.deg, + "dec": coordinates.dec.deg, + "radius": radius.deg} + + response = self._service_api_connection.service_request_async("sector", params) + + # Raise any errors + response.raise_for_status() sector_json = response.json()['results'] sector_dict = {'sectorName': [], @@ -164,7 +204,8 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None): warnings.warn("Coordinates are not in any TESS sector.", NoResultsWarning) return Table(sector_dict) - def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", inflate=True, objectname=None): + def download_cutouts(self, *, coordinates=None, size=5, sector=None, path=".", inflate=True, + objectname=None, moving_target=False, mt_type=None): """ Download cutout target pixel file(s) around the given coordinates with indicated size. @@ -173,7 +214,8 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl coordinates : str or `astropy.coordinates` object, optional The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - One and only one of coordinates and objectname must be supplied. + + NOTE: If moving_target or objectname is supplied, this argument cannot be used. size : int, array-like, `~astropy.units.Quantity` Optional, default 5 pixels. The size of the cutout array. If ``size`` is a scalar number or @@ -197,28 +239,56 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl Set inflate to false to stop before the inflate step. objectname : str, optional The target around which to search, by name (objectname="M104") - or TIC ID (objectname="TIC 141914082"). - One and only one of coordinates and objectname must be supplied. + or TIC ID (objectname="TIC 141914082"). If moving_target is True, input must be the name or ID (as understood by the + `JPL ephemerides service `__) + of a moving target such as an asteroid or comet. + + NOTE: If coordinates is supplied, this argument cannot be used. + moving_target : str, optional + Indicate whether the object is a moving target or not. Default is set to False, in other words, not a moving target. + + NOTE: If coordinates is supplied, this argument cannot be used. + mt_type : str, optional + The moving target type, valid inputs are majorbody and smallbody. If not supplied + first majorbody is tried and then smallbody if a matching majorbody is not found. + + NOTE: If moving_target is supplied, this argument is ignored. Returns ------- response : `~astropy.table.Table` """ - # Get Skycoord object for coordinates/object - coordinates = parse_input_location(coordinates, objectname) + if moving_target: + + # Check that objectname has been passed in and coordinates + # is not + if coordinates: + raise InvalidQueryError("Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname.") + + if not objectname: + raise InvalidQueryError("Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet.") + + astrocut_request = f"moving_target/astrocut?obj_id={objectname}" + if mt_type: + astrocut_request += f"&obj_type={mt_type}" + + else: + + # Get Skycoord object for coordinates/object + coordinates = parse_input_location(coordinates, objectname) + + astrocut_request = f"astrocut?ra={coordinates.ra.deg}&dec={coordinates.dec.deg}" + + # Adding the arguments that are common between moving/still astrocut requests size_dict = _parse_cutout_size(size) + astrocut_request += f"&y={size_dict['y']}&x={size_dict['x']}&units={size_dict['units']}" - path = os.path.join(path, '') - astrocut_request = "ra={}&dec={}&y={}&x={}&units={}".format(coordinates.ra.deg, - coordinates.dec.deg, - size_dict["y"], - size_dict["x"], - size_dict["units"]) if sector: astrocut_request += "§or={}".format(sector) - astrocut_url = self._service_api_connection.REQUEST_URL + "astrocut?" + astrocut_request + astrocut_url = self._service_api_connection.REQUEST_URL + astrocut_request + path = os.path.join(path, '') zipfile_path = "{}tesscut_{}.zip".format(path, time.strftime("%Y%m%d%H%M%S")) self._download_file(astrocut_url, zipfile_path) @@ -246,7 +316,8 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl localpath_table['Local Path'] = [path+x for x in cutout_files] return localpath_table - def get_cutouts(self, coordinates=None, size=5, sector=None, objectname=None): + def get_cutouts(self, *, coordinates=None, size=5, sector=None, + objectname=None, moving_target=False, mt_type=None): """ Get cutout target pixel file(s) around the given coordinates with indicated size, and return them as a list of `~astropy.io.fits.HDUList` objects. @@ -256,7 +327,8 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, objectname=None): coordinates : str or `astropy.coordinates` object, optional The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - One and only one of coordinates and objectname must be supplied. + + NOTE: If moving_target or objectname is supplied, this argument cannot be used. size : int, array-like, `~astropy.units.Quantity` Optional, default 5 pixels. The size of the cutout array. If ``size`` is a scalar number or @@ -271,25 +343,61 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, objectname=None): from all available sectors on which the coordinate appears will be returned. objectname : str, optional The target around which to search, by name (objectname="M104") - or TIC ID (objectname="TIC 141914082"). - One and only one of coordinates and objectname must be supplied. + or TIC ID (objectname="TIC 141914082"). If moving_target is True, input must be the name or ID (as understood by the + `JPL ephemerides service `__) + of a moving target such as an asteroid or comet. + + NOTE: If coordinates is supplied, this argument cannot be used. + moving_target : str, optional + Indicate whether the object is a moving target or not. Default is set to False, in other words, not a moving target. + + NOTE: If coordinates is supplied, this argument cannot be used. + mt_type : str, optional + The moving target type, valid inputs are majorbody and smallbody. If not supplied + first majorbody is tried and then smallbody if a matching majorbody is not found. + + NOTE: If moving_target is supplied, this argument is ignored. Returns ------- response : A list of `~astropy.io.fits.HDUList` objects. """ - # Get Skycoord object for coordinates/object - coordinates = parse_input_location(coordinates, objectname) - + # Setting up the cutout size param_dict = _parse_cutout_size(size) - param_dict["ra"] = coordinates.ra.deg - param_dict["dec"] = coordinates.dec.deg + # Add sector if present if sector: param_dict["sector"] = sector - response = self._service_api_connection.service_request_async("astrocut", param_dict) + if moving_target: + + # Check that objectname has been passed in and coordinates + # is not + if coordinates: + raise InvalidQueryError("Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname.") + + if not objectname: + raise InvalidQueryError("Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet.") + + param_dict["obj_id"] = objectname + + # Add optional parameter if present + if mt_type: + param_dict["obj_type"] = mt_type + + response = self._service_api_connection.service_request_async("mt_astrocut", param_dict) + + else: + + # Get Skycoord object for coordinates/object + coordinates = parse_input_location(coordinates, objectname) + + param_dict["ra"] = coordinates.ra.deg + param_dict["dec"] = coordinates.dec.deg + + response = self._service_api_connection.service_request_async("astrocut", param_dict) + response.raise_for_status() # Raise any errors try: @@ -334,7 +442,7 @@ def __init__(self): "astrocut": {"path": "astrocut"}} self._service_api_connection.set_service_params(services, "zcut") - def get_surveys(self, coordinates, radius="0d"): + def get_surveys(self, coordinates, *, radius="0d"): """ Gives a list of deep field surveys available for a position in the sky @@ -373,7 +481,7 @@ def get_surveys(self, coordinates, radius="0d"): warnings.warn("Coordinates are not in an available deep field survey.", NoResultsWarning) return survey_json - def download_cutouts(self, coordinates, size=5, survey=None, cutout_format="fits", path=".", inflate=True, **img_params): + def download_cutouts(self, coordinates, *, size=5, survey=None, cutout_format="fits", path=".", inflate=True, **img_params): """ Download cutout FITS/image file(s) around the given coordinates with indicated size. @@ -467,7 +575,7 @@ def download_cutouts(self, coordinates, size=5, survey=None, cutout_format="fits localpath_table['Local Path'] = [path+x for x in cutout_files] return localpath_table - def get_cutouts(self, coordinates, size=5, survey=None): + def get_cutouts(self, coordinates, *, size=5, survey=None): """ Get cutout FITS file(s) around the given coordinates with indicated size, and return them as a list of `~astropy.io.fits.HDUList` objects. diff --git a/astroquery/mast/discovery_portal.py b/astroquery/mast/discovery_portal.py index 6ced3cc918..63f7841d4a 100644 --- a/astroquery/mast/discovery_portal.py +++ b/astroquery/mast/discovery_portal.py @@ -116,23 +116,23 @@ class PortalAPI(BaseQuery): Should be used to facilitate all Portal API queries. """ + MAST_REQUEST_URL = conf.server + "/api/v0/invoke" + COLUMNS_CONFIG_URL = conf.server + "/portal/Mashup/Mashup.asmx/columnsconfig" + MAST_DOWNLOAD_URL = conf.server + "/api/v0.1/Download/file" + MAST_BUNDLE_URL = conf.server + "/api/v0.1/Download/bundle" + + TIMEOUT = conf.timeout + PAGESIZE = conf.pagesize + + _column_configs = dict() + _current_service = None + def __init__(self, session=None): super(PortalAPI, self).__init__() if session: self._session = session - self.MAST_REQUEST_URL = conf.server + "/api/v0/invoke" - self.COLUMNS_CONFIG_URL = conf.server + "/portal/Mashup/Mashup.asmx/columnsconfig" - self.MAST_DOWNLOAD_URL = conf.server + "/api/v0.1/Download/file" - self.MAST_BUNDLE_URL = conf.server + "/api/v0.1/Download/bundle" - - self.TIMEOUT = conf.timeout - self.PAGESIZE = conf.pagesize - - self._column_configs = dict() - self._current_service = None - def _request(self, method, url, params=None, data=None, headers=None, files=None, stream=False, auth=None, retrieve_all=True): """ @@ -482,6 +482,7 @@ def _get_columnsconfig_metadata(self, colconf_name): headers = {"User-Agent": self._session.headers["User-Agent"], "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} + response = self._request("POST", self.COLUMNS_CONFIG_URL, data=("colConfigId={}".format(colconf_name)), headers=headers) diff --git a/astroquery/mast/missions.py b/astroquery/mast/missions.py new file mode 100644 index 0000000000..caa264aca7 --- /dev/null +++ b/astroquery/mast/missions.py @@ -0,0 +1,261 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +MAST Missions +================= + +This module contains methods for searching MAST missions. +""" + +import requests +import warnings + +from astropy.table import Table +import astropy.units as u +import astropy.coordinates as coord + +from astroquery.utils import commons, async_to_sync +from astroquery.utils.class_or_instance import class_or_instance +from astroquery.exceptions import InvalidQueryError, MaxResultsWarning + +from astroquery.mast import utils +from astroquery.mast.core import MastQueryWithLogin + +from . import conf + +__all__ = ['MastMissionsClass', 'MastMissions'] + + +@async_to_sync +class MastMissionsClass(MastQueryWithLogin): + """ + MastMissions search class. + Class that allows direct programatic access to retrieve metadata via the MAST search API for a given mission. + """ + + def __init__(self, *, mission='hst', service='search'): + super().__init__() + + self._search_option_fields = ['limit', 'offset', 'sort_by', 'search_key', 'sort_desc', 'select_cols', + 'skip_count', 'user_fields'] + self.service = service + self.mission = mission + self.limit = 5000 + + service_dict = {self.service: {'path': self.service, 'args': {}}} + self._service_api_connection.set_service_params(service_dict, f"{self.service}/{self.mission}") + + def _parse_result(self, response, *, verbose=False): # Used by the async_to_sync decorator functionality + """ + Parse the results of a `~requests.Response` objects and return an `~astropy.table.Table` of results. + + Parameters + ---------- + response : `~requests.Response` + `~requests.Response` objects. + verbose : bool + (presently does nothing - there is no output with verbose set to + True or False) + Default False. Setting to True provides more extensive output. + + Returns + ------- + response : `~astropy.table.Table` + """ + + results = self._service_api_connection._parse_result(response, verbose, data_key='results') + if len(results) >= self.limit: + warnings.warn("Maximum results returned, may not include all sources within radius.", + MaxResultsWarning) + + return results + + @class_or_instance + def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs): + """ + Given a sky position and radius, returns a list of matching dataset IDs. + + Parameters + ---------- + coordinates : str or `~astropy.coordinates` object + The target around which to search. It may be specified as a + string or as the appropriate `~astropy.coordinates` object. + radius : str or `~astropy.units.Quantity` object, optional + Default 3 degrees. + The string must be parsable by `~astropy.coordinates.Angle`. The + appropriate `~astropy.units.Quantity` object from + `~astropy.units` may also be used. Defaults to 3 arcminutes. + limit : int + Optional and default is 5000. + the maximun number of dataset IDs in the results. + offset : int + Optional and default is 0 + the number of records you wish to skip before selecting records. + **kwargs + Other mission-specific keyword args. + Any invalid keys are ignored by the API. + All valid key names can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list` + function. + For example one can specify the output columns(select_cols) or use other filters(conditions) + + Returns + ------- + response : list of `~requests.Response` + """ + + self.limit = limit + + # Put coordinates and radius into consistant format + coordinates = commons.parse_coordinates(coordinates) + + # if radius is just a number we assume degrees + radius = coord.Angle(radius, u.arcmin) + + # basic params + params = {'target': [f"{coordinates.ra.deg} {coordinates.dec.deg}"], + 'radius': radius.arcmin, + 'radius_units': 'arcminutes', + 'limit': limit, + 'offset': offset} + + params['conditions'] = [] + # adding additional user specified parameters + for prop, value in kwargs.items(): + if prop not in self._search_option_fields: + params['conditions'].append({prop: value}) + else: + params[prop] = value + + return self._service_api_connection.service_request_async(self.service, params, use_json=True) + + @class_or_instance + def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u.arcmin, + limit=5000, offset=0, select_cols=[], **criteria): + """ + Given a set of search criteria, returns a list of mission metadata. + + Parameters + ---------- + coordinates : str or `~astropy.coordinates` object + The target around which to search. It may be specified as a + string or as the appropriate `~astropy.coordinates` object. + objectname : str + The name of the target around which to search. + radius : str or `~astropy.units.Quantity` object, optional + Default 3 degrees. + The string must be parsable by `~astropy.coordinates.Angle`. The + appropriate `~astropy.units.Quantity` object from + `~astropy.units` may also be used. Defaults to 3 arcminutes. + limit : int + Optional and default is 5000. + the maximun number of dataset IDs in the results. + offset : int + Optional and default is 0. + the number of records you wish to skip before selecting records. + select_cols: list + names of columns that will be included in the astropy table + **criteria + Criteria to apply. At least one non-positional criteria must be supplied. + Valid criteria are coordinates, objectname, radius (as in + `~astroquery.mast.missions.MastMissionsClass.query_region` and + `~astroquery.mast.missions.MastMissionsClass.query_object` functions), + and all fields listed in the column documentation for the mission being queried. + Any invalid keys passed in criteria are ignored by the API. + List of all valid fields that can be used to match results on criteria can be retrieved by calling + `~astroquery.mast.missions.MastMissionsClass.get_column_list` function. + + Returns + ------- + response : list of `~requests.Response` + """ + + self.limit = limit + + if objectname or coordinates: + coordinates = utils.parse_input_location(coordinates, objectname) + + # if radius is just a number we assume degrees + radius = coord.Angle(radius, u.arcmin) + + # build query + params = {"limit": self.limit, "offset": offset, 'select_cols': select_cols} + if coordinates: + params["target"] = [f"{coordinates.ra.deg} {coordinates.dec.deg}"] + params["radius"] = radius.arcmin + params["radius_units"] = 'arcminutes' + + if not self._service_api_connection.check_catalogs_criteria_params(criteria): + raise InvalidQueryError("At least one non-positional criterion must be supplied.") + + params['conditions'] = [] + for prop, value in criteria.items(): + if prop not in self._search_option_fields: + params['conditions'].append({prop: value}) + else: + params[prop] = value + + return self._service_api_connection.service_request_async(self.service, params, use_json=True) + + @class_or_instance + def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs): + """ + Given an object name, returns a list of matching rows. + + Parameters + ---------- + objectname : str + The name of the target around which to search. + radius : str or `~astropy.units.Quantity` object, optional + Default 3 arcmin. + The string must be parsable by `~astropy.coordinates.Angle`. + The appropriate `~astropy.units.Quantity` object from + `~astropy.units` may also be used. Defaults to 3 arcminutes. + limit : int + Optional and default is 5000. + the maximun number of dataset IDs in the results. + offset : int + Optional and default is 0. + the number of records you wish to skip before selecting records. + **kwargs + Mission-specific keyword args. + Any invalid keys are ignored by the API. + All valid keys can be found by calling `~astroquery.mast.missions.MastMissionsClass.get_column_list` + function. + + Returns + ------- + response : list of `~requests.Response` + """ + + coordinates = utils.resolve_object(objectname) + + return self.query_region_async(coordinates, radius=radius, limit=limit, offset=offset, **kwargs) + + @class_or_instance + def get_column_list(self): + """ + For a mission, return a list of all searchable columns and their descriptions + + Returns + ------- + response : `~astropy.table.Table` that contains columns names, types and their descriptions + """ + + url = f"{conf.server}/search/util/api/v0.1/column_list?mission={self.mission}" + + try: + results = requests.get(url) + results = results.json() + rows = [] + for result in results: + result.pop('field_name') + result.pop('queryable') + result.pop('indexed') + result.pop('default_output') + rows.append((result['column_name'], result['qual_type'], result['description'])) + data_table = Table(rows=rows, names=('name', 'data_type', 'description')) + return data_table + except Exception: + raise Exception(f"Error occured while trying to get column list for mission {self.mission}") + + +MastMissions = MastMissionsClass() diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index fba7fc8dcf..7c75a57b41 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -6,7 +6,6 @@ This module contains various methods for querying MAST observations. """ - import warnings import json import time @@ -38,7 +37,6 @@ from . import conf, utils from .core import MastQueryWithLogin - __all__ = ['Observations', 'ObservationsClass', 'MastClass', 'Mast'] @@ -51,7 +49,14 @@ class ObservationsClass(MastQueryWithLogin): Class for querying MAST observational data. """ - def _parse_result(self, responses, verbose=False): # Used by the async_to_sync decorator functionality + # Calling static class variables + _caom_all = 'Mast.Caom.All' + _caom_cone = 'Mast.Caom.Cone' + _caom_filtered_position = 'Mast.Caom.Filtered.Position' + _caom_filtered = 'Mast.Caom.Filtered' + _caom_products = 'Mast.Caom.Products' + + def _parse_result(self, responses, *, verbose=False): # Used by the async_to_sync decorator functionality """ Parse the results of a list of `~requests.Response` objects and returns an `~astropy.table.Table` of results. @@ -76,13 +81,13 @@ def list_missions(self): Lists data missions archived by MAST and avaiable through `astroquery.mast`. Returns - -------- + ------- response : list List of available missions. """ # getting all the histogram information - service = "Mast.Caom.All" + service = self._caom_all params = {} response = self._portal_api_connection.service_request_async(service, params, format='extjs') json_response = response[0].json() @@ -106,15 +111,15 @@ def get_metadata(self, query_type): The query to get metadata for. Options are observations, and products. Returns - -------- + ------- response : `~astropy.table.Table` The metadata table. """ if query_type.lower() == "observations": - colconf_name = "Mast.Caom.Cone" + colconf_name = self._caom_cone elif query_type.lower() == "products": - colconf_name = "Mast.Caom.Products" + colconf_name = self._caom_products else: raise InvalidQueryError("Unknown query type.") @@ -152,14 +157,14 @@ def _parse_caom_criteria(self, **criteria): # Build the mashup filter object and store it in the correct service_name entry if coordinates or objectname: - mashup_filters = self._portal_api_connection.build_filter_set("Mast.Caom.Cone", - "Mast.Caom.Filtered.Position", - **criteria) + mashup_filters = self._portal_api_connection.build_filter_set(self._caom_cone, + self._caom_filtered_position, + **criteria) coordinates = utils.parse_input_location(coordinates, objectname) else: - mashup_filters = self._portal_api_connection.build_filter_set("Mast.Caom.Cone", - "Mast.Caom.Filtered", - **criteria) + mashup_filters = self._portal_api_connection.build_filter_set(self._caom_cone, + self._caom_filtered, + **criteria) # handle position info (if any) position = None @@ -168,13 +173,13 @@ def _parse_caom_criteria(self, **criteria): # if radius is just a number we assume degrees radius = coord.Angle(radius, u.deg) - # build the coordinates string needed by Mast.Caom.Filtered.Position + # build the coordinates string needed by ObservationsClass._caom_filtered_position position = ', '.join([str(x) for x in (coordinates.ra.deg, coordinates.dec.deg, radius.deg)]) return position, mashup_filters @class_or_instance - def query_region_async(self, coordinates, radius=0.2*u.deg, pagesize=None, page=None): + def query_region_async(self, coordinates, *, radius=0.2*u.deg, pagesize=None, page=None): """ Given a sky position and radius, returns a list of MAST observations. See column documentation `here `__. @@ -209,15 +214,15 @@ def query_region_async(self, coordinates, radius=0.2*u.deg, pagesize=None, page= # if radius is just a number we assume degrees radius = coord.Angle(radius, u.deg) - service = 'Mast.Caom.Cone' + service = self._caom_cone params = {'ra': coordinates.ra.deg, 'dec': coordinates.dec.deg, 'radius': radius.deg} - return self._portal_api_connection.service_request_async(service, params, pagesize, page) + return self._portal_api_connection.service_request_async(service, params, pagesize=pagesize, page=page) @class_or_instance - def query_object_async(self, objectname, radius=0.2*u.deg, pagesize=None, page=None): + def query_object_async(self, objectname, *, radius=0.2*u.deg, pagesize=None, page=None): """ Given an object name, returns a list of MAST observations. See column documentation `here `__. @@ -247,10 +252,10 @@ def query_object_async(self, objectname, radius=0.2*u.deg, pagesize=None, page=N coordinates = utils.resolve_object(objectname) - return self.query_region_async(coordinates, radius, pagesize, page) + return self.query_region_async(coordinates, radius=radius, pagesize=pagesize, page=page) @class_or_instance - def query_criteria_async(self, pagesize=None, page=None, **criteria): + def query_criteria_async(self, *, pagesize=None, page=None, **criteria): """ Given an set of criteria, returns a list of MAST observations. Valid criteria are returned by ``get_metadata("observations")`` @@ -286,18 +291,18 @@ def query_criteria_async(self, pagesize=None, page=None, **criteria): raise InvalidQueryError("At least one non-positional criterion must be supplied.") if position: - service = "Mast.Caom.Filtered.Position" + service = self._caom_filtered_position params = {"columns": "*", "filters": mashup_filters, "position": position} else: - service = "Mast.Caom.Filtered" + service = self._caom_filtered params = {"columns": "*", "filters": mashup_filters} return self._portal_api_connection.service_request_async(service, params) - def query_region_count(self, coordinates, radius=0.2*u.deg, pagesize=None, page=None): + def query_region_count(self, coordinates, *, radius=0.2*u.deg, pagesize=None, page=None): """ Given a sky position and radius, returns the number of MAST observations in that region. @@ -322,7 +327,7 @@ def query_region_count(self, coordinates, radius=0.2*u.deg, pagesize=None, page= response : int """ - # build the coordinates string needed by Mast.Caom.Filtered.Position + # build the coordinates string needed by ObservationsClass._caom_filtered_position coordinates = commons.parse_coordinates(coordinates) # if radius is just a number we assume degrees @@ -331,14 +336,14 @@ def query_region_count(self, coordinates, radius=0.2*u.deg, pagesize=None, page= # turn coordinates into the format position = ', '.join([str(x) for x in (coordinates.ra.deg, coordinates.dec.deg, radius.deg)]) - service = "Mast.Caom.Filtered.Position" + service = self._caom_filtered_position params = {"columns": "COUNT_BIG(*)", "filters": [], "position": position} return int(self._portal_api_connection.service_request(service, params, pagesize, page)[0][0]) - def query_object_count(self, objectname, radius=0.2*u.deg, pagesize=None, page=None): + def query_object_count(self, objectname, *, radius=0.2*u.deg, pagesize=None, page=None): """ Given an object name, returns the number of MAST observations. @@ -364,9 +369,9 @@ def query_object_count(self, objectname, radius=0.2*u.deg, pagesize=None, page=N coordinates = utils.resolve_object(objectname) - return self.query_region_count(coordinates, radius, pagesize, page) + return self.query_region_count(coordinates, radius=radius, pagesize=pagesize, page=page) - def query_criteria_count(self, pagesize=None, page=None, **criteria): + def query_criteria_count(self, *, pagesize=None, page=None, **criteria): """ Given an set of filters, returns the number of MAST observations meeting those criteria. @@ -399,12 +404,12 @@ def query_criteria_count(self, pagesize=None, page=None, **criteria): # send query if position: - service = "Mast.Caom.Filtered.Position" + service = self._caom_filtered_position params = {"columns": "COUNT_BIG(*)", "filters": mashup_filters, "position": position} else: - service = "Mast.Caom.Filtered" + service = self._caom_filtered params = {"columns": "COUNT_BIG(*)", "filters": mashup_filters} @@ -440,12 +445,12 @@ def get_product_list_async(self, observations): if len(observations) == 0: raise InvalidQueryError("Observation list is empty, no associated products.") - service = 'Mast.Caom.Products' + service = self._caom_products params = {'obsid': ','.join(observations)} return self._portal_api_connection.service_request_async(service, params) - def filter_products(self, products, mrp_only=False, extension=None, **filters): + def filter_products(self, products, *, mrp_only=False, extension=None, **filters): """ Takes an `~astropy.table.Table` of MAST observation data products and filters it based on given filters. @@ -500,7 +505,7 @@ def filter_products(self, products, mrp_only=False, extension=None, **filters): return products[np.where(filter_mask)] - def download_file(self, uri, local_path=None, base_url=None, cache=True, cloud_only=False): + def download_file(self, uri, *, local_path=None, base_url=None, cache=True, cloud_only=False): """ Downloads a single file based on the data URI @@ -577,7 +582,7 @@ def download_file(self, uri, local_path=None, base_url=None, cache=True, cloud_o return status, msg, url - def _download_files(self, products, base_dir, cache=True, cloud_only=False,): + def _download_files(self, products, base_dir, *, cache=True, cloud_only=False,): """ Takes an `~astropy.table.Table` of data products and downloads them into the directory given by base_dir. @@ -653,7 +658,7 @@ def _download_curl_script(self, products, out_dir): 'Message': [msg]}) return manifest - def download_products(self, products, download_dir=None, + def download_products(self, products, *, download_dir=None, cache=True, curl_flag=False, mrp_only=False, cloud_only=False, **filters): """ Download data products. @@ -710,7 +715,7 @@ def download_products(self, products, download_dir=None, products = vstack(product_lists) # apply filters - products = self.filter_products(products, mrp_only, **filters) + products = self.filter_products(products, mrp_only=mrp_only, **filters) if not len(products): warnings.warn("No products to download.", NoResultsWarning) @@ -721,15 +726,19 @@ def download_products(self, products, download_dir=None, download_dir = '.' if curl_flag: # don't want to download the files now, just the curl script - manifest = self._download_curl_script(products, download_dir) + manifest = self._download_curl_script(products, + download_dir) else: base_dir = download_dir.rstrip('/') + "/mastDownload" - manifest = self._download_files(products, base_dir, cache, cloud_only) + manifest = self._download_files(products, + base_dir=base_dir, + cache=cache, + cloud_only=cloud_only) return manifest - def get_cloud_uris(self, data_products, include_bucket=True, full_url=False): + def get_cloud_uris(self, data_products, *, include_bucket=True, full_url=False): """ Takes an `~astropy.table.Table` of data products and returns the associated cloud data uris. @@ -758,7 +767,7 @@ def get_cloud_uris(self, data_products, include_bucket=True, full_url=False): return self._cloud_connection.get_cloud_uri_list(data_products, include_bucket, full_url) - def get_cloud_uri(self, data_product, include_bucket=True, full_url=False): + def get_cloud_uri(self, data_product, *, include_bucket=True, full_url=False): """ For a given data product, returns the associated cloud URI. If the product is from a mission that does not support cloud access an @@ -801,7 +810,7 @@ class MastClass(MastQueryWithLogin): more flexible but less user friendly than `ObservationsClass`. """ - def _parse_result(self, responses, verbose=False): # Used by the async_to_sync decorator functionality + def _parse_result(self, responses, *, verbose=False): # Used by the async_to_sync decorator functionality """ Parse the results of a list of `~requests.Response` objects and returns an `~astropy.table.Table` of results. @@ -822,7 +831,7 @@ def _parse_result(self, responses, verbose=False): # Used by the async_to_sync return self._portal_api_connection._parse_result(responses, verbose) @class_or_instance - def service_request_async(self, service, params, pagesize=None, page=None, **kwargs): + def service_request_async(self, service, params, *, pagesize=None, page=None, **kwargs): """ Given a Mashup service and parameters, builds and excecutes a Mashup query. See documentation `here `__ diff --git a/astroquery/mast/services.py b/astroquery/mast/services.py index acc85df035..15e6f516a7 100644 --- a/astroquery/mast/services.py +++ b/astroquery/mast/services.py @@ -25,14 +25,16 @@ __all__ = [] -def _json_to_table(json_obj): +def _json_to_table(json_obj, data_key='data'): """ Takes a JSON object as returned from a MAST microservice request and turns it into an `~astropy.table.Table`. Parameters ---------- - json_obj : dict + json_obj : data array or list of dictionaries A MAST microservice response JSON object (python dictionary) + data_key : str + string that contains the key name in json_obj that stores the data rows Returns ------- @@ -40,42 +42,45 @@ def _json_to_table(json_obj): """ data_table = Table(masked=True) - if not all(x in json_obj.keys() for x in ['info', 'data']): - raise KeyError("Missing required key(s) 'data' and/or 'info.'") + if not all(x in json_obj.keys() for x in ['info', data_key]): + raise KeyError(f"Missing required key(s) {data_key} and/or 'info.'") # determine database type key in case missing type_key = 'type' if json_obj['info'][0].get('type') else 'db_type' # for each item in info, store the type and column name + # for each item in info, type has to be converted from DB data types (SQL server in most cases) + # from missions_mast search service such as varchar, integer, float, boolean etc + # to corresponding numpy type for idx, col, col_type, ignore_value in \ - [(idx, x['name'], x[type_key], "NULL") for idx, x in enumerate(json_obj['info'])]: + [(idx, x['name'], x[type_key].lower(), None) for idx, x in enumerate(json_obj['info'])]: - # if default value is NULL, set ignore value to None - if ignore_value == "NULL": - ignore_value = None # making type adjustments - if col_type == "char" or col_type == "STRING": + if (col_type == "char" or col_type == "string" or 'varchar' in col_type or col_type == "null" or + col_type == 'datetime'): col_type = "str" ignore_value = "" if (ignore_value is None) else ignore_value - elif col_type == "boolean" or col_type == "BINARY": + elif col_type == "boolean" or col_type == "binary": col_type = "bool" - elif col_type == "unsignedByte": + elif col_type == "unsignedbyte": col_type = np.ubyte - elif col_type == "int" or col_type == "short" or col_type == "long" or col_type == "NUMBER": + elif (col_type == "int" or col_type == "short" or col_type == "long" or col_type == "number" + or col_type == 'integer'): # int arrays do not admit Non/nan vals col_type = np.int64 ignore_value = -999 if (ignore_value is None) else ignore_value - elif col_type == "double" or col_type == "float" or col_type == "DECIMAL": + elif col_type == "double" or col_type.lower() == "float" or col_type == "decimal": # int arrays do not admit Non/nan vals col_type = np.float64 ignore_value = -999 if (ignore_value is None) else ignore_value - elif col_type == "DATETIME": - col_type = "str" - ignore_value = "" if (ignore_value is None) else ignore_value # Make the column list (don't assign final type yet or there will be errors) - # Step through data array of values - col_data = np.array([x[idx] for x in json_obj['data']], dtype=object) + try: + # Step through data array of values + col_data = np.array([x[idx] for x in json_obj[data_key]], dtype=object) + except KeyError: + # it's not a data array, fall back to using column name as it is array of dictionaries + col_data = np.array([x[col] for x in json_obj[data_key]], dtype=object) if ignore_value is not None: col_data[np.where(np.equal(col_data, None))] = ignore_value @@ -101,15 +106,16 @@ class ServiceAPI(BaseQuery): Should be used to facilitate all microservice API queries. """ + SERVICE_URL = conf.server + REQUEST_URL = conf.server + "/api/v0.1/" + SERVICES = {} + def __init__(self, session=None): super().__init__() if session: self._session = session - self.REQUEST_URL = conf.server + "/api/v0.1/" - self.SERVICES = {} - self.TIMEOUT = conf.timeout def set_service_params(self, service_dict, service_name="", server_prefix=False): @@ -128,7 +134,7 @@ def set_service_params(self, service_dict, service_name="", server_prefix=False) vs. the default of mast.stsci.edu/service_name """ - service_url = conf.server + service_url = self.SERVICE_URL if server_prefix: service_url = service_url.replace("mast", f"{service_name}.mast") else: @@ -138,7 +144,7 @@ def set_service_params(self, service_dict, service_name="", server_prefix=False) self.SERVICES = service_dict def _request(self, method, url, params=None, data=None, headers=None, - files=None, stream=False, auth=None, cache=False): + files=None, stream=False, auth=None, cache=False, use_json=False): """ Override of the parent method: A generic HTTP request method, similar to `~requests.Session.request` @@ -163,7 +169,9 @@ def _request(self, method, url, params=None, data=None, headers=None, stream : bool See `~requests.request` cache : bool - Default False. Use of bulit in _request caching + Default False. Use of built in caching + use_json: bool + Default False. if True then data is already in json format. Returns ------- @@ -173,8 +181,12 @@ def _request(self, method, url, params=None, data=None, headers=None, start_time = time.time() - response = super()._request(method, url, params=params, data=data, headers=headers, - files=files, cache=cache, stream=stream, auth=auth) + if use_json: + response = super()._request(method, url, params=params, json=data, headers=headers, + files=files, cache=cache, stream=stream, auth=auth) + else: + response = super()._request(method, url, params=params, data=data, headers=headers, + files=files, cache=cache, stream=stream, auth=auth) if (time.time() - start_time) >= self.TIMEOUT: raise TimeoutError("Timeout limit of {} exceeded.".format(self.TIMEOUT)) @@ -182,7 +194,7 @@ def _request(self, method, url, params=None, data=None, headers=None, response.raise_for_status() return response - def _parse_result(self, response, verbose=False): + def _parse_result(self, response, verbose=False, data_key='data'): """ Parses the results of a `~requests.Response` object and returns an `~astropy.table.Table` of results. @@ -194,6 +206,8 @@ def _parse_result(self, response, verbose=False): (presently does nothing - there is no output with verbose set to True or False) Default False. Setting to True provides more extensive output. + data_key : str + the key in response that contains the data rows Returns ------- @@ -201,7 +215,7 @@ def _parse_result(self, response, verbose=False): """ result = response.json() - result_table = _json_to_table(result) + result_table = _json_to_table(result, data_key=data_key) # Check for no results if not result_table: @@ -209,7 +223,7 @@ def _parse_result(self, response, verbose=False): return result_table @class_or_instance - def service_request_async(self, service, params, page_size=None, page=None, **kwargs): + def service_request_async(self, service, params, page_size=None, page=None, use_json=False, **kwargs): """ Given a MAST fabric service and parameters, builds and excecutes a fabric microservice catalog query. See documentation `here `__ @@ -229,6 +243,8 @@ def service_request_async(self, service, params, page_size=None, page=None, **kw Default None. Can be used to override the default behavior of all results being returned to obtain a specific page of results. + use_json: bool, optional + if True, params are directly passed as json object **kwargs : See Catalogs.MAST properties in documentation referenced above @@ -248,9 +264,10 @@ def service_request_async(self, service, params, page_size=None, page=None, **kw compiled_service_args[service_argument] = found_argument.lower() request_url = self.REQUEST_URL + service_url.format(**compiled_service_args) + headers = { 'User-Agent': self._session.headers['User-Agent'], - 'Content-type': 'application/x-www-form-urlencoded', + 'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json' } # Params as a list of tuples to allow for multiple parameters added @@ -265,11 +282,15 @@ def service_request_async(self, service, params, page_size=None, page=None, **kw if page_size is not None: catalogs_request.append(('pagesize', page_size)) - # Decompose filters, sort - for prop, value in kwargs.items(): - params[prop] = value - catalogs_request.extend(self._build_catalogs_params(params)) - response = self._request('POST', request_url, data=catalogs_request, headers=headers) + if not use_json: + # Decompose filters, sort + for prop, value in kwargs.items(): + params[prop] = value + catalogs_request.extend(self._build_catalogs_params(params)) + else: + headers['Content-Type'] = 'application/json' + catalogs_request = params + response = self._request('POST', request_url, data=catalogs_request, headers=headers, use_json=use_json) return response def _build_catalogs_params(self, params): diff --git a/astroquery/mast/tests/data/mission_incorrect_results.json b/astroquery/mast/tests/data/mission_incorrect_results.json new file mode 100644 index 0000000000..0393192137 --- /dev/null +++ b/astroquery/mast/tests/data/mission_incorrect_results.json @@ -0,0 +1,135 @@ +{ + "messages": [], + "info": [ + { + "name": "sci_release_date", + "type": "DATETIME" + }, + { + "name": "sci_actual_duration", + "type": "FLOAT" + }, + { + "name": "sci_dec", + "type": "FLOAT" + }, + { + "name": "sci_pep_id", + "type": "INTEGER" + }, + { + "name": "sci_spec_1234", + "type": "VARCHAR" + }, + { + "name": "sci_aper_1234", + "type": "VARCHAR" + }, + { + "name": "sci_data_set_name", + "type": "VARCHAR" + }, + { + "name": "sci_preview_name", + "type": "VARCHAR" + }, + { + "name": "sci_targname", + "type": "VARCHAR" + }, + { + "name": "sci_instrume", + "type": "VARCHAR" + }, + { + "name": "search_key", + "type": "VARCHAR" + }, + { + "name": "sci_central_wavelength", + "type": "FLOAT" + }, + { + "name": "sci_status", + "type": "VARCHAR" + }, + { + "name": "sci_stop_time", + "type": "DATETIME" + }, + { + "name": "scp_scan_type", + "type": "VARCHAR" + }, + { + "name": "sci_hlsp", + "type": "INTEGER" + }, + { + "name": "sci_refnum", + "type": "INTEGER" + }, + { + "name": "sci_start_time", + "type": "DATETIME" + }, + { + "name": "sci_ra", + "type": "FLOAT" + } +], + "search_params": { + "target": [ + "40.66963 -0.01328" + ], + "radius": 3, + "radius_units": "arcminutes", + "offset": 0, + "limit": 5000, + "sort_by": [ + "ang_sep", + "sci_targname", + "sci_data_set_name" + ], + "sort_desc": [ + false, + false, + false + ], + "skip_count": false, + "select_cols": [ + "sci_data_set_name", + "sci_targname", + "sci_ra", + "sci_dec", + "sci_refnum", + "sci_start_time", + "sci_stop_time", + "sci_actual_duration", + "sci_instrume", + "sci_aper_1234", + "sci_spec_1234", + "sci_central_wavelength", + "sci_pep_id", + "sci_release_date", + "sci_preview_name", + "scp_scan_type", + "sci_hlsp", + "sci_status" + ], + "search_key": [], + "user_fields": [], + "conditions": [ + { + "sci_obs_type": "" + }, + { + "sci_aec": "S" + }, + { + "sci_instrume": "STIS,ACS,WFC3,COS,FGS,FOC,FOS,HRS,HSP,NICMOS,WFPC,WFPC2" + } + ] + }, + "totalResults": 3 +} diff --git a/astroquery/mast/tests/data/mission_results.json b/astroquery/mast/tests/data/mission_results.json new file mode 100644 index 0000000000..a1f62fe682 --- /dev/null +++ b/astroquery/mast/tests/data/mission_results.json @@ -0,0 +1,206 @@ +{ + "messages": [], + "info": [ + { + "name": "sci_release_date", + "type": "DATETIME" + }, + { + "name": "sci_actual_duration", + "type": "FLOAT" + }, + { + "name": "sci_dec", + "type": "FLOAT" + }, + { + "name": "sci_pep_id", + "type": "INTEGER" + }, + { + "name": "sci_spec_1234", + "type": "VARCHAR" + }, + { + "name": "sci_aper_1234", + "type": "VARCHAR" + }, + { + "name": "sci_data_set_name", + "type": "VARCHAR" + }, + { + "name": "sci_preview_name", + "type": "VARCHAR" + }, + { + "name": "sci_targname", + "type": "VARCHAR" + }, + { + "name": "sci_instrume", + "type": "VARCHAR" + }, + { + "name": "search_key", + "type": "VARCHAR" + }, + { + "name": "sci_central_wavelength", + "type": "FLOAT" + }, + { + "name": "sci_status", + "type": "VARCHAR" + }, + { + "name": "sci_stop_time", + "type": "DATETIME" + }, + { + "name": "scp_scan_type", + "type": "VARCHAR" + }, + { + "name": "sci_hlsp", + "type": "INTEGER" + }, + { + "name": "sci_refnum", + "type": "INTEGER" + }, + { + "name": "sci_start_time", + "type": "DATETIME" + }, + { + "name": "sci_ra", + "type": "FLOAT" + } +], + "search_params": { + "target": [ + "40.66963 -0.01328" + ], + "radius": 3, + "radius_units": "arcminutes", + "offset": 0, + "limit": 5000, + "sort_by": [ + "ang_sep", + "sci_targname", + "sci_data_set_name" + ], + "sort_desc": [ + false, + false, + false + ], + "skip_count": false, + "select_cols": [ + "sci_data_set_name", + "sci_targname", + "sci_ra", + "sci_dec", + "sci_refnum", + "sci_start_time", + "sci_stop_time", + "sci_actual_duration", + "sci_instrume", + "sci_aper_1234", + "sci_spec_1234", + "sci_central_wavelength", + "sci_pep_id", + "sci_release_date", + "sci_preview_name", + "scp_scan_type", + "sci_hlsp", + "sci_status" + ], + "search_key": [], + "user_fields": [], + "conditions": [ + { + "sci_obs_type": "" + }, + { + "sci_aec": "S" + }, + { + "sci_instrume": "STIS,ACS,WFC3,COS,FGS,FOC,FOS,HRS,HSP,NICMOS,WFPC,WFPC2" + } + ] + }, + "totalResults": 3, + "results": [ + { + "sci_data_set_name": "W1DG8D06T", + "sci_targname": "HI-LAT", + "sci_ra": 40.68202694444444, + "sci_dec": -0.03533861111111111, + "sci_refnum": 18, + "sci_start_time": "1993-09-24T21:00:16.633000", + "sci_stop_time": "1993-09-24T21:35:16.633000", + "sci_actual_duration": 2100, + "sci_instrume": "WFPC ", + "sci_aper_1234": "ALL", + "sci_spec_1234": "F785LP", + "sci_central_wavelength": 8958, + "sci_pep_id": 4381, + "sci_release_date": "1994-09-24T23:03:09", + "sci_preview_name": "W1DG8D06T", + "scp_scan_type": null, + "sci_hlsp": null, + "sci_status": "PUBLIC", + "search_key": "40.66963 -0.01328W1DG8D06T", + "search_pos": "40.66963 -0.01328", + "ang_sep": 1.5182093051117103 + }, + { + "sci_data_set_name": "W1DG9S01T", + "sci_targname": "HI-LAT", + "sci_ra": 40.68202694444444, + "sci_dec": -0.03533861111111111, + "sci_refnum": 18, + "sci_start_time": "1993-09-24T22:57:16.633000", + "sci_stop_time": "1993-09-24T23:17:16.633000", + "sci_actual_duration": 1200, + "sci_instrume": "WFPC ", + "sci_aper_1234": "ALL", + "sci_spec_1234": "F785LP", + "sci_central_wavelength": 8958, + "sci_pep_id": 4381, + "sci_release_date": "1994-09-24T23:51:35", + "sci_preview_name": "W1DG9S01T", + "scp_scan_type": null, + "sci_hlsp": null, + "sci_status": "PUBLIC", + "search_key": "40.66963 -0.01328W1DG9S01T", + "search_pos": "40.66963 -0.01328", + "ang_sep": 1.5182093051117103 + }, + { + "sci_data_set_name": "J8DM01ELQ", + "sci_targname": "NGC1068-OFF", + "sci_ra": 40.71020833333, + "sci_dec": -0.02798888888889, + "sci_refnum": 8, + "sci_start_time": "2003-01-08T19:00:35.987000", + "sci_stop_time": "2003-01-08T19:00:37.833000", + "sci_actual_duration": 0.7955, + "sci_instrume": "ACS ", + "sci_aper_1234": "WFC2-FIX", + "sci_spec_1234": "F550M;CLEAR2L", + "sci_central_wavelength": 5581.3379, + "sci_pep_id": 9503, + "sci_release_date": "2003-07-11T06:24:30", + "sci_preview_name": "J8DM01ELQ", + "scp_scan_type": null, + "sci_hlsp": null, + "sci_status": "PUBLIC", + "search_key": "40.66963 -0.01328J8DM01ELQ", + "search_pos": "40.66963 -0.01328", + "ang_sep": 2.589715886363766 + } + ] +} diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 7f1d9f685f..e94fcdc5b3 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -1,24 +1,27 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +import json import os import re - from shutil import copyfile +import pytest + from astropy.table import Table -from astropy.tests.helper import pytest from astropy.coordinates import SkyCoord from astropy.io import fits import astropy.units as u -from ...utils.testing_tools import MockResponse -from ...exceptions import (InvalidQueryError, InputWarning) +from astroquery.mast.services import _json_to_table +from astroquery.utils.mocks import MockResponse +from astroquery.exceptions import InvalidQueryError, InputWarning -from ... import mast +from astroquery import mast DATA_FILES = {'Mast.Caom.Cone': 'caom.json', 'Mast.Name.Lookup': 'resolver.json', + 'mission_search_results': 'mission_results.json', 'columnsconfig': 'columnsconfig.json', 'ticcolumns': 'ticcolumns.json', 'ticcol_filtered': 'ticcolumns_filtered.json', @@ -102,7 +105,7 @@ def post_mockreturn(self, method="POST", url=None, data=None, timeout=10, **kwar return [MockResponse(content)] -def service_mockreturn(self, method="POST", url=None, data=None, timeout=10, **kwargs): +def service_mockreturn(self, method="POST", url=None, data=None, timeout=10, use_json=False, **kwargs): if "panstarrs" in url: filename = data_path(DATA_FILES["panstarrs"]) elif "tesscut" in url: @@ -115,6 +118,10 @@ def service_mockreturn(self, method="POST", url=None, data=None, timeout=10, **k filename = data_path(DATA_FILES['z_survey']) else: filename = data_path(DATA_FILES['z_cutout_fit']) + elif use_json and data['radius'] == 5: + filename = data_path(DATA_FILES["mission_incorrect_results"]) + elif use_json: + filename = data_path(DATA_FILES["mission_search_results"]) content = open(filename, 'rb').read() return MockResponse(content) @@ -171,6 +178,91 @@ def zcut_download_mockreturn(url, file_path): return +########################### +# MissionSearchClass Test # +########################### + + +def test_missions_query_region_async(patch_post): + responses = mast.MastMissions.query_region_async(regionCoords, radius=0.002, sci_pi_last_name='GORDON') + assert isinstance(responses, MockResponse) + + +def test_missions_query_object_async(patch_post): + responses = mast.MastMissions.query_object_async("M101", radius="0.002 deg") + assert isinstance(responses, MockResponse) + + +def test_missions_query_object(patch_post): + result = mast.MastMissions.query_object("M101", radius=".002 deg") + assert isinstance(result, Table) + assert len(result) > 0 + + +def test_missions_query_region(patch_post): + result = mast.MastMissions.query_region(regionCoords, radius=0.002 * u.deg) + assert isinstance(result, Table) + assert len(result) > 0 + + +def test_missions_query_criteria_async(patch_post): + pep_id = {'sci_pep_id': '12556'} + obs_type = {'sci_obs_type': "SPECTRUM"} + instruments = {'sci_instrume': "stis,acs,wfc3,cos,fos,foc,nicmos,ghrs"} + datasets = {'sci_data_set_name': ""} + pi_lname = {'sci_pi_last_name': ""} + actual_duration = {'sci_actual_duration': ""} + spec_1234 = {'sci_spec_1234': ""} + release_date = {'sci_release_date': ""} + start_time = {'sci_start_time': ""} + obs_type = {'sci_obs_type': 'all'} + aec = {'sci_aec': 'S'} + responses = mast.MastMissions.query_criteria_async(coordinates=regionCoords, + radius=3, + conditions=[pep_id, + obs_type, + instruments, + datasets, + pi_lname, + spec_1234, + release_date, + start_time, + obs_type, + aec]) + assert isinstance(responses, MockResponse) + + +def test_missions_query_criteria_async_with_missing_results(patch_post): + pep_id = {'sci_pep_id': '12556'} + obs_type = {'sci_obs_type': "SPECTRUM"} + instruments = {'sci_instrume': "stis,acs,wfc3,cos,fos,foc,nicmos,ghrs"} + datasets = {'sci_data_set_name': ""} + pi_lname = {'sci_pi_last_name': ""} + actual_duration = {'sci_actual_duration': ""} + spec_1234 = {'sci_spec_1234': ""} + release_date = {'sci_release_date': ""} + start_time = {'sci_start_time': ""} + obs_type = {'sci_obs_type': 'all'} + aec = {'sci_aec': 'S'} + aperture = {'sci_aper_1234': 'WF3'} + + with pytest.raises(KeyError) as e_info: + responses = mast.MastMissions.query_criteria_async(coordinates=regionCoords, + radius=5, + conditions=[pep_id, + obs_type, + instruments, + datasets, + pi_lname, + spec_1234, + release_date, + start_time, + obs_type, + aec, + aperture]) + table = _json_to_table(json.loads(responses), 'results') + + ################### # MastClass tests # ################### @@ -589,6 +681,23 @@ def test_tesscut_get_sector(patch_post): assert sector_table['camera'][0] == 1 assert sector_table['ccd'][0] == 3 + # Exercising the search by moving target + sector_table = mast.Tesscut.get_sectors(objectname="Ceres", + moving_target=True) + assert isinstance(sector_table, Table) + assert len(sector_table) == 1 + assert sector_table['sectorName'][0] == "tess-s0001-1-3" + assert sector_table['sector'][0] == 1 + assert sector_table['camera'][0] == 1 + assert sector_table['ccd'][0] == 3 + + # Testing catch for multiple designators' + error_str = "Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname." + + with pytest.raises(InvalidQueryError) as invalid_query: + mast.Tesscut.get_sectors(objectname='Ceres', moving_target=True, coordinates=coord) + assert error_str in str(invalid_query.value) + def test_tesscut_download_cutouts(patch_post, tmpdir): @@ -616,6 +725,27 @@ def test_tesscut_download_cutouts(patch_post, tmpdir): assert manifest["Local Path"][0][-4:] == "fits" assert os.path.isfile(manifest[0]['Local Path']) + # Exercising the search by moving target + manifest = mast.Tesscut.download_cutouts(objectname="Eleonora", + moving_target=True, + size=5, + path=str(tmpdir)) + assert isinstance(manifest, Table) + assert len(manifest) == 1 + assert manifest["Local Path"][0][-4:] == "fits" + assert os.path.isfile(manifest[0]['Local Path']) + + # Testing catch for multiple designators' + error_str = "Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname." + + with pytest.raises(InvalidQueryError) as invalid_query: + mast.Tesscut.download_cutouts(objectname="Eleonora", + moving_target=True, + coordinates=coord, + size=5, + path=str(tmpdir)) + assert error_str in str(invalid_query.value) + def test_tesscut_get_cutouts(patch_post, tmpdir): @@ -631,6 +761,25 @@ def test_tesscut_get_cutouts(patch_post, tmpdir): assert len(cutout_hdus_list) == 1 assert isinstance(cutout_hdus_list[0], fits.HDUList) + # Exercising the search by object name + cutout_hdus_list = mast.Tesscut.get_cutouts(objectname='Eleonora', + moving_target=True, + size=5) + assert isinstance(cutout_hdus_list, list) + assert len(cutout_hdus_list) == 1 + assert isinstance(cutout_hdus_list[0], fits.HDUList) + + # Testing catch for multiple designators' + error_str = "Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname." + + with pytest.raises(InvalidQueryError) as invalid_query: + mast.Tesscut.get_cutouts(objectname="Eleonora", + moving_target=True, + coordinates=coord, + size=5) + assert error_str in str(invalid_query.value) + + ###################### # ZcutClass tests # ###################### @@ -667,7 +816,7 @@ def test_zcut_download_cutouts(patch_post, tmpdir): # Testing with png cutout_table = mast.Zcut.download_cutouts(coordinates=coord, size=5, - cutout_format="jpg", path=str(tmpdir)) + cutout_format="jpg", path=str(tmpdir)) assert isinstance(cutout_table, Table) assert len(cutout_table) == 3 assert cutout_table["Local Path"][0][-4:] == ".jpg" @@ -675,7 +824,7 @@ def test_zcut_download_cutouts(patch_post, tmpdir): # Testing with img_param cutout_table = mast.Zcut.download_cutouts(coordinates=coord, size=5, - cutout_format="jpg", path=str(tmpdir), invert=True) + cutout_format="jpg", path=str(tmpdir), invert=True) assert isinstance(cutout_table, Table) assert len(cutout_table) == 3 assert cutout_table["Local Path"][0][-4:] == ".jpg" diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 7ad991948b..71578c3585 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -11,9 +11,11 @@ from astropy.io import fits import astropy.units as u -from astroquery.exceptions import NoResultsWarning from astroquery import mast +from ..utils import ResolverError +from ...exceptions import InvalidQueryError, MaxResultsWarning, NoResultsWarning, RemoteServiceError + OBSID = '1647157' @@ -195,10 +197,13 @@ def test_observations_query_criteria_count(self): # product functions def test_observations_get_product_list_async(self): - responses = mast.Observations.get_product_list_async('2003738726') + + test_obs = mast.Observations.query_criteria(filters=["NUV", "FUV"], objectname="M101") + + responses = mast.Observations.get_product_list_async(test_obs[0]["obsid"]) assert isinstance(responses, list) - responses = mast.Observations.get_product_list_async('2003738726,3000007760') + responses = mast.Observations.get_product_list_async(test_obs[2:3]) assert isinstance(responses, list) observations = mast.Observations.query_object("M8", radius=".02 deg") @@ -269,7 +274,7 @@ def test_observations_download_products(self, tmpdir): assert os.path.isfile(row['Local Path']) # just get the curl script - result = mast.Observations.download_products(test_obs_id, + result = mast.Observations.download_products(test_obs[0]["obsid"], download_dir=str(tmpdir), curl_flag=True, productType=["SCIENCE"], @@ -278,21 +283,32 @@ def test_observations_download_products(self, tmpdir): assert os.path.isfile(result['Local Path'][0]) # check for row input - result1 = mast.Observations.get_product_list(test_obs_id) + result1 = mast.Observations.get_product_list(test_obs[0]["obsid"]) result2 = mast.Observations.download_products(result1[0]) assert isinstance(result2, Table) assert os.path.isfile(result2['Local Path'][0]) assert len(result2) == 1 def test_observations_download_file(self, tmpdir): - test_obs_id = OBSID - # pull a single data product - products = mast.Observations.get_product_list(test_obs_id) + # enabling cloud connection + mast.Observations.enable_cloud_dataset(provider='AWS') + + # get observations from GALEX instrument with query_criteria + observations = mast.Observations.query_criteria(objectname='M1', + radius=0.2, + instrument_name='GALEX') + + assert len(observations) > 0, 'No results found for GALEX query.' + + # pull data products from a single observation + products = mast.Observations.get_product_list(observations['obsid'][0]) + + # pull the URI of a single product uri = products['dataURI'][0] # download it - result = mast.Observations.download_file(uri) + result = mast.Observations.download_file(uri, cloud_only=True) assert result == ('COMPLETE', None, None) def test_get_cloud_uri(self): @@ -364,6 +380,10 @@ def test_catalogs_query_region(self): catalog="HSC", magtype=2) row = np.where(result['MatchID'] == '78095437') + + with pytest.warns(MaxResultsWarning): + result = mast.Catalogs.query_region("322.49324 12.16683", catalog="HSC", magtype=2) + assert isinstance(result, Table) assert result[row]['NumImages'] == 1 assert result[row]['TargetName'] == 'M15' @@ -374,6 +394,10 @@ def test_catalogs_query_region(self): version=2, magtype=2) row = np.where(result['MatchID'] == '82368728') + + with pytest.warns(MaxResultsWarning): + result = mast.Catalogs.query_region("322.49324 12.16683", catalog="HSC", + version=2, magtype=2) assert isinstance(result, Table) assert result[row]['NumImages'] == 11 assert result[row]['TargetName'] == 'NGC7078' @@ -491,6 +515,11 @@ def test_catalogs_query_object(self): assert isinstance(result, Table) assert '441662144' in result['ID'] + result = mast.Catalogs.query_object('M1', + radius=0.2, + catalog='plato') + assert 'PICidDR1' in result.colnames + def test_catalogs_query_criteria_async(self): # without position responses = mast.Catalogs.query_criteria_async(catalog="Tic", @@ -666,6 +695,7 @@ def test_tesscut_get_sectors(self): assert sector_table['ccd'][0] > 0 # This should always return no results + with pytest.warns(NoResultsWarning): coord = SkyCoord(90, -66.5, unit="deg") sector_table = mast.Tesscut.get_sectors(coordinates=coord, @@ -673,6 +703,12 @@ def test_tesscut_get_sectors(self): assert isinstance(sector_table, Table) assert len(sector_table) == 0 + coord = SkyCoord(90, -66.5, unit="deg") + with pytest.warns(NoResultsWarning): + sector_table = mast.Tesscut.get_sectors(coordinates=coord, radius=0) + assert isinstance(sector_table, Table) + assert len(sector_table) == 0 + sector_table = mast.Tesscut.get_sectors(objectname="M104") assert isinstance(sector_table, Table) assert len(sector_table) >= 1 @@ -681,6 +717,47 @@ def test_tesscut_get_sectors(self): assert sector_table['camera'][0] > 0 assert sector_table['ccd'][0] > 0 + # Moving target functionality testing + + moving_target_name = 'Eleonora' + + sector_table = mast.Tesscut.get_sectors(objectname=moving_target_name, + moving_target=True) + assert isinstance(sector_table, Table) + assert len(sector_table) >= 1 + assert sector_table['sectorName'][0] == "tess-s0006-1-1" + assert sector_table['sector'][0] == 6 + assert sector_table['camera'][0] == 1 + assert sector_table['ccd'][0] == 1 + + error_noname = "Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet." + error_nameresolve = f"Could not resolve {moving_target_name} to a sky position." + error_mt_coord = "Only one of moving_target and coordinates may be specified." + error_name_coord = "Only one of objectname and coordinates may be specified." + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.get_sectors(moving_target=True) + assert error_noname in str(error_msg.value) + + with pytest.raises(ResolverError) as error_msg: + mast.Tesscut.get_sectors(objectname=moving_target_name) + assert error_nameresolve in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.get_sectors(coordinates=coord, moving_target=True) + assert error_mt_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.get_sectors(objectname=moving_target_name, + coordinates=coord) + assert error_name_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.get_sectors(objectname=moving_target_name, + coordinates=coord, + moving_target=True) + assert error_mt_coord in str(error_msg.value) + def test_tesscut_download_cutouts(self, tmpdir): coord = SkyCoord(349.62609, -47.12424, unit="deg") @@ -720,6 +797,49 @@ def test_tesscut_download_cutouts(self, tmpdir): for row in manifest: assert os.path.isfile(row['Local Path']) + # Moving target functionality testing + + moving_target_name = 'Eleonora' + + manifest = mast.Tesscut.download_cutouts(objectname=moving_target_name, + moving_target=True, + sector=6, + size=5, + path=str(tmpdir)) + assert isinstance(manifest, Table) + assert len(manifest) == 1 + assert manifest["Local Path"][0][-4:] == "fits" + for row in manifest: + assert os.path.isfile(row['Local Path']) + + error_noname = "Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet." + error_nameresolve = f"Could not resolve {moving_target_name} to a sky position." + error_mt_coord = "Only one of moving_target and coordinates may be specified." + error_name_coord = "Only one of objectname and coordinates may be specified." + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(moving_target=True) + assert error_noname in str(error_msg.value) + + with pytest.raises(ResolverError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name) + assert error_nameresolve in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(coordinates=coord, moving_target=True) + assert error_mt_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name, + coordinates=coord) + assert error_name_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name, + coordinates=coord, + moving_target=True) + assert error_mt_coord in str(error_msg.value) + def test_tesscut_get_cutouts(self, tmpdir): coord = SkyCoord(107.18696, -70.50919, unit="deg") @@ -746,9 +866,49 @@ def test_tesscut_get_cutouts(self, tmpdir): assert len(cutout_hdus_list) >= 1 assert isinstance(cutout_hdus_list[0], fits.HDUList) - ###################### + # Moving target functionality testing + + moving_target_name = 'Eleonora' + + cutout_hdus_list = mast.Tesscut.get_cutouts(objectname=moving_target_name, + moving_target=True, + sector=6, + size=5) + assert isinstance(cutout_hdus_list, list) + assert len(cutout_hdus_list) == 1 + assert isinstance(cutout_hdus_list[0], fits.HDUList) + + error_noname = "Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet." + error_nameresolve = f"Could not resolve {moving_target_name} to a sky position." + error_mt_coord = "Only one of moving_target and coordinates may be specified." + error_name_coord = "Only one of objectname and coordinates may be specified." + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(moving_target=True) + assert error_noname in str(error_msg.value) + + with pytest.raises(ResolverError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name) + assert error_nameresolve in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(coordinates=coord, moving_target=True) + assert error_mt_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name, + coordinates=coord) + assert error_name_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name, + coordinates=coord, + moving_target=True) + assert error_mt_coord in str(error_msg.value) + + ################### # ZcutClass tests # - ###################### + ################### def test_zcut_get_surveys(self): coord = SkyCoord(189.49206, 62.20615, unit="deg") @@ -766,6 +926,12 @@ def test_zcut_get_surveys(self): assert isinstance(survey_list, list) assert len(survey_list) == 0 + coord = SkyCoord(57.10523, -30.08085, unit="deg") + with pytest.warns(NoResultsWarning): + survey_list = mast.Zcut.get_surveys(coordinates=coord, radius=0) + assert isinstance(survey_list, list) + assert len(survey_list) == 0 + def test_zcut_download_cutouts(self, tmpdir): coord = SkyCoord(34.47320, -5.24271, unit="deg") @@ -777,7 +943,7 @@ def test_zcut_download_cutouts(self, tmpdir): for row in cutout_table: assert os.path.isfile(cutout_table[0]['Local Path']) - coord = SkyCoord(189.49206, 62.20615, unit="deg") + coord = SkyCoord(189.28065571, 62.17415175, unit="deg") cutout_table = mast.Zcut.download_cutouts(coordinates=coord, size=[200, 300], path=str(tmpdir)) assert isinstance(cutout_table, Table) @@ -809,6 +975,13 @@ def test_zcut_download_cutouts(self, tmpdir): assert isinstance(cutout_table, Table) assert len(cutout_table) == 0 + cutout_table = mast.Zcut.download_cutouts(coordinates=coord, survey='goods_north', cutout_format="jpg", path=str(tmpdir)) + assert isinstance(cutout_table, Table) + assert len(cutout_table) == 4 + assert cutout_table["Local Path"][0][-4:] == ".jpg" + for row in cutout_table: + assert os.path.isfile(cutout_table[0]['Local Path']) + cutout_table = mast.Zcut.download_cutouts(coordinates=coord, cutout_format="jpg", path=str(tmpdir), stretch='asinh', invert=True) assert isinstance(cutout_table, Table) assert len(cutout_table) >= 1 @@ -818,7 +991,7 @@ def test_zcut_download_cutouts(self, tmpdir): def test_zcut_get_cutouts(self): - coord = SkyCoord(189.49206, 62.20615, unit="deg") + coord = SkyCoord(189.28065571, 62.17415175, unit="deg") cutout_list = mast.Zcut.get_cutouts(coordinates=coord) assert isinstance(cutout_list, list) @@ -836,3 +1009,8 @@ def test_zcut_get_cutouts(self): survey='candels_gn_30mas') assert isinstance(cutout_list, list) assert len(cutout_list) == 0 + + cutout_list = mast.Zcut.get_cutouts(coordinates=coord, survey='3dhst_goods-n') + assert isinstance(cutout_list, list) + assert len(cutout_list) == 1 + assert isinstance(cutout_list[0], fits.HDUList) diff --git a/astroquery/mpc/tests/test_mpc.py b/astroquery/mpc/tests/test_mpc.py index 088ce4ca80..bba6ef1487 100644 --- a/astroquery/mpc/tests/test_mpc.py +++ b/astroquery/mpc/tests/test_mpc.py @@ -52,7 +52,7 @@ from ...exceptions import InvalidQueryError from ... import mpc -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from requests import Request diff --git a/astroquery/nasa_ads/tests/test_nasaads.py b/astroquery/nasa_ads/tests/test_nasaads.py index 7db262b44f..5bb58732cb 100644 --- a/astroquery/nasa_ads/tests/test_nasaads.py +++ b/astroquery/nasa_ads/tests/test_nasaads.py @@ -2,7 +2,7 @@ import requests import pytest from ... import nasa_ads -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse class MockResponseADS(MockResponse): diff --git a/astroquery/nist/tests/test_nist.py b/astroquery/nist/tests/test_nist.py index a9f554e157..780194b68d 100644 --- a/astroquery/nist/tests/test_nist.py +++ b/astroquery/nist/tests/test_nist.py @@ -6,7 +6,7 @@ from astropy.table import Table import astropy.units as u -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ... import nist DATA_FILES = {'lines': 'nist_out.html'} diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 71669a865d..5a44e54a22 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -2,21 +2,21 @@ # Python library # External packages +import pytest from astropy import units as u from astropy.coordinates import SkyCoord -from astropy.tests.helper import remote_data + # Local packages from .. import Noirlab from . import expected as expsia -# #!import pytest # performs similar tests as test_module.py, but performs # the actual HTTP request rather than monkeypatching them. # should be disabled or enabled at will - use the -# remote_data decorator from astropy: +# remote_data decorator from pytest-remotedata: -@remote_data +@pytest.mark.remote_data class TestNoirlabClass: def test_query_region_1(self): diff --git a/astroquery/nrao/tests/test_nrao.py b/astroquery/nrao/tests/test_nrao.py index 93a4eab7cd..44696ec2a8 100644 --- a/astroquery/nrao/tests/test_nrao.py +++ b/astroquery/nrao/tests/test_nrao.py @@ -8,7 +8,7 @@ from astropy.table import Table from ... import nrao -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ...utils import commons diff --git a/astroquery/nvas/tests/test_nvas.py b/astroquery/nvas/tests/test_nvas.py index eee964ad79..bd0e5ef2aa 100644 --- a/astroquery/nvas/tests/test_nvas.py +++ b/astroquery/nvas/tests/test_nvas.py @@ -10,7 +10,7 @@ import pytest from ...import nvas -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ...utils import commons COORDS_GAL = commons.GalacticCoordGenerator( diff --git a/astroquery/oac/tests/test_oac.py b/astroquery/oac/tests/test_oac.py index 5ed3bef35c..50633dbe96 100644 --- a/astroquery/oac/tests/test_oac.py +++ b/astroquery/oac/tests/test_oac.py @@ -6,7 +6,7 @@ import astropy.coordinates as coord import astropy.units as u -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from .. import OAC diff --git a/astroquery/ogle/tests/test_ogle.py b/astroquery/ogle/tests/test_ogle.py index cf4f8f9188..dc33cacc77 100644 --- a/astroquery/ogle/tests/test_ogle.py +++ b/astroquery/ogle/tests/test_ogle.py @@ -5,7 +5,7 @@ import pytest from astropy.coordinates import SkyCoord from astropy import units as u -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse DATA_FILES = {'gal_0_3': 'gal_0_3.txt'} diff --git a/astroquery/ogle/tests/test_ogle_remote.py b/astroquery/ogle/tests/test_ogle_remote.py new file mode 100644 index 0000000000..5fc6df8c63 --- /dev/null +++ b/astroquery/ogle/tests/test_ogle_remote.py @@ -0,0 +1,30 @@ + +import pytest +import astropy.units as u +from astropy.coordinates import SkyCoord + +from .. import Ogle + + +@pytest.mark.remote_data +def test_ogle_single(): + co = SkyCoord(0, 3, unit=(u.degree, u.degree), frame='galactic') + response = Ogle.query_region(coord=co) + assert len(response) == 1 + + +@pytest.mark.remote_data +def test_ogle_list(): + co = SkyCoord(0, 3, unit=(u.degree, u.degree), frame='galactic') + co_list = [co, co, co] + response = Ogle.query_region(coord=co_list) + assert len(response) == 3 + assert response['RA[hr]'][0] == response['RA[hr]'][1] == response['RA[hr]'][2] + + +@pytest.mark.remote_data +def test_ogle_list_values(): + co_list = [[0, 0, 0], [3, 3, 3]] + response = Ogle.query_region(coord=co_list) + assert len(response) == 3 + assert response['RA[hr]'][0] == response['RA[hr]'][1] == response['RA[hr]'][2] diff --git a/astroquery/open_exoplanet_catalogue/oec_query.py b/astroquery/open_exoplanet_catalogue/oec_query.py index bd5ce408f8..9d6922bbd2 100644 --- a/astroquery/open_exoplanet_catalogue/oec_query.py +++ b/astroquery/open_exoplanet_catalogue/oec_query.py @@ -19,7 +19,7 @@ def get_catalogue(filepath=None): Parses the Open Exoplanet Catalogue file. Parameters - ----------- + ---------- filepath : str or None if no filepath is given, remote source is used. diff --git a/astroquery/query.py b/astroquery/query.py index 89356dd3dc..ab7198464e 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -4,6 +4,7 @@ import abc import inspect import pickle +import copy import getpass import hashlib import keyring @@ -26,6 +27,10 @@ def to_cache(response, cache_file): log.debug("Caching data to {0}".format(cache_file)) + response = copy.deepcopy(response) + if hasattr(response, 'request'): + for key in tuple(response.request.hooks.keys()): + del response.request.hooks[key] with open(cache_file, "wb") as f: pickle.dump(response, f) @@ -173,8 +178,7 @@ def __init__(self): self.cache_location = os.path.join( paths.get_cache_dir(), 'astroquery', self.__class__.__name__.split("Class")[0]) - if not os.path.exists(self.cache_location): - os.makedirs(self.cache_location) + os.makedirs(self.cache_location, exist_ok=True) self._cache_active = True def __call__(self, *args, **kwargs): diff --git a/astroquery/sdss/core.py b/astroquery/sdss/core.py index 9e636a8648..975e919ee1 100644 --- a/astroquery/sdss/core.py +++ b/astroquery/sdss/core.py @@ -32,7 +32,8 @@ class SDSSClass(BaseQuery): QUERY_URL_SUFFIX_DR_OLD = '/dr{dr}/en/tools/search/x_sql.asp' QUERY_URL_SUFFIX_DR_10 = '/dr{dr}/en/tools/search/x_sql.aspx' QUERY_URL_SUFFIX_DR_NEW = '/dr{dr}/en/tools/search/x_results.aspx' - XID_URL_SUFFIX_OLD = '/dr{dr}/en/tools/crossid/x_crossid.aspx' + XID_URL_SUFFIX_OLD = '/dr{dr}/en/tools/crossid/x_crossid.asp' + XID_URL_SUFFIX_DR_10 = '/dr{dr}/en/tools/crossid/x_crossid.aspx' XID_URL_SUFFIX_NEW = '/dr{dr}/en/tools/search/X_Results.aspx' IMAGING_URL_SUFFIX = ('{base}/dr{dr}/{instrument}/photoObj/frames/' '{rerun}/{run}/{camcol}/' @@ -81,7 +82,7 @@ def query_crossid_async(self, coordinates, obj_names=None, radius : str or `~astropy.units.Quantity` object, optional The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from - `astropy.units` may also be used. Defaults to 2 arcsec. + `astropy.units` may also be used. Defaults to 5 arcsec. timeout : float, optional Time limit (in seconds) for establishing successful connection with remote server. Defaults to `SDSSClass.TIMEOUT`. @@ -123,7 +124,7 @@ def query_crossid_async(self, coordinates, obj_names=None, raise TypeError("radius should be either Quantity or " "convertible to float.") - sql_query = 'SELECT ' + sql_query = 'SELECT\r\n' # Older versions expect the CRLF to be there. if specobj_fields is None: if photoobj_fields is None: @@ -138,13 +139,16 @@ def query_crossid_async(self, coordinates, obj_names=None, photobj_fields.append('p.objID as obj_id') else: photobj_fields = [] - specobj_fields.append('s.objID as obj_id') + specobj_fields.append('s.SpecObjID as obj_id') sql_query += ', '.join(photobj_fields + specobj_fields) sql_query += ',dbo.fPhotoTypeN(p.type) as type \ FROM #upload u JOIN #x x ON x.up_id = u.up_id \ - JOIN PhotoObjAll p ON p.objID = x.objID ORDER BY x.up_id' + JOIN PhotoObjAll p ON p.objID = x.objID ' + if specobj_fields: + sql_query += 'JOIN SpecObjAll s ON p.objID = s.bestObjID ' + sql_query += 'ORDER BY x.up_id' data = "obj_id ra dec \n" data += " \n ".join(['{0} {1} {2}'.format(obj_names[i], @@ -153,7 +157,8 @@ def query_crossid_async(self, coordinates, obj_names=None, for i in range(len(coordinates))]) # firstcol is hardwired, as obj_names is always passed - request_payload = dict(uquery=sql_query, paste=data, + files = {'upload': ('astroquery', data)} + request_payload = dict(uquery=sql_query, firstcol=1, format='csv', photoScope='nearPrim', radius=radius, @@ -163,9 +168,11 @@ def query_crossid_async(self, coordinates, obj_names=None, request_payload['searchtool'] = 'CrossID' if get_query_payload: - return request_payload + return request_payload, files + url = self._get_crossid_url(data_release) - response = self._request("POST", url, params=request_payload, + response = self._request("POST", url, data=request_payload, + files=files, timeout=timeout, cache=cache) return response @@ -1072,8 +1079,10 @@ def _get_query_url(self, data_release): return url def _get_crossid_url(self, data_release): - if data_release < 11: + if data_release < 10: suffix = self.XID_URL_SUFFIX_OLD + elif data_release == 10: + suffix = self.XID_URL_SUFFIX_DR_10 else: suffix = self.XID_URL_SUFFIX_NEW diff --git a/astroquery/sdss/tests/test_sdss.py b/astroquery/sdss/tests/test_sdss.py index 86e1d7a2d1..38b7d1d4cc 100644 --- a/astroquery/sdss/tests/test_sdss.py +++ b/astroquery/sdss/tests/test_sdss.py @@ -11,7 +11,7 @@ import pytest from ... import sdss -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ...exceptions import TimeoutError from ...utils import commons @@ -136,7 +136,9 @@ def url_tester(data_release): def url_tester_crossid(data_release): - if data_release < 11: + if data_release < 10: + baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/crossid/x_crossid.asp' + if data_release == 10: baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/crossid/x_crossid.aspx' if data_release == 11: return diff --git a/astroquery/sdss/tests/test_sdss_remote.py b/astroquery/sdss/tests/test_sdss_remote.py index 1eaa1e9dc6..26251eaf6c 100644 --- a/astroquery/sdss/tests/test_sdss_remote.py +++ b/astroquery/sdss/tests/test_sdss_remote.py @@ -3,7 +3,7 @@ from numpy.testing import assert_allclose import pytest -from astropy import coordinates +from astropy.coordinates import SkyCoord from astropy.table import Table from urllib.error import URLError @@ -12,15 +12,23 @@ from ...exceptions import TimeoutError # DR11 is a quasi-internal data release that does not have SkyServer support. -dr_list = (8, 9, 10, 12, 13, 14, 15, 16) +dr_list = (8, 9, 10, 12, 13, 14, 15, 16, 17) @pytest.mark.remote_data class TestSDSSRemote: # Test Case: A Seyfert 1 galaxy - coords = coordinates.SkyCoord('0h8m05.63s +14d50m23.3s') + coords = SkyCoord('0h8m05.63s +14d50m23.3s') mintimeout = 1e-2 + @pytest.fixture() + def large_results(self): + # Large list of objects for regression tests + query = "select top 1000 z, ra, dec, bestObjID from specObj where class = 'galaxy' and programname = 'eboss'" + results = sdss.SDSS.query_sql(query) + coords_large = SkyCoord(ra=results['ra'], dec=results['dec'], unit='deg') + return coords_large + def test_images_timeout(self): """ This test *must* be run before `test_sdss_image` because that query @@ -166,11 +174,32 @@ def test_query_non_default_field(self): assert query1.colnames == ['r', 'psfMag_r'] assert query2.colnames == ['ra', 'dec', 'r'] - def test_query_crossid(self): - query1 = sdss.SDSS.query_crossid(self.coords) + @pytest.mark.parametrize("dr", dr_list) + def test_query_crossid(self, dr): + query1 = sdss.SDSS.query_crossid(self.coords, data_release=dr) query2 = sdss.SDSS.query_crossid([self.coords, self.coords]) assert isinstance(query1, Table) assert query1['objID'][0] == 1237652943176138868 assert isinstance(query2, Table) assert query2['objID'][0] == query1['objID'][0] == query2['objID'][1] + + @pytest.mark.parametrize("dr", dr_list) + def test_spectro_query_crossid(self, dr): + query1 = sdss.SDSS.query_crossid(self.coords, + specobj_fields=['specObjID', 'z'], + data_release=dr, cache=False) + query2 = sdss.SDSS.query_crossid([self.coords, self.coords], + specobj_fields=['specObjID', 'z'], + data_release=dr, cache=False) + assert isinstance(query1, Table) + assert query1['specObjID'][0] == 845594848269461504 + + assert isinstance(query2, Table) + assert query2['specObjID'][0] == query2['specObjID'][1] == query1['specObjID'][0] + + def test_large_crossid(self, large_results): + # Regression test for #589 + + results = sdss.SDSS.query_crossid(large_results) + assert len(results) == 894 diff --git a/astroquery/simbad/tests/test_simbad.py b/astroquery/simbad/tests/test_simbad.py index ad6a0b4e0c..a5071aa3d1 100644 --- a/astroquery/simbad/tests/test_simbad.py +++ b/astroquery/simbad/tests/test_simbad.py @@ -8,7 +8,7 @@ import numpy as np from ... import simbad -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ...utils import commons from ...query import AstroQuery from ...exceptions import TableParseError diff --git a/astroquery/simbad/tests/test_simbad_remote.py b/astroquery/simbad/tests/test_simbad_remote.py index 3d000fc527..842f97146a 100644 --- a/astroquery/simbad/tests/test_simbad_remote.py +++ b/astroquery/simbad/tests/test_simbad_remote.py @@ -6,7 +6,7 @@ from astropy.coordinates import SkyCoord import astropy.units as u from astropy.table import Table -from astroquery.utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from astroquery.simbad import Simbad # Maybe we need to expose SimbadVOTableResult to be in the public API? from astroquery.simbad.core import SimbadVOTableResult diff --git a/astroquery/skyview/tests/test_skyview.py b/astroquery/skyview/tests/test_skyview.py index 22f5068305..467e0a030a 100644 --- a/astroquery/skyview/tests/test_skyview.py +++ b/astroquery/skyview/tests/test_skyview.py @@ -7,7 +7,7 @@ from astropy import units as u from ...utils import commons -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ...skyview import SkyView objcoords = {'Eta Carinae': coordinates.SkyCoord(ra=161.264775 * u.deg, diff --git a/astroquery/splatalogue/tests/test_splatalogue.py b/astroquery/splatalogue/tests/test_splatalogue.py index 072acf4f9f..974e83e3d4 100644 --- a/astroquery/splatalogue/tests/test_splatalogue.py +++ b/astroquery/splatalogue/tests/test_splatalogue.py @@ -7,7 +7,7 @@ from astropy import units as u from ... import splatalogue -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse SPLAT_DATA = 'CO_colons.csv' diff --git a/astroquery/svo_fps/tests/test_svo_fps.py b/astroquery/svo_fps/tests/test_svo_fps.py index cf928db91a..ed61171a96 100644 --- a/astroquery/svo_fps/tests/test_svo_fps.py +++ b/astroquery/svo_fps/tests/test_svo_fps.py @@ -2,7 +2,7 @@ import os from astropy import units as u -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ..core import SvoFps DATA_FILES = {'filter_index': 'svo_fps_WavelengthEff_min=12000_WavelengthEff_max=12100.xml', diff --git a/astroquery/template_module/tests/test_module.py b/astroquery/template_module/tests/test_module.py index a3bf19b061..f2574589e1 100644 --- a/astroquery/template_module/tests/test_module.py +++ b/astroquery/template_module/tests/test_module.py @@ -18,7 +18,7 @@ import astropy.coordinates as coord import astropy.units as u -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse # finally import the module which is to be tested # and the various configuration items created diff --git a/astroquery/ukidss/tests/test_ukidss.py b/astroquery/ukidss/tests/test_ukidss.py index a46c10782b..bd1fcdc8c3 100644 --- a/astroquery/ukidss/tests/test_ukidss.py +++ b/astroquery/ukidss/tests/test_ukidss.py @@ -9,7 +9,7 @@ from ... import ukidss from ...utils import commons -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ...exceptions import InvalidQueryError DATA_FILES = {"vo_results": "vo_results.html", diff --git a/astroquery/utils/commons.py b/astroquery/utils/commons.py index 98582a0501..223ce8443e 100644 --- a/astroquery/utils/commons.py +++ b/astroquery/utils/commons.py @@ -61,6 +61,10 @@ def FK4CoordGenerator(*args, **kwargs): ASTROPY_LT_5_0 = not minversion('astropy', '5.0') ASTROPY_LT_5_1 = not minversion('astropy', '5.1') +ASTROPY_LT_5_1 = not minversion('astropy', '5.1dev197') +# Update the line above once 5.1 is released +# ASTROPY_LT_5_1 = not minversion('astropy', '5.1') + @deprecated('0.4.4', alternative='astroquery.query.BaseQuery._request') def send_request(url, data, timeout, request_type='POST', headers={}, diff --git a/astroquery/utils/download_file_list.py b/astroquery/utils/download_file_list.py index 8df86a231d..1ab484dea0 100644 --- a/astroquery/utils/download_file_list.py +++ b/astroquery/utils/download_file_list.py @@ -6,6 +6,7 @@ from io import StringIO import astropy.io.fits as fits +from astropy.utils.decorators import deprecated from .commons import get_readable_fileobj __all__ = ['download_list_of_fitsfiles'] @@ -26,6 +27,7 @@ def validify_filename(filestr): return filestr +@deprecated('0.4.5') def download_list_of_fitsfiles(linklist, output_directory=None, output_prefix=None, save=False, overwrite=False, verbose=False, diff --git a/astroquery/utils/mocks.py b/astroquery/utils/mocks.py index 9277bcab68..b1260fe1ae 100644 --- a/astroquery/utils/mocks.py +++ b/astroquery/utils/mocks.py @@ -3,7 +3,7 @@ import json # The MockResponse class is currently relied upon in code and thus -# temporarily got moved out of testing_tools to avoid adding pytest as a +# temporarily got moved here to avoid adding pytest as a # mandatory dependency diff --git a/astroquery/utils/system_tools.py b/astroquery/utils/system_tools.py index 57e3c68b54..220306bfea 100644 --- a/astroquery/utils/system_tools.py +++ b/astroquery/utils/system_tools.py @@ -1,15 +1,8 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +import gzip import os - -# Import DEVNULL for py3 or py3 -try: - from subprocess import DEVNULL -except ImportError: - DEVNULL = open(os.devnull, 'wb') - -# Check availability of some system tools -# Exceptions are raised if not found +import shutil def gunzip(filename): @@ -19,17 +12,13 @@ def gunzip(filename): ---------- filename : str Fully qualified path of the file to decompress. + Returns ------- filename : str Name of the decompressed file (or input filename if gzip is not available). """ - import shutil - import gzip - - # system-wide 'gzip' was removed, Python gzip used instead. - # See #1538 : https://github.com/astropy/astroquery/issues/1538 # ".fz" denotes RICE rather than gzip compression if not filename.endswith('.fz'): @@ -40,21 +29,9 @@ def gunzip(filename): else: return filename + # If there is an update issue of astropy#2793 that got merged, this should # be replaced with it. - def in_ipynb(): - try: - cfg = get_ipython().config - app = cfg['IPKernelApp'] - # ipython 1.0 console has no 'parent_appname', - # but ipynb does - if ('parent_appname' in app and - app['parent_appname'] == 'ipython-notebook'): - return True - else: - return False - except NameError: - # NameError will occur if this is called from python (not ipython) - return False + return 'JPY_PARENT_PID' in os.environ diff --git a/astroquery/utils/tap/conn/tapconn.py b/astroquery/utils/tap/conn/tapconn.py index e041a44c01..9edcf90961 100755 --- a/astroquery/utils/tap/conn/tapconn.py +++ b/astroquery/utils/tap/conn/tapconn.py @@ -24,7 +24,7 @@ import mimetypes import time -from urllib.parse import urlencode +from urllib.parse import urlencode from astroquery.utils.tap.xmlparser import utils from astroquery.utils.tap import taputils @@ -87,7 +87,7 @@ def __init__(self, ishttps, self.__connPort = port self.__connPortSsl = sslport if server_context is not None: - if(server_context.startswith("/")): + if server_context.startswith("/"): self.__serverContext = server_context else: self.__serverContext = f"/{server_context}" @@ -106,8 +106,8 @@ def __init__(self, ishttps, self.__connectionHandler = connhandler def __create_context(self, context): - if (context is not None and context != ""): - if(str(context).startswith("/")): + if context is not None and context != "": + if str(context).startswith("/"): return f"{self.__serverContext}{context}" else: return f"{self.__serverContext}/{context}" @@ -391,6 +391,29 @@ def execute_table_edit(self, data, context = self.__get_table_edit_context() return self.__execute_post(context, data, content_type, verbose) + def execute_table_tool(self, data, + content_type=CONTENT_TYPE_POST_DEFAULT, + verbose=False): + """Executes a POST upload request + The connection is done through HTTP or HTTPS depending on the login + status (logged in -> HTTPS) + + Parameters + ---------- + data : str, mandatory + POST data + content_type: str, optional, default: application/x-www-form-urlencoded + HTTP(s) content-type header value + verbose : bool, optional, default 'False' + flag to display information about the process + + Returns + ------- + An HTTP(s) response object + """ + context = self.__get_table_edit_context() + return self.__execute_post(context, data, content_type, verbose) + def __execute_post(self, context, data, content_type=CONTENT_TYPE_POST_DEFAULT, verbose=False): diff --git a/astroquery/utils/tap/conn/tests/DummyConnHandler.py b/astroquery/utils/tap/conn/tests/DummyConnHandler.py index 6703a29098..11cda51da7 100644 --- a/astroquery/utils/tap/conn/tests/DummyConnHandler.py +++ b/astroquery/utils/tap/conn/tests/DummyConnHandler.py @@ -170,3 +170,9 @@ def execute_table_edit(self, data, verbose=False): return self.__execute_post(subcontext="tableEdit", data=data, content_type=content_type, verbose=verbose) + + def execute_table_tool(self, data, + content_type="application/x-www-form-urlencoded", + verbose=False): + return self.__execute_post(subcontext="TableTool", data=data, + content_type=content_type, verbose=verbose) diff --git a/astroquery/utils/tap/core.py b/astroquery/utils/tap/core.py index b935ebdd5b..6dd273c221 100755 --- a/astroquery/utils/tap/core.py +++ b/astroquery/utils/tap/core.py @@ -11,8 +11,7 @@ European Space Agency (ESA) Created on 30 jun. 2016 - - +Modified on 1 jun. 2021 by mhsarmiento """ from astroquery.utils.tap import taputils from astroquery.utils.tap.conn.tapconn import TapConn @@ -32,7 +31,6 @@ from astropy.table.table import Table import tempfile - __all__ = ['Tap', 'TapPlus'] VERSION = "20200428.1" @@ -246,6 +244,8 @@ def launch_job(self, query, name=None, output_file=None, ---------- query : str, mandatory query to be executed + name : str, optional, default None + custom name defined by the user for the job that is going to be created output_file : str, optional, default None file name where the results are saved if dumpToFile is True. If this parameter is not provided, the jobid is used instead @@ -265,6 +265,8 @@ def launch_job(self, query, name=None, output_file=None, ------- A Job object """ + output_file_updated = taputils.get_suitable_output_file_name_for_current_output_format(output_file, + output_format) query = taputils.set_top_in_query(query, 2000) if verbose: print(f"Launched query: '{query}'") @@ -310,7 +312,7 @@ def launch_job(self, query, name=None, output_file=None, headers = response.getheaders() suitableOutputFile = taputils.get_suitable_output_file(self.__connHandler, False, - output_file, + output_file_updated, headers, isError, output_format) @@ -358,6 +360,8 @@ def launch_job_async(self, query, name=None, output_file=None, ---------- query : str, mandatory query to be executed + name : str, optional, default None + custom name defined by the user for the job that is going to be created output_file : str, optional, default None file name where the results are saved if dumpToFile is True. If this parameter is not provided, the jobid is used instead @@ -383,6 +387,10 @@ def launch_job_async(self, query, name=None, output_file=None, ------- A Job object """ + + output_file_updated = taputils.get_suitable_output_file_name_for_current_output_format(output_file, + output_format) + if verbose: print(f"Launched query: '{query}'") if upload_resource is not None: @@ -412,7 +420,7 @@ def launch_job_async(self, query, name=None, output_file=None, headers = response.getheaders() suitableOutputFile = taputils.get_suitable_output_file(self.__connHandler, True, - output_file, + output_file_updated, headers, isError, output_format) @@ -571,7 +579,7 @@ def __launchJobMultipart(self, query, uploadResource, uploadTableName, "FORMAT": str(outputFormat), "tapclient": str(TAP_CLIENT_ID), "QUERY": str(query), - "UPLOAD": ""+str(uploadValue)} + "UPLOAD": "" + str(uploadValue)} if autorun is True: args['PHASE'] = 'RUN' if name is not None: @@ -657,7 +665,7 @@ def __parseUrl(self, url, verbose=False): if urlInfoPos < 0: raise ValueError("Invalid URL format") - urlInfo = url[(urlInfoPos+3):] + urlInfo = url[(urlInfoPos + 3):] items = urlInfo.split("/") @@ -672,7 +680,7 @@ def __parseUrl(self, url, verbose=False): if portPos > 0: # port found host = hostPort[0:portPos] - port = int(hostPort[portPos+1:]) + port = int(hostPort[portPos + 1:]) else: # no port found host = hostPort @@ -693,10 +701,10 @@ def __parseUrl(self, url, verbose=False): tapContext = f"/{items[2]}" else: data = [] - for i in range(1, itemsSize-1): + for i in range(1, itemsSize - 1): data.append(f"/{items[i]}") serverContext = utils.util_create_string_from_buffer(data) - tapContext = f"/{items[itemsSize-1]}" + tapContext = f"/{items[itemsSize - 1]}" if verbose: print(f"protocol: '{protocol}'") print(f"host: '{host}'") @@ -713,6 +721,7 @@ class TapPlus(Tap): """TAP plus class Provides TAP and TAP+ capabilities """ + def __init__(self, url=None, host=None, server_context=None, @@ -1245,7 +1254,7 @@ def get_datalinks(self, ids, verbose=False): print("Retrieving datalink.") if ids is None: raise ValueError("Missing mandatory argument 'ids'") - if isinstance(ids, str): + if isinstance(ids, str): ids_arg = f"ID={ids}" else: if isinstance(ids, int): @@ -1368,7 +1377,7 @@ def login(self, user=None, password=None, credentials_file=None, user = ins.readline().strip() password = ins.readline().strip() if user is None: - user = input("User: ") + user = input("User: ") if user is None: print("Invalid user name") return @@ -1532,7 +1541,7 @@ def __uploadTableMultipart(self, resource, table_name=None, chunk = f.read() files = [['FILE', os.path.basename(resource), chunk]] contentType, body = connHandler.encode_multipart(args, files) - else: # upload from URL + else: # upload from URL args = { "TASKID": str(-1), "TABLE_NAME": str(table_name), @@ -1628,14 +1637,14 @@ def delete_user_table(self, table_name=None, force_removal=False, raise ValueError("Table name cannot be null") if force_removal is True: args = { - "TABLE_NAME": str(table_name), - "DELETE": "TRUE", - "FORCE_REMOVAL": "TRUE"} + "TABLE_NAME": str(table_name), + "DELETE": "TRUE", + "FORCE_REMOVAL": "TRUE"} else: args = { - "TABLE_NAME": str(table_name), - "DELETE": "TRUE", - "FORCE_REMOVAL": "FALSE"} + "TABLE_NAME": str(table_name), + "DELETE": "TRUE", + "FORCE_REMOVAL": "FALSE"} connHandler = self.__getconnhandler() data = connHandler.url_encode(args) response = connHandler.execute_upload(data, verbose=verbose) @@ -1648,15 +1657,90 @@ def delete_user_table(self, table_name=None, force_removal=False, msg = f"Table '{table_name}' deleted." print(msg) + def rename_table(self, table_name=None, new_table_name=None, new_column_names_dict=None, + verbose=False): + """ This method allows you to update the column names of a user table. + + Parameters + ---------- + table_name: str, required + old name of the user's table + new_table_name: str, required + new name of the user's table + new_column_names_dict: dict str:str + dict with pairs "old_column1_name:new_column1_name" + verbose : bool, optional, default 'False' + flag to display information about the process + + """ + if new_column_names_dict is None: + new_column_names_dict = {} + args = {} + + if table_name is None: + raise ValueError( + "Argument 'table_name' is mandatory. " + "Please introduce the name of the table that is going to be updated") + if (new_table_name is None) and (new_column_names_dict is None): + raise ValueError("Please introduce as minimum a new name for the table or a new name for a column with " + "format old_column1_name:new_column1_name, ... ,old_columnN_name:new_columnN_name") + if new_table_name is not None: + if new_column_names_dict is None: + # case 1: We only need to rename the table + args = self.get_args_4_rename_table(table_name, new_table_name) + else: + # case 2: We need to rename both, table and column names + args = self.get_args_4_rename_table(table_name, new_table_name, new_column_names_dict) + + if new_table_name is None: + if new_column_names_dict: + # case 3: We only need to rename the columns but same table name + args = self.get_args_4_rename_table(table_name, table_name, new_column_names_dict) + + connHandler = self.__getconnhandler() + data = connHandler.url_encode(args) + response = connHandler.execute_table_tool(data, verbose=verbose) + + if verbose: + print(response.status, response.reason) + print(response.getheaders()) + connHandler.check_launch_response_status(response, + verbose, + 200) + if verbose: + msg = f"Table '{table_name}' updated." + print(msg) + + def get_args_4_rename_table(self, table_name, new_table_name, new_column_names_dict): + + args = {} + + if not new_column_names_dict: + args = { + "action": "rename", + "new_table_name": new_table_name, + "table_name": table_name + } + else: + new_column_names = ','.join(f'{key}:{value}' for key, value in new_column_names_dict.items()) + + args = { + "action": "rename", + "new_column_names": new_column_names, + "new_table_name": new_table_name, + "table_name": table_name + } + return args + def update_user_table(self, table_name=None, list_of_changes=[], verbose=False): """Updates a user table Parameters ---------- - table_name : str, required + table_name : str table to be updated - list_of_changes : list, required + list_of_changes : list list of lists, each one of them containing sets of [column_name, field_name, value]. column_name is the name of the column to be updated @@ -1738,11 +1822,11 @@ def update_user_table(self, table_name=None, list_of_changes=[], def get_table_update_arguments(table_name, columns, list_of_changes): num_cols = len(columns) args = { - "ACTION": "edit", - "NUMTABLES": str(1), - "TABLE0_NUMCOLS": str(num_cols), - "TABLE0": str(table_name), - } + "ACTION": "edit", + "NUMTABLES": str(1), + "TABLE0_NUMCOLS": str(num_cols), + "TABLE0": str(table_name), + } index = 0 for column in columns: found_in_changes = False @@ -1872,11 +1956,11 @@ def set_ra_dec_columns(self, table_name=None, Parameters ---------- - table_name : str, required + table_name : str table to be set - ra_column_name : str, required + ra_column_name : str ra column to be set - dec_column_name : str, required + dec_column_name : str dec column to be set verbose : bool, optional, default 'False' flag to display information about the process @@ -1890,11 +1974,11 @@ def set_ra_dec_columns(self, table_name=None, raise ValueError("Dec column name cannot be null") args = { - "ACTION": "radec", - "TABLE_NAME": str(table_name), - "RA": str(ra_column_name), - "DEC": str(dec_column_name), - } + "ACTION": "radec", + "TABLE_NAME": str(table_name), + "RA": str(ra_column_name), + "DEC": str(dec_column_name), + } connHandler = self.__getconnhandler() data = connHandler.url_encode(args) response = connHandler.execute_table_edit(data, verbose=verbose) @@ -1933,7 +2017,7 @@ def login(self, user=None, password=None, credentials_file=None, user = ins.readline().strip() password = ins.readline().strip() if user is None: - user = input("User: ") + user = input("User: ") if user is None: log.info("Invalid user name") return diff --git a/astroquery/utils/tap/model/job.py b/astroquery/utils/tap/model/job.py index 4e74fb09c5..8dc1eb1986 100755 --- a/astroquery/utils/tap/model/job.py +++ b/astroquery/utils/tap/model/job.py @@ -299,7 +299,8 @@ def save_results(self, verbose=False): output = self.outputFile else: output = self.outputFileUser - print(f"Saving results to: {output}") + if verbose: + print(f"Saving results to: {output}") self.connHandler.dump_to_file(output, response) def wait_for_job_end(self, verbose=False): diff --git a/astroquery/utils/tap/model/modelutils.py b/astroquery/utils/tap/model/modelutils.py index b313edfc39..5acdf71677 100644 --- a/astroquery/utils/tap/model/modelutils.py +++ b/astroquery/utils/tap/model/modelutils.py @@ -14,9 +14,12 @@ """ + import os from astropy.table import Table as APTable -from astropy import units as u +from astropy import units as u, io + +from astroquery.utils.tap.xmlparser import utils def check_file_exists(file_name): @@ -28,8 +31,13 @@ def check_file_exists(file_name): def read_results_table_from_file(file_name, output_format, correct_units=True): + + astropy_format = utils.get_suitable_astropy_format(output_format) + if check_file_exists(file_name): - result = APTable.read(file_name, format=output_format) + + result = APTable.read(file_name, format=astropy_format) + if correct_units: for cn in result.colnames: col = result[cn] diff --git a/astroquery/utils/tap/model/tests/test_job.py b/astroquery/utils/tap/model/tests/test_job.py index 6fc9e50863..1e01f5e8a1 100644 --- a/astroquery/utils/tap/model/tests/test_job.py +++ b/astroquery/utils/tap/model/tests/test_job.py @@ -35,7 +35,7 @@ def test_job_basic(): job.get_results() -def test_job_get_results(): +def test_job_get_results(capsys, tmpdir): job = Job(async_job=True) jobid = "12345" outputFormat = "votable" @@ -82,6 +82,14 @@ def test_job_get_results(): if cn not in res.colnames: pytest.fail(f"{cn} column name not found: {res.colnames}") + # Regression test for #2299; messages were printed even with `verbose=False` + capsys.readouterr() + job._Job__resultInMemory = False + job.save_results(verbose=False) + assert 'Saving results to:' not in capsys.readouterr().out + job.save_results(verbose=True) + assert 'Saving results to:' in capsys.readouterr().out + def test_job_phase(): job = Job(async_job=True) diff --git a/astroquery/utils/tap/taputils.py b/astroquery/utils/tap/taputils.py index dc7f14acf2..feeb01356d 100644 --- a/astroquery/utils/tap/taputils.py +++ b/astroquery/utils/tap/taputils.py @@ -16,6 +16,7 @@ """ import re +import warnings from datetime import datetime TAP_UTILS_QUERY_TOP_PATTERN = re.compile( @@ -101,7 +102,7 @@ def set_top_in_query(query, top): else: # no all nor distinct: add top after select p = q.replace("\n", " ").find("SELECT ") - nq = f"{query[0:p+7]} TOP {top} {query[p+7:]}" + nq = f"{query[0:p + 7]} TOP {top} {query[p + 7:]}" return nq @@ -140,7 +141,7 @@ def parse_http_response_error(responseStr, status): pos2 = responseStr.find('', pos1) if pos2 == -1: return parse_http_votable_response_error(responseStr, status) - msg = responseStr[(pos1+len(TAP_UTILS_HTTP_ERROR_MSG_START)):pos2] + msg = responseStr[(pos1 + len(TAP_UTILS_HTTP_ERROR_MSG_START)):pos2] return f"Error {status}:\n{msg}" @@ -162,7 +163,7 @@ def parse_http_votable_response_error(responseStr, status): pos2 = responseStr.find(TAP_UTILS_VOTABLE_INFO, pos1) if pos2 == -1: return f"Error {status}:\n{responseStr}" - msg = responseStr[(pos1+len(TAP_UTILS_HTTP_VOTABLE_ERROR)):pos2] + msg = responseStr[(pos1 + len(TAP_UTILS_HTTP_VOTABLE_ERROR)):pos2] return f"Error {status}: {msg}" @@ -178,7 +179,7 @@ def get_jobid_from_location(location): ------- A jobid. """ - pos = location.rfind('/')+1 + pos = location.rfind('/') + 1 jobid = location[pos:] return jobid @@ -217,26 +218,75 @@ def get_table_name(full_qualified_table_name): pos = full_qualified_table_name.rfind('.') if pos == -1: return full_qualified_table_name - name = full_qualified_table_name[pos+1:] + name = full_qualified_table_name[pos + 1:] return name -def get_suitable_output_file(conn_handler, async_job, outputFile, headers, - isError, output_format): - dateTime = datetime.now().strftime("%Y%m%d%H%M%S") - fileName = "" - if outputFile is None: - fileName = conn_handler.get_file_from_header(headers) - if fileName is None: +def get_suitable_output_file(conn_handler, async_job, output_file, headers, + is_error, output_format): + date_time = datetime.now().strftime("%Y%m%d%H%M%S") + if output_file is None: + file_name = conn_handler.get_file_from_header(headers) + if file_name is None: ext = conn_handler.get_suitable_extension(headers) if not async_job: - fileName = f"sync_{dateTime}{ext}" + file_name = f"sync_{date_time}{ext}" else: ext = conn_handler.get_suitable_extension_by_format( output_format) - fileName = f"async_{dateTime}{ext}" + file_name = f"async_{date_time}{ext}" else: - fileName = outputFile - if isError: - fileName += ".error" - return fileName + file_name = output_file + if is_error: + file_name += ".error" + return file_name + + +def get_suitable_output_file_name_for_current_output_format(output_file, output_format): + """ renames the name given for the output_file if the results for current_output format are returned compressed by default + and the name selected by the user does not contain the correct extension. + + output_file : str, optional, default None + file name selected by the user + output_format : str, optional, default 'votable' + results format. Available formats in TAP are: 'votable', 'votable_plain', + 'fits', 'csv', 'ecsv' and 'json'. Default is 'votable'. + Returned results for formats 'votable' 'ecsv' and 'fits' are compressed + gzip files. + + Returns + ------- + A string with the new name for the file. + """ + compressed_extension = ".gz" + format_with_results_compressed = ['votable', 'fits', 'ecsv'] + output_file_with_extension = output_file + + if output_file is not None: + if output_format in format_with_results_compressed: + # In this case we will have to take also into account the .fits format + if not output_file.endswith(compressed_extension): + warnings.warn('By default, results in "votable", "ecsv" and "fits" format are returned in ' + f'compressed format therefore your file {output_file} ' + f'will be renamed to {output_file}.gz') + if output_format == 'votable': + if output_file.endswith('.vot'): + output_file_with_extension = output_file + '.gz' + else: + output_file_with_extension = output_file + '.vot.gz' + elif output_format == 'fits': + if output_file.endswith('.fits'): + output_file_with_extension = output_file + '.gz' + else: + output_file_with_extension = output_file + '.fits.gz' + elif output_format == 'ecsv': + if output_file.endswith('.ecsv'): + output_file_with_extension = output_file + '.gz' + else: + output_file_with_extension = output_file + '.ecsv.gz' + # the output type is not compressed by default by the TAP SERVER but the users gives a .gz extension + elif output_file.endswith(compressed_extension): + output_file_renamed = output_file.removesuffix('.gz') + warnings.warn(f'The output format selected is not compatible with compression. {output_file}' + f' will be renamed to {output_file_renamed}') + return output_file_with_extension diff --git a/astroquery/utils/tap/tests/data/1645110907148I-result.fits.gz b/astroquery/utils/tap/tests/data/1645110907148I-result.fits.gz new file mode 100644 index 0000000000..3e3d85884d Binary files /dev/null and b/astroquery/utils/tap/tests/data/1645110907148I-result.fits.gz differ diff --git a/astroquery/utils/tap/tests/data/job_1.vot b/astroquery/utils/tap/tests/data/job_1.vot index 695342347d..d73db0eb8d 100644 --- a/astroquery/utils/tap/tests/data/job_1.vot +++ b/astroquery/utils/tap/tests/data/job_1.vot @@ -1,14 +1,14 @@ - + - -alpha + +ra - -delta + +dec source_id diff --git a/astroquery/utils/tap/tests/data/table_test_rename.vot b/astroquery/utils/tap/tests/data/table_test_rename.vot new file mode 100644 index 0000000000..a89ff45966 --- /dev/null +++ b/astroquery/utils/tap/tests/data/table_test_rename.vot @@ -0,0 +1,29 @@ + + + + + +
+ +ra + + +dec + + +source_id + + +object_id + + + + +AD/wAAAAAAAAQAAAAAAAAAAAAAABYQAAAAEAQAgAAAAAAABAEAAAAAAAAAAAAAFi +AAAAAgBAFAAAAAAAAEAYAAAAAAAAAAAAAWMAAAAD + + + +
+ + diff --git a/astroquery/utils/tap/tests/data/test_table_rename.xml b/astroquery/utils/tap/tests/data/test_table_rename.xml new file mode 100644 index 0000000000..e40ab58bc4 --- /dev/null +++ b/astroquery/utils/tap/tests/data/test_table_rename.xml @@ -0,0 +1,52 @@ + + + user_mhenar + + + user_test.table1 + + + + + table1_oid + + + + + + + INTEGER + indexed + primary + + + source_id + + + None + None + VARCHAR + + + ra + + + + + DOUBLE + + + dec + + + + + DOUBLE + +
+ + diff --git a/astroquery/utils/tap/tests/test_tap.py b/astroquery/utils/tap/tests/test_tap.py index 03398154e8..5e1818dd0d 100644 --- a/astroquery/utils/tap/tests/test_tap.py +++ b/astroquery/utils/tap/tests/test_tap.py @@ -11,12 +11,14 @@ European Space Agency (ESA) Created on 30 jun. 2016 - - """ import os +from unittest.mock import patch + import numpy as np import pytest +from requests import HTTPError + from astroquery.utils.tap.model.tapcolumn import TapColumn from astroquery.utils.tap.conn.tests.DummyConnHandler import DummyConnHandler @@ -179,7 +181,7 @@ def test_launch_sync_job(): dTmp = {"q": query} dTmpEncoded = connHandler.url_encode(dTmp) p = dTmpEncoded.find("=") - q = dTmpEncoded[p+1:] + q = dTmpEncoded[p + 1:] dictTmp = { "REQUEST": "doQuery", "LANG": "ADQL", @@ -207,13 +209,13 @@ def test_launch_sync_job(): results = job.get_results() assert len(results) == 3 __check_results_column(results, - 'alpha', - 'alpha', + 'ra', + 'ra', None, np.float64) __check_results_column(results, - 'delta', - 'delta', + 'dec', + 'dec', None, np.float64) __check_results_column(results, @@ -238,8 +240,8 @@ def test_launch_sync_job_redirect(): resultsReq = f'sync/{jobid}' resultsLocation = f'http://test:1111/tap/{resultsReq}' launchResponseHeaders = [ - ['location', resultsLocation] - ] + ['location', resultsLocation] + ] responseLaunchJob.set_data(method='POST', context=None, body=None, @@ -248,7 +250,7 @@ def test_launch_sync_job_redirect(): dTmp = {"q": query} dTmpEncoded = connHandler.url_encode(dTmp) p = dTmpEncoded.find("=") - q = dTmpEncoded[p+1:] + q = dTmpEncoded[p + 1:] dictTmp = { "REQUEST": "doQuery", "LANG": "ADQL", @@ -310,13 +312,13 @@ def test_launch_sync_job_redirect(): results = job.get_results() assert len(results) == 3 __check_results_column(results, - 'alpha', - 'alpha', + 'ra', + 'ra', None, np.float64) __check_results_column(results, - 'delta', - 'delta', + 'dec', + 'dec', None, np.float64) __check_results_column(results, @@ -341,8 +343,8 @@ def test_launch_async_job(): responseLaunchJob.set_message("ERROR") # list of list (httplib implementation for headers in response) launchResponseHeaders = [ - ['location', f'http://test:1111/tap/async/{jobid}'] - ] + ['location', f'http://test:1111/tap/async/{jobid}'] + ] responseLaunchJob.set_data(method='POST', context=None, body=None, @@ -406,13 +408,13 @@ def test_launch_async_job(): results = job.get_results() assert len(results) == 3 __check_results_column(results, - 'alpha', - 'alpha', + 'ra', + 'ra', None, np.float64) __check_results_column(results, - 'delta', - 'delta', + 'dec', + 'dec', None, np.float64) __check_results_column(results, @@ -447,8 +449,8 @@ def test_start_job(): responseLaunchJob.set_message("OK") # list of list (httplib implementation for headers in response) launchResponseHeaders = [ - ['location', f'http://test:1111/tap/async/{jobid}'] - ] + ['location', f'http://test:1111/tap/async/{jobid}'] + ] responseLaunchJob.set_data(method='POST', context=None, body=None, @@ -525,8 +527,8 @@ def test_abort_job(): responseLaunchJob.set_message("OK") # list of list (httplib implementation for headers in response) launchResponseHeaders = [ - ['location', f'http://test:1111/tap/async/{jobid}'] - ] + ['location', f'http://test:1111/tap/async/{jobid}'] + ] responseLaunchJob.set_data(method='POST', context=None, body=None, @@ -563,8 +565,8 @@ def test_job_parameters(): responseLaunchJob.set_message("OK") # list of list (httplib implementation for headers in response) launchResponseHeaders = [ - ['location', f'http://test:1111/tap/async/{jobid}'] - ] + ['location', f'http://test:1111/tap/async/{jobid}'] + ] responseLaunchJob.set_data(method='POST', context=None, body=None, @@ -955,6 +957,45 @@ def test_update_user_table(): tap.update_user_table(table_name=tableName, list_of_changes=list_of_changes) +def test_rename_table(): + tableName = 'user_test.table_test_rename' + newTableName = 'user_test.table_test_rename_new' + newColumnNames = {'ra': 'alpha', 'dec': 'delta'} + connHandler = DummyConnHandler() + tap = TapPlus("http://test:1111/tap", connhandler=connHandler) + dummyResponse = DummyResponse() + dummyResponse.set_status_code(200) + dummyResponse.set_message("OK") + tableDataFile = data_path('test_table_rename.xml') + tableData = utils.read_file_content(tableDataFile) + dummyResponse.set_data(method='GET', + context=None, + body=tableData, + headers=None) + + with pytest.raises(Exception): + tap.rename_table() + with pytest.raises(Exception): + tap.rename_table(table_name=tableName) + with pytest.raises(Exception): + tap.rename_table(table_name=tableName, new_table_name=None, new_column_names_dict=None) + + # Test OK. + responseRenameTable = DummyResponse() + responseRenameTable.set_status_code(200) + responseRenameTable.set_message("OK") + dictArgs = { + "action": "rename", + "new_column_names": "ra:alpha,dec:delta", + "new_table_name": newTableName, + "table_name": tableName, + } + data = connHandler.url_encode(dictArgs) + req = f"TableTool?{data}" + connHandler.set_response(req, responseRenameTable) + tap.rename_table(table_name=tableName, new_table_name=newTableName, new_column_names_dict=newColumnNames) + + def __find_table(schemaName, tableName, tables): qualifiedName = f"{schemaName}.{tableName}" for table in (tables): @@ -985,3 +1026,40 @@ def __check_results_column(results, columnName, description, unit, assert c.description == description assert c.unit == unit assert c.dtype == dataType + + +@patch.object(TapPlus, 'login') +def test_login(mock_login): + conn_handler = DummyConnHandler() + tap = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap.login("user", "password") + assert (mock_login.call_count == 1) + mock_login.side_effect = HTTPError("Login error") + with pytest.raises(HTTPError): + tap.login("user", "password") + assert (mock_login.call_count == 2) + + +@patch.object(TapPlus, 'login_gui') +@patch.object(TapPlus, 'login') +def test_login_gui(mock_login_gui, mock_login): + conn_handler = DummyConnHandler() + tap = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap.login_gui() + assert (mock_login_gui.call_count == 0) + mock_login_gui.side_effect = HTTPError("Login error") + with pytest.raises(HTTPError): + tap.login("user", "password") + assert (mock_login.call_count == 1) + + +@patch.object(TapPlus, 'logout') +def test_logout(mock_logout): + conn_handler = DummyConnHandler() + tap = TapPlus("http://test:1111/tap", connhandler=conn_handler) + tap.logout() + assert (mock_logout.call_count == 1) + mock_logout.side_effect = HTTPError("Login error") + with pytest.raises(HTTPError): + tap.logout() + assert (mock_logout.call_count == 2) diff --git a/astroquery/utils/tap/xmlparser/utils.py b/astroquery/utils/tap/xmlparser/utils.py index 76e6a7355a..526b86be04 100644 --- a/astroquery/utils/tap/xmlparser/utils.py +++ b/astroquery/utils/tap/xmlparser/utils.py @@ -14,7 +14,7 @@ """ - +import gzip import io from astropy import units as u from astropy.table import Table as APTable @@ -24,11 +24,18 @@ def util_create_string_from_buffer(buffer): return ''.join(map(str, buffer)) -def read_http_response(response, outputFormat, correct_units=True): - astropyFormat = get_suitable_astropy_format(outputFormat) +def read_http_response(response, output_format, correct_units=True): + astropy_format = get_suitable_astropy_format(output_format) + # If we want to use astropy.table, we have to read the data data = io.BytesIO(response.read()) - result = APTable.read(data, format=astropyFormat) + + try: + result = APTable.read(io.BytesIO(gzip.decompress(data.read())), format=astropy_format) + except OSError: + # data is not a valid gzip file by BadGzipFile. + result = APTable.read(data, format=astropy_format) + pass if correct_units: for cn in result.colnames: @@ -44,14 +51,18 @@ def read_http_response(response, outputFormat, correct_units=True): return result -def get_suitable_astropy_format(outputFormat): - if "csv" == outputFormat: - return "ascii.csv" - return outputFormat +def get_suitable_astropy_format(output_format): + if 'ecsv' == output_format: + return 'ascii.ecsv' + elif 'csv' == output_format: + return 'ascii.csv' + elif 'votable_plain' == output_format: + return 'votable' + return output_format -def read_file_content(filePath): - fileHandler = open(filePath, 'r') - fileContent = fileHandler.read() - fileHandler.close() - return fileContent +def read_file_content(file_path): + file_handler = open(file_path, 'r') + file_content = file_handler.read() + file_handler.close() + return file_content diff --git a/astroquery/utils/testing_tools.py b/astroquery/utils/testing_tools.py deleted file mode 100644 index d950399c32..0000000000 --- a/astroquery/utils/testing_tools.py +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst - -import socket - -import pytest - -# Import MockResponse to keep the API while it's temporarily factored out to -# a separate file to avoid requiring pytest as a dependency in non-test code -from .mocks import MockResponse - -# save original socket method for restoration -socket_original = socket.socket - - -@pytest.fixture -def turn_off_internet(verbose=False): - __tracebackhide__ = True - if verbose: - print("Internet access disabled") - - def guard(*args, **kwargs): - pytest.fail("An attempt was made to connect to the internet") - setattr(socket, 'socket', guard) - return socket - - -@pytest.fixture -def turn_on_internet(verbose=False): - if verbose: - print("Internet access enabled") - setattr(socket, 'socket', socket_original) - return socket diff --git a/astroquery/utils/tests/test_cache_remote.py b/astroquery/utils/tests/test_cache_remote.py new file mode 100644 index 0000000000..0c8a4ec996 --- /dev/null +++ b/astroquery/utils/tests/test_cache_remote.py @@ -0,0 +1,27 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest +import shutil +import tempfile + +from astroquery.mpc import MPC + + +@pytest.mark.remote_data +class TestRandomThings: + + @pytest.fixture() + def temp_dir(self, request): + my_temp_dir = tempfile.mkdtemp() + + def fin(): + shutil.rmtree(my_temp_dir) + request.addfinalizer(fin) + return my_temp_dir + + def test_quantity_hooks_cache(self, temp_dir): + # Regression test for #2294 + mpc = MPC() + mpc.cache_location = temp_dir + mpc.get_observations(12893, cache=True) + mpc.get_observations(12894, cache=True) diff --git a/astroquery/utils/tests/test_download_file_list.py b/astroquery/utils/tests/test_download_file_list.py new file mode 100644 index 0000000000..9bf03861fc --- /dev/null +++ b/astroquery/utils/tests/test_download_file_list.py @@ -0,0 +1,10 @@ +import pytest + +from astropy.utils.exceptions import AstropyDeprecationWarning + +from astroquery.utils.download_file_list import download_list_of_fitsfiles + + +def test_download_list_of_fitsfiles_deprecation(): + with pytest.warns(AstropyDeprecationWarning): + download_list_of_fitsfiles([]) diff --git a/astroquery/utils/tests/test_system_tools.py b/astroquery/utils/tests/test_system_tools.py index addcc14f6b..32e98d6e58 100644 --- a/astroquery/utils/tests/test_system_tools.py +++ b/astroquery/utils/tests/test_system_tools.py @@ -1,44 +1,18 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -try: - import gzip - - HAS_GZIP = True -except ImportError: - HAS_GZIP = False - -import shutil -import os -from os.path import exists -import tempfile - -import pytest +import gzip from ..system_tools import gunzip -@pytest.mark.skipif("not HAS_GZIP") -def test_gunzip(): - - temp_dir = tempfile.mkdtemp() - filename = f"{temp_dir}{os.sep}test_gunzip.txt.gz" - unziped_filename = filename.rsplit(".", 1)[0] - +def test_gunzip(tmp_path): + filename = tmp_path / 'test_gunzip.txt.gz' # First create a gzip file content = b"Bla" with gzip.open(filename, "wb") as f: f.write(content) - - try: - # Then test our gunzip command works and creates an unziped file - gunzip(filename) - assert exists(unziped_filename) - - # Check content is the same - with open(unziped_filename, "rb") as f: - new_content = f.read() - assert new_content == content - - finally: - # Clean - shutil.rmtree(temp_dir) + # Then test our gunzip command works + gunzip(str(filename)) + with open(filename.with_suffix(''), "rb") as f: + new_content = f.read() + assert new_content == content diff --git a/astroquery/vizier/tests/test_vizier.py b/astroquery/vizier/tests/test_vizier.py index 8e0bc5670a..22100c0808 100644 --- a/astroquery/vizier/tests/test_vizier.py +++ b/astroquery/vizier/tests/test_vizier.py @@ -8,7 +8,7 @@ from ... import vizier from ...utils import commons -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse VO_DATA = {'HIP,NOMAD,UCAC': "viz.xml", diff --git a/astroquery/vo_conesearch/validator/inspect.py b/astroquery/vo_conesearch/validator/inspect.py index 529a2f7319..8f0b656c1b 100644 --- a/astroquery/vo_conesearch/validator/inspect.py +++ b/astroquery/vo_conesearch/validator/inspect.py @@ -138,7 +138,7 @@ def print_cat(self, key, fout=None): If not found, nothing is written out. Parameters - ----------- + ---------- key : str Catalog key. diff --git a/astroquery/vo_conesearch/validator/tests/test_validate.py b/astroquery/vo_conesearch/validator/tests/test_validate.py index 5262d19cf1..f30354e51e 100644 --- a/astroquery/vo_conesearch/validator/tests/test_validate.py +++ b/astroquery/vo_conesearch/validator/tests/test_validate.py @@ -18,8 +18,8 @@ from numpy.testing import assert_allclose # ASTROPY -from astropy.tests.helper import catch_warnings from astropy.utils.data import get_pkg_data_filename +from astropy.utils.exceptions import AstropyUserWarning # LOCAL from .. import conf, validate, tstquery @@ -83,9 +83,8 @@ def teardown_class(self): @pytest.mark.remote_data def test_tstquery(): - with catch_warnings() as w: + with pytest.warns(AstropyUserWarning, match='too large') as w: d = tstquery.parse_cs('ivo://cds.vizier/i/252', cap_index=4) assert len(w) == 1 - assert 'too large' in str(w[0].message) assert_allclose([d['RA'], d['DEC'], d['SR']], [45, 0.07460390065517808, 0.1]) diff --git a/astroquery/xmatch/tests/test_xmatch.py b/astroquery/xmatch/tests/test_xmatch.py index b4acd849eb..8d1024dbca 100644 --- a/astroquery/xmatch/tests/test_xmatch.py +++ b/astroquery/xmatch/tests/test_xmatch.py @@ -8,7 +8,7 @@ from astropy.units import arcsec from ...utils import commons -from ...utils.testing_tools import MockResponse +from astroquery.utils.mocks import MockResponse from ...xmatch import XMatch DATA_FILES = { diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000000..1485a0c781 --- /dev/null +++ b/conftest.py @@ -0,0 +1,28 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +from pytest_astropy_header.display import (PYTEST_HEADER_MODULES, + TESTED_VERSIONS) + + +def pytest_configure(config): + config.option.astropy_header = True + + PYTEST_HEADER_MODULES['Astropy'] = 'astropy' + PYTEST_HEADER_MODULES['APLpy'] = 'aplpy' + PYTEST_HEADER_MODULES['pyregion'] = 'pyregion' + PYTEST_HEADER_MODULES['regions'] = 'regions' + PYTEST_HEADER_MODULES['pyVO'] = 'pyvo' + PYTEST_HEADER_MODULES['mocpy'] = 'mocpy' + PYTEST_HEADER_MODULES['astropy-healpix'] = 'astropy_healpix' + PYTEST_HEADER_MODULES['vamdclib'] = 'vamdclib' + + # keyring doesn't provide __version__ any more + # PYTEST_HEADER_MODULES['keyring'] = 'keyring' + + # add '_testrun' to the version name so that the user-agent indicates that + # it's being run in a test + from astroquery import version + version.version += '_testrun' + + TESTED_VERSIONS['astroquery'] = version.version + TESTED_VERSIONS['astropy_helpers'] = version.astropy_helpers_version diff --git a/docs/alma/alma.rst b/docs/alma/alma.rst index fa1d883535..2b191800aa 100644 --- a/docs/alma/alma.rst +++ b/docs/alma/alma.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.alma: ******************************** @@ -25,17 +23,14 @@ You can get interactive help to find out what keywords to query for: .. code-block:: python - >>> from astroquery.alma import Alma - >>> Alma.help() - Most common ALMA query keywords are listed below. These keywords are part - of the ALMA ObsCore model, an IVOA standard for metadata representation - (3rd column). They were also present in original ALMA Web form and, for - backwards compatibility can be accessed with their old names (2nd column). - More elaborate queries on the ObsCore model are possible with `query_sia` - or `query_tap` methods - Description Original ALMA keyword ObsCore keyword + >>> from astroquery.alma import Alma + >>> Alma.help() # doctest: +IGNORE_OUTPUT + + Most common ALMA query keywords are listed below. These keywords are part of the ALMA ObsCore model, an IVOA standard for metadata representation (3rd column). They were also present in original ALMA Web form and, for backwards compatibility can be accessed with their old names (2nd column). + More elaborate queries on the ObsCore model are possible with `query_sia` or `query_tap` methods + Description Original ALMA keyword ObsCore keyword ------------------------------------------------------------------------------------------------------- - + Position Source name (astropy Resolver) source_name_resolver SkyCoord.from_name Source name (ALMA) source_name_alma target_name @@ -44,25 +39,25 @@ You can get interactive help to find out what keywords to query for: Angular resolution (arcsec) spatial_resolution spatial_resolution Largest angular scale (arcsec) spatial_scale_max spatial_scale_max Field of view (arcsec) fov s_fov - + Energy Frequency (GHz) frequency frequency Bandwidth (Hz) bandwidth bandwidth Spectral resolution (KHz) spectral_resolution em_resolution Band band_list band_list - + Time Observation date start_date t_min Integration time (s) integration_time t_exptime - + Polarization - Polarisation type (Single, Dual, Full) polarisation_type pol_states - + Polarisation type (Single, Dual, Full) polarisation_type pol_states + Observation - Line sensitivity (10 km/s) (mJy/beam) line_sensitivity sensitivity_10kms + Line sensitivity (10 km/s) (mJy/beam) line_sensitivity sensitivity_10kms Continuum sensitivity (mJy/beam) continuum_sensitivity cont_sensitivity_bandwidth Water vapour (mm) water_vapour pvw - + Project Project code project_code proposal_id Project title project_title obs_title @@ -71,7 +66,7 @@ You can get interactive help to find out what keywords to query for: Project abstract project_abstract proposal_abstract Publication count publication_count NA Science keyword science_keyword science_keyword - + Publication Bibcode bibcode bib_reference Title pub_title pub_title @@ -79,10 +74,16 @@ You can get interactive help to find out what keywords to query for: Authors authors authors Abstract pub_abstract pub_abstract Year publication_year pub_year - + Options Public data only public_data data_rights Science observations only science_observations calib_level + + Examples of queries: + Alma.query('proposal_id':'2011.0.00131.S'} + Alma.query({'band_list': ['5', '7']} + Alma.query({'source_name_alma': 'GRB021004'}) + Alma.query(payload=dict(project_code='2017.1.01355.L', source_name_alma='G008.67')) Authentication ============== @@ -90,24 +91,24 @@ Authentication Users can log in to acquire proprietary data products. Login is performed via the ALMA CAS (central authentication server). -.. code-block:: python +.. doctest-skip:: >>> from astroquery.alma import Alma >>> alma = Alma() >>> # First example: TEST is not a valid username, it will fail - >>> alma.login("TEST") # doctest: +SKIP + >>> alma.login("TEST") TEST, enter your ALMA password: - - Authenticating TEST on asa.alma.cl... + + Authenticating TEST on asa.alma.cl ... Authentication failed! >>> # Second example: pretend ICONDOR is a valid username - >>> alma.login("ICONDOR", store_password=True) # doctest: +SKIP + >>> alma.login("ICONDOR", store_password=True) ICONDOR, enter your ALMA password: - + Authenticating ICONDOR on asa.alma.cl... Authentication successful! >>> # After the first login, your password has been stored - >>> alma.login("ICONDOR") # doctest: +SKIP + >>> alma.login("ICONDOR") Authenticating ICONDOR on asa.alma.cl... Authentication successful! @@ -123,13 +124,11 @@ Querying Targets and Regions You can query by object name or by circular region: -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.alma import Alma >>> m83_data = Alma.query_object('M83') - >>> print(len(m83_data)) - 830 - >>> m83_data.colnames + >>> m83_data.colnames # doctest: +IGNORE_OUTPUT ['obs_publisher_did', 'obs_collection', 'facility_name', 'instrument_name', 'obs_id', 'dataproduct_type', 'calib_level', 'target_name', 's_ra', 's_dec', 's_fov', 's_region', 's_resolution', 't_min', 't_max', @@ -137,71 +136,92 @@ You can query by object name or by circular region: 'pol_states', 'o_ucd', 'access_url', 'access_format', 'proposal_id', 'data_rights', 'gal_longitude', 'gal_latitude', 'band_list', 'em_resolution', 'bandwidth', 'antenna_arrays', 'is_mosaic', - 'obs_release_date', 'spatial_resolution', 'frequency_support', - 'frequency', 'velocity_resolution', 'obs_creator_name', 'pub_title', - 'first_author', 'authors', 'pub_abstract', 'publication_year', - 'proposal_abstract', 'schedblock_name', 'proposal_authors', - 'sensitivity_10kms', 'cont_sensitivity_bandwidth', 'pwv', 'group_ous_uid', - 'member_ous_uid', 'asdm_uid', 'obs_title', 'type', 'scan_intent', - 'science_observation', 'spatial_scale_max', 'qa2_passed', 'bib_reference', - 'science_keyword', 'scientific_category', 'lastModified'] - - -Please note that some of the column names are duplicated. First group of names -(the ones containing "_") are column names as they appear in the ALMA ObsCore -model while the second group are copies created to maintain backwards -compatibility with previous version of the library. + 'obs_release_date', 'spatial_resolution', 'frequency_support', 'frequency', + 'velocity_resolution', 'obs_creator_name', 'pub_title', 'first_author', + 'authors', 'pub_abstract', 'publication_year', 'proposal_abstract', + 'schedblock_name', 'proposal_authors', 'sensitivity_10kms', + 'cont_sensitivity_bandwidth', 'pwv', 'group_ous_uid', 'member_ous_uid', + 'asdm_uid', 'obs_title', 'type', 'scan_intent', 'science_observation', + 'spatial_scale_max', 'qa2_passed', 'bib_reference', 'science_keyword', + 'scientific_category', 'lastModified'] Region queries are just like any other in astroquery: -.. code-block:: python +.. doctest-remote-data:: >>> from astropy import coordinates >>> from astropy import units as u >>> galactic_center = coordinates.SkyCoord(0*u.deg, 0*u.deg, ... frame='galactic') >>> gc_data = Alma.query_region(galactic_center, 1*u.deg) - >>> print(len(gc_data)) - 383 + >>> print(gc_data) # doctest: +IGNORE_OUTPUT + obs_publisher_did obs_collection facility_name ... scientific_category lastModified + ... + --------------------------- -------------- ------------- ... --------------------------- ----------------------- + ADS/JAO.ALMA#2012.1.00133.S ALMA JAO ... ISM and star formation 2021-09-30T16:34:41.133 + Querying by other parameters ============================ As of version 0.3.4, you can also query other fields by keyword. For example, -if you want to find all projects with a particular PI, you could do: +if you want to find all projects in a region with a particular PI, you could do: -.. code-block:: python +.. doctest-remote-data:: + + >>> rslt = Alma.query_region('W51', radius=25*u.arcmin, pi_name='*Ginsburg*') + +or if you wanted all projects by a given PI: - >>> rslt = Alma.query_object('W51', pi_name='*Ginsburg*', public=False) +.. doctest-remote-data:: + + >>> rslt = Alma.query(payload=dict(pi_name='Ginsburg, Adam')) The ''query_sia'' method offers another way to query ALMA using the IVOA SIA -subset of keywords returning results in 'ObsCore' format. +subset of keywords returning results in 'ObsCore' format. For example, +to query for all images that have ``'XX'`` polarization (note that this query is too large +to run, it is just shown as an example): -.. code-block:: python +.. doctest-remote-data:: - >>> Alma.query_sia(query_sia(pol='XX')) + >>> Alma.query_sia(pol='XX') # doctest: +SKIP Finally, the ''query_tap'' method is the most general way of querying the ALMA metadata. This method is used to send queries to the service using the 'ObsCore' columns as constraints. The returned result is also in 'ObsCore' format. -.. code-block:: python +.. doctest-remote-data:: + + >>> Alma.query_tap("select * from ivoa.obscore where target_name like '%M83%'") # doctest: +IGNORE_OUTPUT + + obs_publisher_did obs_collection facility_name ... scientific_category lastModified + ... + str33 str4 str3 ... str200 object + --------------------------- -------------- ------------- ... ---------------------- ----------------------- + ADS/JAO.ALMA#2016.1.00164.S ALMA JAO ... Active galaxies 2021-09-30T16:34:41.133 + +One can also query by keyword, spatial resolution, etc: + +.. doctest-remote-data:: + + >>> Alma.query_tap("select * from ivoa.obscore WHERE spatial_resolution<=0.1 AND science_keyword " + ... "in ('Disks around high-mass stars', 'Asymptotic Giant Branch (AGB) stars') " + ... "AND science_observation='T'") # doctest: +IGNORE_OUTPUT - >>> Alma.query_tap("select * from ivoa.obscore where target_name like '%M83%'") Use the ''help_tap'' method to learn about the ALMA 'ObsCore' keywords and their types. -.. code-block:: python +.. doctest-remote-data:: - >>> Alma.help_tap() + >>> Alma.help_tap() # doctest: +IGNORE_OUTPUT Table to query is "voa.ObsCore". For example: "select top 1 * from ivoa.ObsCore" The scheme of the table is as follows. - + Name Type Unit Description ------------------------------------------------------------------------------------------ access_format char(9) Content format of the data @@ -210,8 +230,8 @@ their types. asdm_uid char(32*) UID of the ASDM containing this Field. authors char(4000*) Full list of first author and all co-authors band_list char(30*) Space delimited list of bands - bandwidth double Hz Total Bandwidth - bib_reference char(30*) Bibliography code + bandwidth double GHz Total Bandwidth + bib_reference char(4000*) Bibliography code calib_level int calibration level (2 or 3). 2 if product_type = MOUS, 3 if product_type = GOUS cont_sensitivity_bandwidth double mJy/beam Estimated noise in the aggregated continuum bandwidth. Note this is an indication only, it does not include the effects of flagging or dynamic range limitations. data_rights char(11) Access to data. @@ -268,53 +288,52 @@ their types. type char(16*) Type flags. velocity_resolution double m/s Estimated velocity resolution from all the spectral windows, from frequency resolution. + Downloading Data ================ -You can download ALMA data with astroquery, but be forewarned, cycle 0 and -cycle 1 data sets tend to be >100 GB! +You can download ALMA data with astroquery, but be forewarned, many data sets +are >100 GB! -.. code-block:: python +.. doctest-remote-data:: - >>> import numpy as np - >>> uids = np.unique(m83_data['Member ous id']) - >>> print(uids) - Member ous id + >>> import numpy as np + >>> from astroquery.alma import Alma + >>> m83_data = Alma.query_object('M83') + >>> uids = np.unique(m83_data['member_ous_uid']) + >>> print(uids) + member_ous_uid ----------------------- - uid://A002/X3216af/X31 - uid://A002/X5a9a13/X689 + uid://A001/X11f/X30 + uid://A001/X122/Xf3 + ... -New to most recent versions of the library is that data does not need to be -staged any longer. The ```stage_data``` method has been deprecated, but the -new ```get_data_info``` method can be used instead to get information about -the data such as the files, their urls, sizes etc: +The new ```get_data_info``` method can be used to get information about the +data such as the file names, their urls, sizes etc. -.. code-block:: python +.. doctest-remote-data:: - >>> link_list = Alma.get_data_info(uids) - >>> link_list['content_length'].sum() - 538298369462 - >>> len(link_list) - >>> 47 + >>> link_list = Alma.get_data_info(uids[:3]) By default, ALMA data is delivered as tarball files. However, the content of some of these files can be listed and accessed individually. To get information on the individual files: -.. code-block:: python - >>> link_list = Alma.get_data_info(uids, expand_tarfiles=True) - >>> len(link_list) - >>> 50 +.. doctest-remote-data:: -You can then go on to download that data. The download will be cached so that repeat -queries of the same file will not re-download the data. The default cache -directory is ``~/.astropy/cache/astroquery/Alma/``, but this can be changed by -changing the ``cache_location`` variable: + >>> link_list = Alma.get_data_info(uids[:3], expand_tarfiles=True) -.. code-block:: python +You can then go on to download those files. The download will be cached so +that repeat queries of the same file will not re-download the data. The +default cache directory is ``~/.astropy/cache/astroquery/Alma/``, but this can +be changed by changing the ``cache_location`` variable: + + +.. doctest-skip:: + >>> 1/0 >>> myAlma = Alma() >>> myAlma.cache_location = '/big/external/drive/' >>> myAlma.download_files(link_list, cache=True) @@ -323,34 +342,45 @@ You can also do the downloading all in one step: .. code-block:: python - >>> myAlma.retrieve_data_from_uid(uids[0]) + >>> myAlma.retrieve_data_from_uid(uids[0]) # doctest: +SKIP + +If you have huge files, sometimes the transfer fails, so you will need to +restart the download. By default, the module will resume downloading where the +failure occurred. You can check whether the downloads all succeeded before +triggering a new download by using the ``verify_only`` keyword, which will not +download but will return useful information about the state of your downloads: + +.. code-block:: python + + >>> myAlma.download_files(link_list, cache=True, verify_only=True) # doctest: +SKIP + Downloading FITS data ===================== -If you want just the QA2-produced FITS files, you can download the tarball, -extract the FITS file, then delete the tarball: +If you want just the QA2-produced FITS files, you can directly access the FITS +files: -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.alma.core import Alma >>> from astropy import coordinates >>> from astropy import units as u - >>> orionkl = coordinates.SkyCoord('5:35:14.461 -5:21:54.41', frame='fk5', - ... unit=(u.hour, u.deg)) - >>> result = Alma.query_region(orionkl, radius=0.034*u.deg) - >>> uid_url_table = Alma.get_data_info(result['obs_id']) - >>> # Extract the data with tarball file size < 1GB - >>> small_uid_url_table = uid_url_table[uid_url_table['content_length'] < 10**9] - >>> # get the first 10 files... - >>> tarball_files = uid_url_table[uid_url_table['content_type'] == 'application/x-tar'] - >>> filelist = Alma.download_and_extract_files(tarball_files[1:10]['access_url]) - -You might want to look at the READMEs from a bunch of files so you know what kind of S/N to expect: + >>> s255ir = coordinates.SkyCoord(93.26708333, 17.97888889, frame='fk5', + ... unit=(u.deg, u.deg)) + >>> result = Alma.query_region(s255ir, radius=0.034*u.deg) + >>> uid_url_table = Alma.get_data_info(result['obs_id'][0], expand_tarfiles=True) + >>> # downselect to just the FITSf files + >>> fits_urls = [url for url in uid_url_table['access_url'] if '.fits' in url] + >>> filelist = Alma.download_files(fits_urls[:5]) # doctest: +SKIP -.. code-block:: python +You might want to look at the READMEs from a bunch of files so you know what +kind of S/N to expect: + +.. doctest-remote-data:: - >>> filelist = Alma.download_and_extract_files(tarball_files['access_url'], regex='.*README$') + >>> readmes = [url for url in uid_url_table['access_url'] if 'README' in url] + >>> filelist = Alma.download_files(readmes) # doctest: +IGNORE_OUTPUT Further Examples @@ -358,6 +388,7 @@ Further Examples There are some nice examples of using the ALMA query tool in conjunction with other astroquery tools in :doc:`../gallery`, especially :ref:`gallery-almaskyview`. + Reference/API ============= diff --git a/docs/dace/dace.rst b/docs/dace/dace.rst index a0095a3395..651fcc150a 100644 --- a/docs/dace/dace.rst +++ b/docs/dace/dace.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.dace: ************************ @@ -9,34 +7,32 @@ DACE (`astroquery.dace`) This module let you query DACE (Data Analysis Center for Exoplanets) data. This project is developed at Observatory of Geneva and can be accessed online at https://dace.unige.ch +Getting started +=============== -API -=== Query radial velocities ----------------------- If you need to get radial velocities data for an object you can do the following and get a `~astropy.table.Table` : -.. code-block:: python + +.. doctest-remote-data:: >>> from astroquery.dace import Dace >>> radial_velocities_table = Dace.query_radial_velocities('HD40307') - >>> print(radial_velocities_table) - - rjd rv rv_err ins_name - ------------------ ------------------ ------------------ --------- ... - 51131.82401522994 31300.4226771379 5.420218247708816 CORALIE98 - 51139.809670339804 31295.5671320506 4.0697289792344185 CORALIE98 - 51188.67579095997 31294.3391634734 3.4386352834851026 CORALIE98 - 51259.531961040106 31298.3278930888 7.0721030870398245 CORALIE98 + >>> radial_velocities_table.pprint(max_lines=5, max_width=120) + berv berv_err ... date_night raw_file + ----------------- -------- ... ---------- ------------------------------------------------------------------------- + 1.73905237267071 NaN ... 1998-11-13 coralie98/DRS-3.3/reduced/1998-11-13/CORALIE.1998-11-14T07:42:28.001.fits + 1.30280483191029 NaN ... 1998-11-21 coralie98/DRS-3.3/reduced/1998-11-21/CORALIE.1998-11-22T07:21:45.001.fits + ... ... ... ... ... + -3.37421721287255 NaN ... 2014-03-31 harps/DRS-3.5/reduced/2014-03-31/HARPS.2014-03-31T23:53:00.821.fits + -3.32906204496065 NaN ... 2014-04-05 harps/DRS-3.5/reduced/2014-04-05/HARPS.2014-04-06T01:00:15.375.fits + Length = 600 rows - ... ... ... ... - 56403.48691046983 31333.3379143329 0.5476157667089154 HARPS03 - 56596.82446234021 31334.9563430348 0.508056405864858 HARPS03 - 56602.871036310215 31337.4095684621 0.4167374664543639 HARPS03 Reference/API ============= .. automodapi:: astroquery.dace - :no-inheritance-diagram: \ No newline at end of file + :no-inheritance-diagram: diff --git a/docs/esa/hubble.rst b/docs/esa/hubble.rst index dabb4dd0d3..c29ba32816 100644 --- a/docs/esa/hubble.rst +++ b/docs/esa/hubble.rst @@ -2,9 +2,9 @@ .. _astroquery.esa.hubble: -************************************ -esa.hubble (`astroquery.esa.hubble`) -************************************ +***************************************** +ESA HST Archive (`astroquery.esa.hubble`) +***************************************** The Hubble Space Telescope (HST) is a joint ESA/NASA orbiting astronomical observatory operating from the near-infrared into the ultraviolet. Launched @@ -339,6 +339,39 @@ votable). The result of the query will be stored in the file result.get_results() or printing it by doing print(result). +------------------------------------------------------ +9. Getting related members of HAP and HST observations +------------------------------------------------------ + +This function takes in an observation id of a Composite or Simple observation. +If the observation is Simple the method returns the Composite observation that +is built on this simple observation. If the observation is Composite then the +method returns the simple observations that make it up. + +.. code-block:: python + + >>> from astroquery.esa.hubble import ESAHubble + >>> esahubble = ESAHubble() + >>> result = esahubble.get_member_observations("jdrz0c010") + >>> print(result) + + +-------------------------------------------------------- +10. Getting link between Simple HAP and HST observations +-------------------------------------------------------- + +This function takes in an observation id of a Simple HAP or HST observation and +returns the corresponding HAP or HST observation + +.. code-block:: python + + >>> from astroquery.esa.hubble import ESAHubble + >>> esahubble = ESAHubble() + >>> result = esahubble.get_hap_hst_link("hst_16316_71_acs_sbc_f150lp_jec071i9") + >>> print(result) + + + Reference/API ============= diff --git a/docs/esa/jwst.rst b/docs/esa/jwst.rst index de072c462e..d14bc37095 100644 --- a/docs/esa/jwst.rst +++ b/docs/esa/jwst.rst @@ -2,9 +2,9 @@ .. _astroquery.esa.jwst: -********************************* -JWST TAP+ (`astroquery.esa.jwst`) -********************************* +**************************************** +ESA JWST Archive (`astroquery.esa.jwst`) +**************************************** The James Webb Space Telescope (JWST) is a collaborative project between NASA, ESA, and the Canadian Space Agency (CSA). Although radically different in @@ -651,7 +651,7 @@ To remove asynchronous ----------------------- Authenticated users are able to access to TAP+ capabilities (shared tables, persistent jobs, etc.) -In order to authenticate a user, ``login``, ``login_gui`` or ``login_token_gui`` methods must be called. After a successful +In order to authenticate a user, ``login`` method must be called. After a successful authentication, the user will be authenticated until ``logout`` method is called. All previous methods (``query_object``, ``cone_search``, ``load_table``, ``load_tables``, ``launch_job``) explained for @@ -667,18 +667,6 @@ The main differences are: 2.1. Login/Logout ~~~~~~~~~~~~~~~~~ -Using the graphic interface: - - -*Note: Tkinter module is required to use login_gui method.* - -.. code-block:: python - - >>> from astroquery.esa.jwst import Jwst - >>> from astroquery.esa.jwst import Jwst - >>> Jwst.login_gui() - - Using the command line: .. code-block:: python diff --git a/docs/esa/xmm_newton.rst b/docs/esa/xmm_newton.rst index 643297fbd0..c6fa76aeea 100644 --- a/docs/esa/xmm_newton.rst +++ b/docs/esa/xmm_newton.rst @@ -2,9 +2,9 @@ .. _astroquery.esa.xmm_newton: -**************************************** -xmm_newton (`astroquery.esa.xmm_newton`) -**************************************** +**************************************************** +ESA XMM-Newton Archive (`astroquery.esa.xmm_newton`) +**************************************************** The X-ray Multi-Mirror Mission, XMM-Newton, is an ESA X-ray observatory launched on 10 December 1999. @@ -19,9 +19,9 @@ XMM-Newton Science Operations Centre. Examples ======== ------------------------------- +-------------------------- 1. Getting XMM-Newton data ------------------------------- +-------------------------- .. code-block:: python @@ -36,8 +36,35 @@ it will store them in a tar called 'result0505720401.tar'. The parameters availa For more details of the parameters check the section 3.4 at: 'http://nxsa.esac.esa.int/nxsa-web/#aio' +-------------------------------------- +2. Getting XMM-Newton proprietary data +-------------------------------------- +To access proprietary data an extra variable is needed in the XMMNewton.download_data method. This variabe is prop which +can be True or False. If True a username and password is needed. A username and password can be passed by adding another +variable to the XMMNewton.download_data method called credentials_file. This variable is a string with the path to +~/.astropy/config/astroquery.cfg file. Inside this file add your desired username and password, e.g. + +.. code-block:: + + [xmm_newton] + username = your_username + password = your_password + +If the credentials_file variable is not provided the method will ask for the username and password to be added manually +from the commandline + +.. code-block:: python + + >>> from astroquery.esa.xmm_newton import XMMNewton + >>> + >>> XMMNewton.download_data('0505720401',level="PPS",extension="PDF",instname="M1",filename="result0505720401.tar",prop=True) + INFO: File result0505720401.tar downloaded to current directory [astroquery.esa.xmm_newton.core] + +This will download all PPS files for the observation '0505720401' and instrument MOS1, with 'PDF' extension and any +proprietary data. It will store them in a tar called 'result0505720401.tar'. + ------------------------------- -2. Getting XMM-Newton postcards +3. Getting XMM-Newton postcards ------------------------------- .. code-block:: python @@ -52,7 +79,7 @@ This will download the EPIC postcard for the observation '0505720401' and it wil 'P0505720401EPX000OIMAGE8000.PNG'. ------------------------------------------ -3. Getting XMM-Newton metadata through TAP +4. Getting XMM-Newton metadata through TAP ------------------------------------------ This function provides access to the XMM-Newton Science Archive database using the Table Access Protocol (TAP) and via the Astronomical Data @@ -81,7 +108,7 @@ This will execute an ADQL query to download the first 10 observations in the XMM stored in the file 'results10.csv'. The result of this query can be printed by doing print(result). ----------------------------------- -4. Getting table details of XSA TAP +5. Getting table details of XSA TAP ----------------------------------- .. code-block:: python @@ -104,7 +131,7 @@ stored in the file 'results10.csv'. The result of this query can be printed by d This will show the available tables in XSA TAP service in the XMM-Newton Science Archive. ------------------------------------- -5. Getting columns details of XSA TAP +6. Getting columns details of XSA TAP ------------------------------------- .. code-block:: python @@ -123,7 +150,7 @@ This will show the available tables in XSA TAP service in the XMM-Newton Science This will show the column details of the table 'v_all_observations' in XSA TAP service in the XMM-Newton Science Archive. -------------------------------------------- -6. Getting EPIC images from a given TAR file +7. Getting EPIC images from a given TAR file -------------------------------------------- .. code-block:: python @@ -139,7 +166,7 @@ This will show the column details of the table 'v_all_observations' in XSA TAP s This will extract the European Photon Imaging Camera (EPIC) images within the specified TAR file, bands, and instruments. It will also return a dictionary containing the paths to the extracted files. ------------------------------------------------------------------------------ -7. Getting the European Photon Imaging Camera (EPIC) metadata from the XSA TAP +8. Getting the European Photon Imaging Camera (EPIC) metadata from the XSA TAP ------------------------------------------------------------------------------ This function retrieves the EPIC metadata from a given target. diff --git a/docs/exoplanet_orbit_database/exoplanet_orbit_database.rst b/docs/exoplanet_orbit_database/exoplanet_orbit_database.rst index 61293374a4..b3d09d8350 100644 --- a/docs/exoplanet_orbit_database/exoplanet_orbit_database.rst +++ b/docs/exoplanet_orbit_database/exoplanet_orbit_database.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.exoplanet_orbit_database: **************************************************************** @@ -12,23 +10,22 @@ Accessing the planet table You can access the complete tables from each table source, with units assigned to columns wherever possible. -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.exoplanet_orbit_database import ExoplanetOrbitDatabase - >>> eod_table = ExoplanetOrbitDatabase.get_confirmed_planets_table() - + >>> eod_table = ExoplanetOrbitDatabase.get_table() >>> eod_table[:2] -
- pl_hostname pl_letter pl_discmethod ... pl_nnotes rowupdate NAME_LOWERCASE - ... - str27 str1 str29 ... int64 str10 str29 - ----------- --------- ------------- ... --------- ---------- -------------- - Kepler-151 b Transit ... 1 2014-05-14 kepler-151 b - Kepler-152 b Transit ... 1 2014-05-14 kepler-152 b + + A AUPPER ... NAME_LOWERCASE sky_coord + AU AU ... deg,deg + float64 float64 ... str19 object + --------- ----------- ... -------------- ----------------------------------- + 0.0780099 0.00130017 ... kepler-107d 297.0282083332539,48.20861111111111 + 0.0344721 0.000675924 ... kepler-1049b 287.3467499971389,47.7729444445504 You can query for the row from each table corresponding to one exoplanet: -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.exoplanet_orbit_database import ExoplanetOrbitDatabase >>> hatp11b = ExoplanetOrbitDatabase.query_planet('HAT-P-11 b') @@ -41,20 +38,17 @@ The properties of each planet are stored in a table, with `columns defined by the Exoplanet Orbit Database `_. There is also a special column of sky coordinates for each target, named ``sky_coord``. -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.exoplanet_orbit_database import ExoplanetOrbitDatabase >>> hatp11b = ExoplanetOrbitDatabase.query_planet('HAT-P-11 b') - >>> hatp11b['PER'] # Planet period - >>> hatp11b['R'] # Planet radius - >>> hatp11b['sky_coord'] # Position of host star + (297.70890417, 48.08029444)> Reference/API ============= diff --git a/docs/fermi/fermi.rst b/docs/fermi/fermi.rst index 00af84af42..70fd81334a 100644 --- a/docs/fermi/fermi.rst +++ b/docs/fermi/fermi.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - ********************************** Fermi Queries (`astroquery.fermi`) ********************************** @@ -10,16 +8,15 @@ Getting started The following example illustrates a Fermi LAT query, centered on M 31 for the energy range 1 to 100 GeV for the first day in 2013. -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery import fermi >>> result = fermi.FermiLAT.query_object('M31', energyrange_MeV='1000, 100000', ... obsdates='2013-01-01 00:00:00, 2013-01-02 00:00:00') - >>> print(result) - ['https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L1309041729388EB8D2B447_SC00.fits', - 'https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L1309041729388EB8D2B447_PH00.fits'] - >>> from astropy.io import fits - >>> sc = fits.open('https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L1309041729388EB8D2B447_SC00.fits') + >>> print(result) # doctest: +IGNORE_OUTPUT + ['https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L210111120827756AAA3A88_PH00.fits', + 'https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L210111120827756AAA3A88_SC00.fits'] + Reference/API diff --git a/docs/gaia/gaia.rst b/docs/gaia/gaia.rst index 48a81f61e7..ffa11617c2 100644 --- a/docs/gaia/gaia.rst +++ b/docs/gaia/gaia.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.gaia: ***************************** @@ -77,17 +75,16 @@ Examples This query searches for all the objects contained in an arbitrary rectangular projection of the sky. -It is possible to choose which data release to query, by default the Gaia DR2 catalogue is used. For example: - -.. code-block:: python +It is possible to choose which data release to query, by default the Gaia DR2 catalogue is used. For example:: - >>> Gaia.MAIN_GAIA_TABLE = "gaiadr2.gaia_source" # Select Data Release 2, default + >>> from astroquery.gaia import Gaia >>> Gaia.MAIN_GAIA_TABLE = "gaiaedr3.gaia_source" # Select early Data Release 3 + >>> Gaia.MAIN_GAIA_TABLE = "gaiadr2.gaia_source" # Reselect Data Release 2, default The following example searches for all the sources contained in an squared region of side = 0.1 degrees around an specific point in RA/Dec coordinates. -.. code-block:: python +.. doctest-remote-data:: >>> import astropy.units as u >>> from astropy.coordinates import SkyCoord @@ -97,86 +94,60 @@ degrees around an specific point in RA/Dec coordinates. >>> width = u.Quantity(0.1, u.deg) >>> height = u.Quantity(0.1, u.deg) >>> r = Gaia.query_object_async(coordinate=coord, width=width, height=height) - >>> r.pprint() - - dist solution_id ... epoch_photometry_url + INFO: Query finished. [astroquery.utils.tap.core] + >>> r.pprint(max_lines=12, max_width=130) + dist solution_id ... datalink_url ... - --------------------- ------------------- ... -------------------- - 0.0026034636994048854 1635721458409799680 ... - 0.0038518741347606357 1635721458409799680 ... - 0.00454542650096783 1635721458409799680 ... - 0.005613919443965546 1635721458409799680 ... - 0.005846434715822121 1635721458409799680 ... - 0.006209042666371929 1635721458409799680 ... - 0.007469463683838576 1635721458409799680 ... - 0.008202004514524316 1635721458409799680 ... - 0.008338509690874027 1635721458409799680 ... - 0.008406677772258921 1635721458409799680 ... - ... ... ... ... - 0.01943176697471851 1635721458409799680 ... - 0.019464719601172412 1635721458409799680 ... - 0.019467068628703368 1635721458409799680 ... - 0.019752561500226976 1635721458409799680 ... - 0.01991656886177004 1635721458409799680 ... - 0.020149589233310516 1635721458409799680 ... - 0.020307185970548904 1635721458409799680 ... - 0.020454730686780127 1635721458409799680 ... - 0.020802655215768254 1635721458409799680 ... - 0.021615117161838747 1635721458409799680 ... + --------------------- ------------------- ... ----------------------------------------------------------------------------------- + 0.0026034636994048854 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090334814214528 + 0.0038518741347606357 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090339113063296 + 0.00454542650096783 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090334814217600 + ... ... ... ... + 0.020307185970548904 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636089514478069888 + 0.020454730686780127 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636066940131244288 + 0.020802655215768254 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636067141990822272 + 0.021615117161838747 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090369173963776 Length = 50 rows + Queries return a limited number of rows controlled by ``Gaia.ROW_LIMIT``. To change the default behaviour set this appropriately. -.. code-block:: python +.. doctest-remote-data:: >>> Gaia.ROW_LIMIT = 8 >>> r = Gaia.query_object_async(coordinate=coord, width=width, height=height) - >>> r.pprint() - - dist solution_id ... epoch_photometry_url + INFO: Query finished. [astroquery.utils.tap.core] + >>> r.pprint(max_width=140) + dist solution_id ... datalink_url ... - --------------------- ------------------- ... -------------------- - 0.0026034636994048854 1635721458409799680 ... - 0.0038518741347606357 1635721458409799680 ... - 0.00454542650096783 1635721458409799680 ... - 0.005613919443965546 1635721458409799680 ... - 0.005846434715822121 1635721458409799680 ... - 0.006209042666371929 1635721458409799680 ... - 0.007469463683838576 1635721458409799680 ... - 0.008202004514524316 1635721458409799680 ... + --------------------- ------------------- ... ----------------------------------------------------------------------------------- + 0.0026034636994048854 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090334814214528 + 0.0038518741347606357 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090339113063296 + 0.00454542650096783 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090334814217600 + 0.005613919443965546 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636089583198816640 + 0.005846434715822121 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090334814218752 + 0.006209042666371929 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090334814213632 + 0.007469463683838576 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090339112308864 + 0.008202004514524316 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636089583198816512 To return an unlimited number of rows set ``Gaia.ROW_LIMIT`` to -1. -.. code-block:: python +.. doctest-remote-data:: >>> Gaia.ROW_LIMIT = -1 >>> r = Gaia.query_object_async(coordinate=coord, width=width, height=height) - >>> r.pprint() - - dist solution_id ... epoch_photometry_url + INFO: Query finished. [astroquery.utils.tap.core] + >>> r.pprint(max_lines=12, max_width=140) + dist solution_id ... datalink_url ... - --------------------- ------------------- ... -------------------- - 0.0026034636994048854 1635721458409799680 ... - 0.0038518741347606357 1635721458409799680 ... - 0.00454542650096783 1635721458409799680 ... - 0.005613919443965546 1635721458409799680 ... - 0.005846434715822121 1635721458409799680 ... - 0.006209042666371929 1635721458409799680 ... - 0.007469463683838576 1635721458409799680 ... - 0.008202004514524316 1635721458409799680 ... - 0.008338509690874027 1635721458409799680 ... - 0.008406677772258921 1635721458409799680 ... - ... ... ... ... - 0.049718018073992835 1635721458409799680 ... - 0.04977869666747251 1635721458409799680 ... - 0.05006096698512638 1635721458409799680 ... - 0.05038566478030134 1635721458409799680 ... - 0.050827895451955894 1635721458409799680 ... - 0.050860907684754444 1635721458409799680 ... - 0.051038347209386326 1635721458409799680 ... - 0.05121063325107872 1635721458409799680 ... - 0.051957226883925664 1635721458409799680 ... - 0.05320916763883812 1635721458409799680 ... + --------------------- ------------------- ... ----------------------------------------------------------------------------------- + 0.0026034636994048854 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090334814214528 + 0.0038518741347606357 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090339113063296 + 0.00454542650096783 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636090334814217600 + ... ... ... ... + 0.05121063325107872 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636065840618481024 + 0.051957226883925664 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6636093637644158592 + 0.05320916763883812 1635721458409799680 ... https://gea.esac.esa.int/data-server/datalink/links?ID=Gaia+DR2+6633086847005369088 Length = 176 rows @@ -186,18 +157,19 @@ To return an unlimited number of rows set ``Gaia.ROW_LIMIT`` to -1. This query performs a cone search centered at the specified RA/Dec coordinates with the provided radius argument. -.. code-block:: python +.. doctest-remote-data:: >>> import astropy.units as u >>> from astropy.coordinates import SkyCoord >>> from astroquery.gaia import Gaia >>> + >>> Gaia.ROW_LIMIT = 50 # Ensure the default row limit. >>> coord = SkyCoord(ra=280, dec=-60, unit=(u.degree, u.degree), frame='icrs') >>> radius = u.Quantity(1.0, u.deg) >>> j = Gaia.cone_search_async(coord, radius) + INFO: Query finished. [astroquery.utils.tap.core] >>> r = j.get_results() >>> r.pprint() - solution_id designation ... dist ... ------------------- ---------------------------- ... --------------------- @@ -206,7 +178,23 @@ radius argument. 1635721458409799680 Gaia DR2 6636090334814217600 ... 0.00454542650096783 1635721458409799680 Gaia DR2 6636089583198816640 ... 0.005613919443965546 1635721458409799680 Gaia DR2 6636090334814218752 ... 0.005846434715822121 + 1635721458409799680 Gaia DR2 6636090334814213632 ... 0.006209042666371929 + 1635721458409799680 Gaia DR2 6636090339112308864 ... 0.007469463683838576 + 1635721458409799680 Gaia DR2 6636089583198816512 ... 0.008202004514524316 + 1635721458409799680 Gaia DR2 6636089583198817664 ... 0.008338509690874027 + 1635721458409799680 Gaia DR2 6636089578899968384 ... 0.008406677772258921 ... ... ... ... + + 1635721458409799680 Gaia DR2 6636089510180765312 ... 0.01943176697471851 + 1635721458409799680 Gaia DR2 6636066871411763712 ... 0.019464719601172412 + 1635721458409799680 Gaia DR2 6636089514475519232 ... 0.019467068628703368 + 1635721458409799680 Gaia DR2 6636090407832546944 ... 0.019752561500226976 + 1635721458409799680 Gaia DR2 6636066940132132352 ... 0.01991656886177004 + 1635721458409799680 Gaia DR2 6636066871411763968 ... 0.020149589233310516 + 1635721458409799680 Gaia DR2 6636089514478069888 ... 0.020307185970548904 + 1635721458409799680 Gaia DR2 6636066940131244288 ... 0.020454730686780127 + 1635721458409799680 Gaia DR2 6636067141990822272 ... 0.020802655215768254 + 1635721458409799680 Gaia DR2 6636090369173963776 ... 0.021615117161838747 Length = 50 rows @@ -220,62 +208,65 @@ Table and columns metadata are specified by IVOA TAP_ recommendation To load only table names metadata (TAP+ capability): -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gaia import Gaia >>> tables = Gaia.load_tables(only_names=True) + INFO: Retrieving tables... [astroquery.utils.tap.core] + INFO: Parsing tables... [astroquery.utils.tap.core] + INFO: Done. [astroquery.utils.tap.core] >>> for table in (tables): - >>> print(table.get_qualified_name()) - - public.dual - public.tycho2 - public.igsl_source - public.hipparcos - ... - gaiadr2.gaia_source + ... print(table.get_qualified_name()) + external.external.apassdr9 + external.external.gaiadr2_astrophysical_parameters + external.external.gaiadr2_geometric_distance + external.external.gaiaedr3_distance + ... + tap_schema.tap_schema.keys + tap_schema.tap_schema.schemas + tap_schema.tap_schema.tables To load all tables metadata (TAP compatible): -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gaia import Gaia >>> tables = Gaia.load_tables() - >>> for table in (tables): - >>> print(table.get_qualified_name()) + INFO: Retrieving tables... [astroquery.utils.tap.core] + INFO: Parsing tables... [astroquery.utils.tap.core] + INFO: Done. [astroquery.utils.tap.core] + >>> print(tables[0]) + TAP Table name: external.external.apassdr9 + Description: The AAVSO Photometric All-Sky Survey - Data Release 9 + This publication makes use of data products from the AAVSO + Photometric All Sky Survey (APASS). Funded by the Robert Martin Ayers + Sciences Fund and the National Science Foundation. Original catalogue released by Henden et al. 2015 AAS Meeting #225, id.336.16. Data retrieved using the VizieR catalogue access tool, CDS, Strasbourg, France. The original description of the VizieR service was published in A&AS 143, 23. VizieR catalogue II/336. + Num. columns: 25 - public.dual - public.tycho2 - public.igsl_source - public.hipparcos - ... - gaiadr2.gaia_source To load only a table (TAP+ capability): -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gaia import Gaia - >>> table = Gaia.load_table('gaiadr2.gaia_source') - >>> print(f"table = {table}") - - Table name: gaiadr2.gaia_source + >>> gaiadr2_table = Gaia.load_table('gaiadr2.gaia_source') + Retrieving table 'gaiadr2.gaia_source' + >>> print(gaiadr2_table) + TAP Table name: gaiadr2.gaiadr2.gaia_source Description: This table has an entry for every Gaia observed source as listed in the Main Database accumulating catalogue version from which the catalogue release has been generated. It contains the basic source parameters, that is only final data (no epoch data) and no spectra (neither final nor epoch). - Num. columns: 96 + Num. columns: 95 Once a table is loaded, its columns can be inspected: -.. code-block:: python - - >>> from astroquery.gaia import Gaia - >>> gaiadr2_table = Gaia.load_table('gaiadr2.gaia_source') - >>> for column in (gaiadr2_table.columns): - >>> print(column.name) +.. doctest-remote-data:: + >>> for column in gaiadr2_table.columns: + ... print(column.name) solution_id designation source_id @@ -285,6 +276,9 @@ Once a table is loaded, its columns can be inspected: ra_error dec dec_error + parallax + parallax_error + parallax_over_error ... 1.4. Synchronous query @@ -303,7 +297,7 @@ The results can be saved in memory (default) or in a file. Query without saving results in a file: -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gaia import Gaia >>> @@ -312,21 +306,37 @@ Query without saving results in a file: ... "matched_observations,duplicated_source,phot_variable_flag " ... "from gaiadr2.gaia_source order by source_id") >>> r = job.get_results() - >>> print(r['solution_id']) - - solution_id - ------------------- - 1635378410781933568 - 1635378410781933568 - 1635378410781933568 - 1635378410781933568 - ... + >>> print(r['ra_dec_corr']) + ra_dec_corr + ------------- + 0.022670548 + 0.06490505 + 0.11690165 + 0.042778816 + 0.095711425 + 0.56088775 + -0.0028029205 + 0.11152559 + 0.6039746 + 0.06599529 + ... + 0.1803336 + 0.089540906 + 0.23512067 + 0.066183 + -0.29090926 + 0.21693705 + 0.1531835 + 0.14783339 + 0.32718197 + -0.05562011 + 0.008669683 Length = 100 rows Query saving results in a file (you may use 'output_format' to specified the results data format, available formats are: 'votable', 'votable_plain', 'fits', 'csv' and 'json', default is 'votable'): -.. code-block:: python +.. doctest-skip:: >>> from astroquery.gaia import Gaia >>> job = Gaia.launch_job("select top 100 " @@ -335,41 +345,28 @@ available formats are: 'votable', 'votable_plain', 'fits', 'csv' and 'json', def ... "from gaiadr2.gaia_source order by source_id", ... dump_to_file=True, output_format='votable') >>> print(job.outputFile) - 1592474300458O-result.vot.gz - >>> r = job.get_results() >>> print(r['solution_id']) - solution_id ------------------- - 1635378410781933568 - 1635378410781933568 - 1635378410781933568 - 1635378410781933568 + 1635721458409799680 + 1635721458409799680 + 1635721458409799680 + 1635721458409799680 + 1635721458409799680 ... Length = 100 rows Note: you can inspect the status of the job by typing: -.. code-block:: python +.. doctest-skip:: >>> print(job) - -
- name dtype unit description - -------------------- ------- ---- --------------------------------------------------- - solution_id int64 Solution Identifier - ref_epoch float64 yr Reference epoch - ra_dec_corr float32 Correlation between right ascension and declination - astrometric_n_obs_al int32 Total number of observations AL - matched_observations int16 Amount of observations matched to this source - duplicated_source bool Source with duplicate sources - phot_variable_flag object Photometric variability flag Jobid: None Phase: COMPLETED Owner: None - Output file: sync_20200525141041.xml.gz + Output file: 1592474300458O-result.vot.gz Results: None @@ -382,16 +379,14 @@ You have to provide the local path to the file you want to upload. In the follow the file 'my_table.xml' is located to the relative location where your python program is running. See note below. -.. code-block:: python +.. doctest-skip:: >>> from astroquery.gaia import Gaia - >>> upload_resource = 'my_table.xml' >>> j = Gaia.launch_job(query="select * from tap_upload.table_test", ... upload_resource=upload_resource, upload_table_name="table_test", verbose=True) >>> r = j.get_results() >>> r.pprint() - source_id alpha delta --------- ----- ----- a 1.0 2.0 @@ -400,11 +395,10 @@ running. See note below. Note: to obtain the current location, type: -.. code-block:: python +.. doctest-skip:: >>> import os >>> print(os.getcwd()) - /Current/directory/path 1.6. Asynchronous query @@ -417,15 +411,13 @@ Queries retrieved results can be stored locally in memory (by default) or in a f Query without saving results in a file: -.. code-block:: python - +.. doctest-remote-data:: >>> from astroquery.gaia import Gaia - >>> job = Gaia.launch_job_async("select top 100 designation,ra,dec " ... "from gaiadr2.gaia_source order by source_id") + INFO: Query finished. [astroquery.utils.tap.core] >>> r = job.get_results() >>> print(r) - designation ra dec deg deg ---------------------- ------------------ -------------------- @@ -433,50 +425,31 @@ Query without saving results in a file: Gaia DR2 34361129088 45.004316164207644 0.021045032689712983 Gaia DR2 38655544960 45.0049742449841 0.019877000365797714 Gaia DR2 309238066432 44.99503703932583 0.03815183599451371 - Gaia DR2 343597448960 44.96389532530429 0.043595184822725674 ... Length = 100 rows Query saving results in a file (you may use 'output_format' to specified the results data format, available formats are: 'votable', 'votable_plain', 'fits', 'csv' and 'json', default is 'votable'): -.. code-block:: python +.. doctest-skip-all:: >>> from astroquery.gaia import Gaia - - >>> job = Gaia.launch_job_async("select top 100 * " + >>> job = Gaia.launch_job_async("select top 100 ra, dec " ... "from gaiadr2.gaia_source order by source_id", ... dump_to_file=True, output_format='votable') - - Saving results to: 1592474453797O-result.vot.gz - + Saving results to: 1611860482314O-result.vot.gz >>> print(job) - - Jobid: 1487845273526O + Jobid: 1611860482314O Phase: COMPLETED Owner: None - Output file: async_20170223112113.vot + Output file: 1611860482314O-result.vot.gz Results: None - >>> r = job.get_results() - >>> print(r['solution_id']) - - solution_id - ------------------- - 1635378410781933568 - 1635378410781933568 - 1635378410781933568 - 1635378410781933568 - ... - Length = 100 rows - 1.7. Asynchronous job removal ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To remove asynchronous jobs - -.. code-block:: python +To remove asynchronous jobs:: >>> from astroquery.gaia import Gaia >>> Gaia.remove_jobs(["job_id_1","job_id_2",...]) @@ -536,9 +509,7 @@ A file where the credentials are stored can be used to login: >>> Gaia.login(credentials_file='my_credentials_file') -If you do not provide any parameters at all, a prompt will ask for the user name and password. - -.. code-block:: python +If you do not provide any parameters at all, a prompt will ask for the user name and password:: >>> from astroquery.gaia import Gaia >>> Gaia.login() @@ -546,12 +517,8 @@ If you do not provide any parameters at all, a prompt will ask for the user name >>> Password: pwd (not visible) -To logout +To logout:: - -.. code-block:: python - - >>> from astroquery.gaia import Gaia >>> Gaia.logout() @@ -561,30 +528,24 @@ To logout In the Gaia archive user tables can be shared among user groups. -To obtain a list of the tables shared to a user type the following: - -.. code-block:: python +To obtain a list of the tables shared to a user type the following:: >>> from astroquery.gaia import Gaia >>> tables = Gaia.load_tables(only_names=True, include_shared_tables=True) + INFO: Retrieving tables... [astroquery.utils.tap.core] + INFO: Parsing tables... [astroquery.utils.tap.core] + INFO: Done. [astroquery.utils.tap.core] >>> for table in (tables): - >>> print(table.get_qualified_name()) - - public.dual - public.tycho2 - public.igsl_source - tap_schema.tables - tap_schema.keys - tap_schema.columns - tap_schema.schemas - tap_schema.key_columns - gaiadr1.phot_variable_time_series_gfov - gaiadr1.ppmxl_neighbourhood - gaiadr1.gsc23_neighbourhood - ... - user_schema_1.table1 - user_schema_2.table1 - ... + ... print(table.get_qualified_name()) + external.external.apassdr9 + external.external.gaiadr2_geometric_distance + external.external.gaiaedr3_distance + external.external.galex_ais + ... ... ... + gaiadr2.gaiadr2.vari_time_series_statistics + gaiadr2.gaiadr2.panstarrs1_original_valid + gaiadr2.gaiadr2.gaia_source + gaiadr2.gaiadr2.ruwe 2.3. Uploading table to user space ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -608,9 +569,7 @@ An already generated VOTable, accessible through a URL, can be uploaded to Gaia The following example launches a query to Vizier TAP ('url' parameter). The result is a VOTable that can be uploaded to the user private area. -Your schema name will be automatically added to the provided table name. - -.. code-block:: python +Your schema name will be automatically added to the provided table name:: >>> from astroquery.gaia import Gaia >>> Gaia.login() @@ -618,22 +577,17 @@ Your schema name will be automatically added to the provided table name. >>> url = ("http://tapvizier.u-strasbg.fr/TAPVizieR/tap/sync/?" ... "REQUEST=doQuery&lang=ADQL&FORMAT=votable&" ... "QUERY=select+*+from+TAP_SCHEMA.columns+where+table_name='II/336/apass9'") - >>> job = Gaia.upload_table(upload_resource=url, table_name="table_test_from_url", ... table_description="Some description") - Job '1539932326689O' created to upload table 'table_test_from_url'. Now, you can query your table as follows (a full qualified table name must be provided, -i.e.: *user_.*): - -.. code-block:: python +i.e.: *user_.*):: >>> full_qualified_table_name = 'user_.table_test_from_url' >>> query = 'select * from ' + full_qualified_table_name >>> job = Gaia.launch_job(query=query) >>> results = job.get_results() - >>> print(results) 2.3.2. Uploading table from file @@ -656,15 +610,12 @@ Your schema name will be automatically added to the provided table name. Uploaded table 'table_test_from_file'. Now, you can query your table as follows (a full qualified table name must be provided, -i.e.: *user_.*): - -.. code-block:: python +i.e.: *user_.*):: >>> full_qualified_table_name = 'user_.table_test_from_file' >>> query = 'select * from ' + full_qualified_table_name >>> job = Gaia.launch_job(query=query) >>> results = job.get_results() - >>> print(results) 2.3.3. Uploading table from an astropy Table @@ -674,7 +625,6 @@ A in memory PyTable (See https://wiki.python.org/moin/PyTables) can be uploaded Your schema name will be automatically added to the provided table name. - .. code-block:: python >>> from astroquery.gaia import Gaia @@ -682,22 +632,18 @@ Your schema name will be automatically added to the provided table name. >>> a=[1,2,3] >>> b=['a','b','c'] >>> table = Table([a,b], names=['col1','col2'], meta={'meta':'first table'}) - >>> # Upload >>> Gaia.login() >>> Gaia.upload_table(upload_resource=table, table_name='table_test_from_astropy') Now, you can query your table as follows (a full qualified table name must be provided, -i.e.: *user_.*): - -.. code-block:: python +i.e.: *user_.*):: >>> full_qualified_table_name = 'user_.table_test_from_astropy' >>> query = 'select * from ' + full_qualified_table_name >>> job = Gaia.launch_job(query=query) >>> results = job.get_results() - >>> print(results) @@ -708,39 +654,30 @@ The results generated by an *asynchronous* job (from a query executed in the Gai ingested in a table in the user private area. The following example generates a job in the Gaia archive and then, the results are ingested in a -table named: user_.'t': - -.. code-block:: python +table named: user_.'t':: >>> from astroquery.gaia import Gaia >>> Gaia.login() >>> j1 = Gaia.launch_job_async("select top 10 * from gaiadr2.gaia_source") >>> job = Gaia.upload_table_from_job(j1) - Created table 't1539932994481O' from job: '1539932994481O'. Now, you can query your table as follows (a full qualified table name must be provided, -i.e.: *user_.t*): - -.. code-block:: python +i.e.: *user_.t*):: >>> full_qualified_table_name = 'user_.t1539932994481O' >>> query = 'select * from ' + full_qualified_table_name >>> job = Gaia.launch_job(query=query) >>> results = job.get_results() - >>> print(results) 2.4. Deleting table ~~~~~~~~~~~~~~~~~~~ -A table from the user private area can be deleted as follows: - -.. code-block:: python +A table from the user private area can be deleted as follows:: >>> from astroquery.gaia import Gaia >>> Gaia.login_gui() >>> job = Gaia.delete_user_table("table_test_from_file") - Table 'table_test_from_file' deleted. @@ -749,10 +686,10 @@ A table from the user private area can be deleted as follows: It can be useful for the user to modify the metadata of a given table. For example, a user might want to change the description (UCD) of a column, or the flags that give extra information -about certain column. This is possible using: - -.. code-block:: python +about certain column. This is possible using:: + >>> from astroquery.gaia import Gaia + >>> Gaia.login_gui() >>> Gaia.update_user_table(table_name, list_of_changes) where the list of changes is a list of 3 items: @@ -770,16 +707,12 @@ The metadata parameter to be changed can be 'utype', 'ucd', 'flags' or 'indexed' .. _UCD: http://www.ivoa.net/documents/latest/UCD.html .. _UTypes: http://www.ivoa.net/documents/Notes/UTypesUsage/index.html -For instance, the 'ra' column in the gaiadr2.gaia_source catalogue is specified as: - -.. code-block:: python +For instance, the 'ra' column in the gaiadr2.gaia_source catalogue is specified as:: Utype: Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1 Ucd: pos.eq.ra;meta.main -and the 'dec' column as: - -.. code-block:: python +and the 'dec' column as:: Utype: Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2 Ucd: pos.eq.dec;meta.main @@ -797,9 +730,7 @@ We want to set: * 'flags' of 'raj2000' column to 'Ra' * 'flags' of 'dej2000' column to 'Dec' -We can type the following: - -.. code-block:: python +We can type the following:: >>> from astroquery.gaia import Gaia >>> Gaia.login_gui() @@ -808,7 +739,6 @@ We can type the following: ... ["nobs","utype","utype sample"], ... ["raj2000","flags","Ra"], ... ["dej2000","flags","Dec"]]) - Retrieving table 'user_joe.table' Parsing table 'user_joe.table'... Done. @@ -827,19 +757,14 @@ source in the second table. Later, the table can be used to obtain the actual da In order to perform a cross match, both tables must have defined RA and Dec columns (Ra/Dec column flags must be set: see previous section to know how to assign those flags). -The following example uploads a table and then, the table is used in a cross match: - -.. code-block:: python +The following example uploads a table and then, the table is used in a cross match:: >>> from astroquery.gaia import Gaia >>> Gaia.login() - >>> table = file or astropy.table >>> Gaia.upload_table(upload_resource=table, table_name='my_sources') - >>> # the table will be uploaded into the user private space into the database >>> # the table can be referenced as . - >>> full_qualified_table_name = 'user_.my_sources' >>> xmatch_table_name = 'xmatch_table' >>> Gaia.cross_match(full_qualified_table_name_a=full_qualified_table_name, @@ -847,9 +772,7 @@ The following example uploads a table and then, the table is used in a cross mat ... results_table_name=xmatch_table_name, radius=1.0) -Once you have your cross match finished, you can obtain the results: - -.. code-block:: python +Once you have your cross match finished, you can obtain the results:: >>> xmatch_table = 'user_.' + xmatch_table_name >>> query = ('SELECT c."dist"*3600 as dist, a.*, b.* FROM gaiadr2.gaia_source AS a, ' @@ -859,7 +782,6 @@ Once you have your cross match finished, you can obtain the results: ... 'c.my_sources_my_sources_oid = b.my_sources_oid)' >>> job = Gaia.launch_job(query=query) >>> results = job.get_results() - >>> print(f"results = {results}") Cross-matching catalogues is one of the most popular operations executed in the Gaia archive. For more details about how to run different cross-matches we direct the reader to: https://gea.esac.esa.int/archive-help/tutorials/crossmatch/index.html @@ -898,7 +820,7 @@ will be able to access to your shared table in a query. >>> Gaia.login() >>> groups = Gaia.load_groups() >>> for group in groups: - >>> print(group.title) + ... print(group.title) 2.7.4. Adding users to a group ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -927,7 +849,7 @@ will be able to access to your shared table in a query. >>> from astroquery.gaia import Gaia >>> Gaia.login() >>> Gaia.share_table(group_name="my_group", - ... table_name="user_.my_table", + ... table_name="user_.my_table", ... description="description") diff --git a/docs/gama/gama.rst b/docs/gama/gama.rst index e1d8975c75..e317c217a4 100644 --- a/docs/gama/gama.rst +++ b/docs/gama/gama.rst @@ -1,4 +1,3 @@ -.. doctest-skip-all .. _astroquery.gama: ******************************** @@ -20,50 +19,36 @@ This sends an SQL query, passed as a string, to the GAMA server and returns a `~astropy.table.Table`. For example, to return basic information on the first 100 spectroscopic objects in the database: -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gama import GAMA >>> result = GAMA.query_sql('SELECT * FROM SpecAll LIMIT 100') - Downloading http://www.gama-survey.org/dr2/query/../tmp/GAMA_VHI6pj.fits - |===========================================| 37k/ 37k (100.00%) 00s >>> print(result) - SPECID SURVEY SURVEY_CODE RA ... DIST IS_SBEST IS_BEST - ------------------ ------ ----------- --------- ... ---- -------- ------- - 131671727225700352 SDSS 1 132.16668 ... 0.1 1 1 - 131671727229894656 SDSS 1 132.17204 ... 0.13 1 1 - 131671727246671872 SDSS 1 132.24395 ... 0.13 1 1 - 131671727255060480 SDSS 1 132.1767 ... 0.06 1 1 - 131671727267643392 SDSS 1 132.63599 ... 0.05 1 1 - 131671727271837696 SDSS 1 132.85366 ... 0.02 1 1 - 131671727276032000 SDSS 1 132.70244 ... 0.03 1 1 - 131671727292809216 SDSS 1 132.19579 ... 0.12 1 1 - 131671727301197824 SDSS 1 132.57563 ... 0.0 1 1 - 131671727309586432 SDSS 1 133.01007 ... 0.06 1 1 - 131671727313780736 SDSS 1 132.76907 ... 0.04 1 1 - 131671727322169344 SDSS 1 132.81014 ... 0.03 1 1 - 131671727334752256 SDSS 1 132.85607 ... 0.02 1 1 - 131671727338946560 SDSS 1 132.90222 ... 0.04 1 1 - 131671727351529472 SDSS 1 133.00397 ... 0.05 1 1 - 131671727355723776 SDSS 1 132.96032 ... 0.05 1 1 - 131671727359918080 SDSS 1 132.92164 ... 0.03 1 1 - ... ... ... ... ... ... ... ... - 131671727791931392 SDSS 1 131.59537 ... 0.03 1 1 - 131671727796125696 SDSS 1 131.58167 ... 0.11 1 1 - 131671727800320000 SDSS 1 131.47693 ... 0.05 1 1 - 131671727804514304 SDSS 1 131.47471 ... 0.03 1 1 - 131671727808708608 SDSS 1 131.60197 ... 0.03 1 1 - 131671727825485824 SDSS 1 132.18426 ... 0.05 1 1 - 131671727833874432 SDSS 1 132.2593 ... 0.05 1 1 - 131671727838068736 SDSS 1 132.1901 ... 0.09 1 1 - 131671727854845952 SDSS 1 132.30575 ... 0.04 1 1 - 131671727859040256 SDSS 1 132.419 ... 0.04 1 1 - 131671727867428864 SDSS 1 132.29052 ... 0.15 1 1 - 131671727871623168 SDSS 1 132.37213 ... 0.01 1 1 - 131671727880011776 SDSS 1 132.36358 ... 0.1 1 1 - 131671727892594688 SDSS 1 132.3956 ... 0.05 1 1 - 131671727896788992 SDSS 1 131.89562 ... 0.15 1 1 - 131671727900983296 SDSS 1 131.85848 ... 0.05 1 1 - 131671727905177600 SDSS 1 132.12958 ... 0.09 0 0 + SPECID SURVEY SURVEY_CODE RA ... DIST IS_SBEST IS_BEST + ------------------- ------ ----------- --------- ... ---- -------- ------- + 1030358159811700736 SDSS 1 211.73487 ... 0.05 1 1 + 1030358434689607680 SDSS 1 211.51452 ... 0.14 1 1 + 1030358984445421568 SDSS 1 211.78462 ... 0.02 1 1 + 1030359809079142400 SDSS 1 211.63878 ... 0.05 1 1 + 1030360358834956288 SDSS 1 211.79006 ... 0.04 1 1 + 1030360633712863232 SDSS 1 211.71473 ... 0.05 0 0 + 1030361183468677120 SDSS 1 211.74528 ... 0.04 1 0 + 1030361733224491008 SDSS 1 211.50587 ... 0.02 1 1 + 1030363382491932672 SDSS 1 211.63321 ... 0.02 1 1 + 1030363657369839616 SDSS 1 211.54913 ... 0.06 0 0 + ... ... ... ... ... ... ... ... + 1031441727430354944 SDSS 1 212.46214 ... 0.06 1 1 + 1031442002308261888 SDSS 1 212.37016 ... 0.11 1 1 + 1031442552064075776 SDSS 1 212.43301 ... 0.08 1 1 + 1031444751087331328 SDSS 1 212.37388 ... 0.1 1 1 + 1031445300843145216 SDSS 1 212.34656 ... 0.02 1 1 + 1031445575721052160 SDSS 1 212.89604 ... 0.03 1 1 + 1031446125476866048 SDSS 1 212.75493 ... 0.03 1 1 + 1031446400354772992 SDSS 1 212.90264 ... 0.04 1 1 + 1031446950110586880 SDSS 1 212.96246 ... 0.05 1 1 + 1031447774744307712 SDSS 1 213.0112 ... 0.09 1 1 + 1031448324500121600 SDSS 1 212.70039 ... 0.05 0 0 + Length = 100 rows Reference/API ============= diff --git a/docs/gemini/gemini.rst b/docs/gemini/gemini.rst index 6204264181..b1616b42f2 100644 --- a/docs/gemini/gemini.rst +++ b/docs/gemini/gemini.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.gemini: ************************************ @@ -12,27 +10,27 @@ Getting Started This module can be used to query the Gemini Archive. Below are examples of querying the data with various parameters. + Positional Queries ------------------ Positional queries can be based on a sky position. Radius is an optional parameter and the default is 0.3 degrees. -.. code-block:: python - >>> from astroquery.gemini import Observations - >>> from astropy import coordinates, units +.. doctest-remote-data:: - >>> coord = coordinates.SkyCoord(210.80242917, 54.34875, unit="deg") - >>> data = Observations.query_region(coordinates=coord, radius=0.3*units.deg) - >>> print(data[0:5]) - - exposure_time detector_roi_setting detector_welldepth_setting telescope ... - ------------- -------------------- -------------------------- ------------ ... - 119.9986 Full Frame -- Gemini-North ... - 119.9983 Full Frame -- Gemini-North ... - 119.9986 Full Frame -- Gemini-North ... - 119.9983 Full Frame -- Gemini-North ... - 99.9983 Full Frame -- Gemini-North ... + >>> from astroquery.gemini import Observations + >>> from astropy import coordinates, units + >>> coord = coordinates.SkyCoord(210.80242917, 54.34875, unit="deg") + >>> data = Observations.query_region(coordinates=coord, radius=0.3*units.deg) + >>> print(data[0:5]) + exposure_time detector_roi_setting ... release dec + ------------- -------------------- ... ---------- --------------- + 119.9986 Full Frame ... 2008-08-21 54.34877772501 + 119.9983 Full Frame ... 2008-09-25 54.376194395654 + 119.9986 Full Frame ... 2008-09-25 54.366916626746 + 119.9983 Full Frame ... 2008-09-25 54.274527402457 + 99.9983 Full Frame ... 2013-08-16 54.307561057825 Observation Name Queries @@ -40,20 +38,18 @@ Observation Name Queries You may also do a query by the name of the object you are interested in. -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gemini import Observations - >>> data = Observations.query_object(objectname='m101') >>> print(data[0:5]) - - exposure_time detector_roi_setting detector_welldepth_setting telescope ... - ------------- -------------------- -------------------------- ------------ ... - 119.9986 Full Frame -- Gemini-North ... - 119.9983 Full Frame -- Gemini-North ... - 119.9986 Full Frame -- Gemini-North ... - 119.9983 Full Frame -- Gemini-North ... - 99.9983 Full Frame -- Gemini-North ... + exposure_time detector_roi_setting ... release dec + ------------- -------------------- ... ---------- --------------- + -- Undefined ... 2013-12-21 -- + -- Undefined ... 2013-12-21 -- + 49.9987 Full Frame ... 2013-08-28 54.348777039949 + 49.9987 Full Frame ... 2013-08-28 54.346975563951 + 49.9989 Full Frame ... 2013-08-28 54.347048438693 Observation Criteria Queries @@ -68,32 +64,31 @@ Some examples of available search fields are the instrument used, such as GMOS-N and the program ID. For a complete list of available search fields, see `~astroquery.gemini.ObservationsClass.query_criteria` -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gemini import Observations - >>> data = Observations.query_criteria(instrument='GMOS-N', ... program_id='GN-CAL20191122', ... observation_type='BIAS') >>> print(data[0:5]) + exposure_time detector_roi_setting detector_welldepth_setting ... release dec + ------------- -------------------- -------------------------- ... ---------- --- + 0.0 Central Stamp -- ... 2019-11-22 -- + 0.0 Full Frame -- ... 2019-11-22 -- + 0.0 Full Frame -- ... 2019-11-22 -- + 0.0 Full Frame -- ... 2019-11-22 -- + 0.0 Full Frame -- ... 2019-11-22 -- + - exposure_time detector_roi_setting detector_welldepth_setting telescope mdready ... - ------------- -------------------- -------------------------- ------------ ------- ... - 0.0 Central Stamp -- Gemini-North True ... - 0.0 Full Frame -- Gemini-North True ... - 0.0 Full Frame -- Gemini-North True ... - 0.0 Full Frame -- Gemini-North True ... - 0.0 Full Frame -- Gemini-North True ... In addition, the criteria query can accept additional parameters via the ``*rawqueryargs`` and ``**rawquerykwargs`` optional parameters. The ``orderby`` parameter can be used to pass the desired sort order. -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gemini import Observations - >>> data = Observations.query_criteria('centralspectrum', ... instrument='GMOS-N', ... program_id='GN-CAL20191122', @@ -120,20 +115,18 @@ Note that *NotFail*, *notengineering*, *RAW*, and *cols* are all sent automatica terms need be passed into the method. If QA or engineering search terms are passed, those will replace the *NotFail* or *notengineering* terms respectively. -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gemini import Observations - >>> data = Observations.query_raw('GMOS-N', 'BIAS', progid='GN-CAL20191122') >>> print(data[0:5]) - - exposure_time detector_roi_setting detector_welldepth_setting telescope mdready ... - ------------- -------------------- -------------------------- ------------ ------- ... - 0.0 Central Stamp -- Gemini-North True ... - 0.0 Full Frame -- Gemini-North True ... - 0.0 Full Frame -- Gemini-North True ... - 0.0 Full Frame -- Gemini-North True ... - 0.0 Full Frame -- Gemini-North True ... + exposure_time detector_roi_setting detector_welldepth_setting ... release dec + ------------- -------------------- -------------------------- ... ---------- --- + 0.0 Central Stamp -- ... 2019-11-22 -- + 0.0 Full Frame -- ... 2019-11-22 -- + 0.0 Full Frame -- ... 2019-11-22 -- + 0.0 Full Frame -- ... 2019-11-22 -- + 0.0 Full Frame -- ... 2019-11-22 -- Authenticated Sessions @@ -143,10 +136,9 @@ The Gemini module allows for authenticated sessions using your GOA account. Thi with on the GOA homepage at ``__. The `astroquery.gemini.ObservationsClass.login` method returns `True` if successful. -.. code-block:: python +.. doctest-skip:: >>> from astroquery.gemini import Observations - >>> Observations.login(username, password) >>> # do something with your elevated access @@ -158,11 +150,10 @@ As a convenience, you can request file downloads directly from the Gemini module URL and fetches the file. It will use any authenticated session you may have, so it will retrieve any proprietary data you may be permissioned for. -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.gemini import Observations - - >>> Observations.get_file("GS2020AQ319-10.fits", download_dir="/tmp") + >>> Observations.get_file("GS2020AQ319-10.fits", download_dir="/tmp") # doctest: +IGNORE_OUTPUT Reference/API diff --git a/docs/hips2fits/hips2fits.rst b/docs/hips2fits/hips2fits.rst index 460b17da2d..75426db772 100644 --- a/docs/hips2fits/hips2fits.rst +++ b/docs/hips2fits/hips2fits.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.hips2fits: ****************************************** @@ -37,7 +35,7 @@ Examples With a user defined astropy WCS ------------------------------- -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.hips2fits import hips2fits >>> import matplotlib.pyplot as plt @@ -70,12 +68,13 @@ With a user defined astropy WCS ... cmap=Colormap('viridis'), ... ) >>> im = plt.imshow(result) - >>> plt.show(im) + >>> plt.show() # doctest: +SKIP .. image:: ./query_wcs.png Without WCS ------------ + +.. doctest-remote-data:: >>> from astroquery.hips2fits import hips2fits >>> import matplotlib.pyplot as plt @@ -98,7 +97,7 @@ Without WCS ... cmap=Colormap('viridis'), ... ) >>> im = plt.imshow(result) - >>> plt.show(im) + >>> plt.show() # doctest: +SKIP .. image:: ./query_no_wcs.png diff --git a/docs/index.rst b/docs/index.rst index d2f46bdcc4..4e0e502484 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,9 +41,17 @@ already installed, please make sure you use the ``--upgrade`` install option as $ pip install --pre astroquery +To install all the mandatory and optional dependencies add the ``[all]`` +identifyer to the pip command above (or use ``[docs]`` or ``[test]`` for the +dependencies required to build the documentation or run the tests): + +.. code-block:: bash + + $ pip install --pre astroquery[all] + In addition to the automated releases, we also keep doing regular, tagged version for maintenance and packaging purposes. These can be ``pip`` installed without the ``--pre`` option and -are available from the ``conda-forge`` conda channel. +are also available from the ``conda-forge`` conda channel. .. code-block:: bash @@ -185,23 +193,25 @@ The following modules have been completed using a common API: cadc/cadc.rst casda/casda.rst cds/cds.rst + linelists/cdms/cdms.rst + dace/dace.rst esa/hubble.rst esa/iso.rst esa/jwst.rst esa/xmm_newton.rst esasky/esasky.rst eso/eso.rst + image_cutouts/first/first.rst gaia/gaia.rst gama/gama.rst gemini/gemini.rst heasarc/heasarc.rst hips2fits/hips2fits.rst hitran/hitran.rst + ipac/irsa/irsa_dust/irsa_dust.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst - ipac/irsa/irsa_dust/irsa_dust.rst jplspec/jplspec.rst - linelists/cdms/cdms.rst magpis/magpis.rst mast/mast.rst mpc/mpc.rst @@ -221,7 +231,6 @@ The following modules have been completed using a common API: vo_conesearch/vo_conesearch.rst vsa/vsa.rst xmatch/xmatch.rst - dace/dace.rst These others are functional, but do not follow a common & consistent API: @@ -269,22 +278,22 @@ for each source) alfalfa/alfalfa.rst exoplanet_orbit_database/exoplanet_orbit_database.rst gama/gama.rst + ipac/irsa/irsa_dust/irsa_dust.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst - ipac/irsa/irsa_dust/irsa_dust.rst mast/mast.rst ipac/nexsci/nasa_exoplanet_archive.rst ipac/ned/ned.rst ogle/ogle.rst open_exoplanet_catalogue/open_exoplanet_catalogue.rst sdss/sdss.rst - ipac/irsa/sha/sha.rst simbad/simbad.rst + ipac/irsa/sha/sha.rst ukidss/ukidss.rst - vsa/vsa.rst vizier/vizier.rst - xmatch/xmatch.rst vo_conesearch/vo_conesearch.rst + vsa/vsa.rst + xmatch/xmatch.rst Archives -------- @@ -301,25 +310,25 @@ generally return a table listing the available data first. casda/casda.rst esa/hubble.rst esa/jwst.rst + esa/xmm_newton.rst eso/eso.rst fermi/fermi.rst gaia/gaia.rst + gemini/gemini.rst heasarc/heasarc.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst magpis/magpis.rst - gemini/gemini.rst mast/mast.rst ipac/ned/ned.rst noirlab/noirlab.rst nrao/nrao.rst nvas/nvas.rst sdss/sdss.rst + skyview/skyview.rst ipac/irsa/sha/sha.rst ukidss/ukidss.rst vsa/vsa.rst - skyview/skyview.rst - esa/xmm_newton.rst Simulations ----------- @@ -342,13 +351,13 @@ well as cross section and collision rates. Those services are: :maxdepth: 1 atomic/atomic.rst + linelists/cdms/cdms.rst + hitran/hitran.rst + jplspec/jplspec.rst lamda/lamda.rst nist/nist.rst splatalogue/splatalogue.rst vamdc/vamdc.rst - hitran/hitran.rst - linelists/cdms/cdms.rst - jplspec/jplspec.rst Other ----- @@ -359,12 +368,12 @@ above categories. Those services are here: .. toctree:: :maxdepth: 1 - nasa_ads/nasa_ads.rst - utils/tap.rst + astrometry_net/astrometry_net.rst + imcce/imcce.rst jplhorizons/jplhorizons.rst jplsbdb/jplsbdb.rst - imcce/imcce.rst - astrometry_net/astrometry_net.rst + nasa_ads/nasa_ads.rst + utils/tap.rst Topical Collections @@ -376,8 +385,8 @@ topical submodules: .. toctree:: :maxdepth: 1 - solarsystem/solarsystem.rst image_cutouts/image_cutouts.rst + solarsystem/solarsystem.rst Developer documentation diff --git a/docs/ipac/ned/ned.rst b/docs/ipac/ned/ned.rst index 476fd0325f..6104028a44 100644 --- a/docs/ipac/ned/ned.rst +++ b/docs/ipac/ned/ned.rst @@ -29,7 +29,7 @@ instance if you want to query NGC 224 No. Object Name RA ... Redshift Points Diameter Points Associations degrees ... --- ----------- ---------- ... --------------- --------------- ------------ - 1 MESSIER 031 10.68479 ... 37 7 2 + 1 MESSIER 031 10.68479 ... 37 13 2 Query a region @@ -140,10 +140,10 @@ These queries can be used to retrieve all objects that appear in the specified No. Object Name RA ... Diameter Points Associations degrees ... --- ------------------------- ---------- ... --------------- ------------ - 1 NGC 0262 12.19642 ... 8 0 + 1 NGC 0262 12.19642 ... 12 0 2 NGC 0449 19.0302 ... 7 0 3 NGC 0591 23.38028 ... 7 0 - 4 UGC 01214 25.99084 ... 7 0 + 4 UGC 01214 25.99084 ... 12 0 ... ... ... ... ... ... 33 WISEA J202325.39+113134.6 305.85577 ... 2 0 34 UGC 12149 340.28163 ... 8 0 @@ -245,11 +245,11 @@ for the specified object. We look at a simple example: 2 12h29m06.7000s ... Uncertain origin 3 12h29m06.7000s ... Uncertain origin ... ... ... ... ... - 113 12h29m07.9s ... Uncertain origin - 114 12h29m04s ... Uncertain origin - 115 12h29m06s ... Uncertain origin - 116 12h29m08s ... Uncertain origin - Length = 117 rows + 114 12h29m07.9s ... Uncertain origin + 115 12h29m04s ... Uncertain origin + 116 12h29m06s ... Uncertain origin + 117 12h29m08s ... Uncertain origin + Length = 118 rows Reference/API diff --git a/docs/ipac/nexsci/nasa_exoplanet_archive.rst b/docs/ipac/nexsci/nasa_exoplanet_archive.rst index 897ff34c9a..400a6ec543 100644 --- a/docs/ipac/nexsci/nasa_exoplanet_archive.rst +++ b/docs/ipac/nexsci/nasa_exoplanet_archive.rst @@ -27,7 +27,7 @@ For example, the following query searches the ``ps`` table of confirmed exoplane .. doctest-remote-data:: >>> from astroquery.ipac.nexsci.nasa_exoplanet_archive import NasaExoplanetArchive - >>> NasaExoplanetArchive.query_object("K2-18 b") + >>> NasaExoplanetArchive.query_object("K2-18 b") # doctest: +IGNORE_OUTPUT pl_name pl_letter hostname ... sy_kmagerr1 sy_kmagerr2 sky_coord ... deg,deg @@ -55,7 +55,7 @@ Similarly, cone searches can be executed using the `~astroquery.ipac.nexsci.nasa >>> from astroquery.ipac.nexsci.nasa_exoplanet_archive import NasaExoplanetArchive >>> NasaExoplanetArchive.query_region( ... table="pscomppars", coordinates=SkyCoord(ra=172.56 * u.deg, dec=7.59 * u.deg), - ... radius=1.0 * u.deg) + ... radius=1.0 * u.deg) # doctest: +IGNORE_OUTPUT pl_name pl_letter hostname ... htm20 sky_coord ... deg,deg @@ -71,7 +71,7 @@ For example, a full table can be queried as follows: .. doctest-remote-data:: >>> from astroquery.ipac.nexsci.nasa_exoplanet_archive import NasaExoplanetArchive - >>> NasaExoplanetArchive.query_criteria(table="cumulative", select="*") + >>> NasaExoplanetArchive.query_criteria(table="cumulative", select="*") # doctest: +IGNORE_OUTPUT kepid kepoi_name kepler_name ... koi_fittype koi_score sky_coord ... deg,deg @@ -113,7 +113,7 @@ In this section, we demonstrate >>> from astroquery.ipac.nexsci.nasa_exoplanet_archive import NasaExoplanetArchive >>> NasaExoplanetArchive.query_criteria(table="pscomppars", select="top 10 pl_name,ra,dec", - ... where="disc_facility like '%TESS%'") + ... where="disc_facility like '%TESS%'") # doctest: +IGNORE_OUTPUT pl_name ra dec sky_coord deg deg deg,deg @@ -155,7 +155,7 @@ In this section, we demonstrate >>> from astroquery.ipac.nexsci.nasa_exoplanet_archive import NasaExoplanetArchive >>> NasaExoplanetArchive.query_criteria( - ... table="pscomppars", where="hostname like 'Kepler%'", order="hostname") + ... table="pscomppars", where="hostname like 'Kepler%'", order="hostname") # doctest: +IGNORE_OUTPUT pl_name pl_letter hostname ... htm20 sky_coord ... deg,deg @@ -182,7 +182,7 @@ In this section, we demonstrate >>> from astroquery.ipac.nexsci.nasa_exoplanet_archive import NasaExoplanetArchive >>> NasaExoplanetArchive.query_criteria( ... table="koi", where="koi_vet_date>to_date('2015-01-24','yyyy-mm-dd')", - ... select="kepoi_name,koi_vet_date", order="koi_vet_date") + ... select="kepoi_name,koi_vet_date", order="koi_vet_date") # doctest: +IGNORE_OUTPUT kepoi_name koi_vet_date str9 str10 diff --git a/docs/mast/mast.rst b/docs/mast/mast.rst index 2f4f209a7b..5fb7bdb6f7 100644 --- a/docs/mast/mast.rst +++ b/docs/mast/mast.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.mast: ******************************** @@ -20,48 +18,45 @@ Positional queries can be based on a sky position or a target name. The observation fields are documented `here `__. -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> obs_table = Observations.query_region("322.49324 12.16683") - >>> print(obs_table[:10]) - - dataproduct_type obs_collection instrument_name ... distance - ---------------- -------------- --------------- ... -------- - cube SWIFT UVOT ... 0.0 - cube SWIFT UVOT ... 0.0 - cube SWIFT UVOT ... 0.0 - cube SWIFT UVOT ... 0.0 - cube SWIFT UVOT ... 0.0 - cube SWIFT UVOT ... 0.0 - cube SWIFT UVOT ... 0.0 - cube SWIFT UVOT ... 0.0 - cube SWIFT UVOT ... 0.0 - cube SWIFT UVOT ... 0.0 +.. doctest-remote-data:: + + >>> from astroquery.mast import Observations + ... + >>> obs_table = Observations.query_region("322.49324 12.16683") + >>> print(obs_table[:10]) # doctest: +IGNORE_OUTPUT + intentType obs_collection provenance_name ... srcDen obsid distance + ---------- -------------- --------------- ... ------ ----------- -------- + science SWIFT -- ... 5885.0 15000731855 0.0 + science SWIFT -- ... 5885.0 15000731856 0.0 + science SWIFT -- ... 5885.0 15000790494 0.0 + science SWIFT -- ... 5885.0 15000731857 0.0 + science SWIFT -- ... 5885.0 15000791686 0.0 + science SWIFT -- ... 5885.0 15000791687 0.0 + science SWIFT -- ... 5885.0 15000729841 0.0 + science SWIFT -- ... 5885.0 15000754475 0.0 + science SWIFT -- ... 5885.0 15000779206 0.0 + science SWIFT -- ... 5885.0 15000779204 0.0 Radius is an optional parameter and the default is 0.2 degrees. -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> obs_table = Observations.query_object("M8",radius=".02 deg") - >>> print(obs_table[:10]) - - dataproduct_type obs_collection instrument_name ... distance - ---------------- -------------- --------------- ... ------------- - cube K2 Kepler ... 39.4914065162 - spectrum IUE LWP ... 0.0 - spectrum IUE LWP ... 0.0 - spectrum IUE LWP ... 0.0 - spectrum IUE LWR ... 0.0 - spectrum IUE LWR ... 0.0 - spectrum IUE LWR ... 0.0 - spectrum IUE LWR ... 0.0 - spectrum IUE LWR ... 0.0 - spectrum IUE LWR ... 0.0 - +.. doctest-remote-data:: + + >>> from astroquery.mast import Observations + ... + >>> obs_table = Observations.query_object("M8",radius=".02 deg") + >>> print(obs_table[:10]) # doctest: +IGNORE_OUTPUT + intentType obs_collection provenance_name ... srcDen obsid distance + ---------- -------------- --------------- ... ------ ----------- -------- + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 + science SPITZER_SHA SSC Pipeline ... nan 19000016510 0.0 Observation Criteria Queries @@ -73,7 +68,10 @@ Criteria are supplied as keyword arguments, where valid criteria are "coordinate "objectname", "radius" (as in `~astroquery.mast.ObservationsClass.query_region` and `~astroquery.mast.ObservationsClass.query_object`), and all observation fields listed `here `__. -**Note:** The obstype keyword has been replaced by intentType, with valid values "calibration" and "science." If the intentType keyword is not supplied, both science and calibration observations will be returned. + +**Note:** The obstype keyword has been replaced by intentType, with valid values +"calibration" and "science." If the intentType keyword is not supplied, both science +and calibration observations will be returned. Argument values are one or more acceptable values for the criterion, except for fields with a float datatype where the argument should be in the form @@ -82,38 +80,36 @@ However, only one wildcarded value can be processed per criterion. RA and Dec must be given in decimal degrees, and datetimes in MJD. -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> obs_table = Observations.query_criteria(dataproduct_type=["image"], - ... proposal_pi="Osten*", - ... s_dec=[43.5,45.5]) - >>> print(obs_table) - - dataproduct_type calib_level obs_collection ... dataURL obsid objID - ---------------- ----------- -------------- ... ------- ---------- ---------- - image 1 HST ... None 2003520266 2011133418 - image 1 HST ... None 2003520267 2011133419 - image 1 HST ... None 2003520268 2011133420 - - >>> obs_table = Observations.query_criteria(filters=["*UV","Kepler"],objectname="M101") - >>> print(obs_table) - - dataproduct_type calib_level obs_collection ... objID1 distance - ---------------- ----------- -------------- ... ---------- ------------- - image 2 GALEX ... 1000055044 0.0 - image 2 GALEX ... 1000004937 3.83290685323 - image 2 GALEX ... 1000045953 371.718371962 - image 2 GALEX ... 1000055047 229.810616011 - image 2 GALEX ... 1000016644 229.810616011 - image 2 GALEX ... 1000045952 0.0 - image 2 GALEX ... 1000048357 0.0 - image 2 GALEX ... 1000001326 0.0 - image 2 GALEX ... 1000001327 371.718371962 - image 2 GALEX ... 1000004203 0.0 - image 2 GALEX ... 1000016641 0.0 - image 2 GALEX ... 1000048943 3.83290685323 +.. doctest-remote-data:: + + >>> from astroquery.mast import Observations + ... + >>> obs_table = Observations.query_criteria(dataproduct_type=["image"], + ... proposal_pi="Osten*", + ... s_dec=[43.5,45.5]) + >>> print(obs_table) # doctest: +IGNORE_OUTPUT + dataproduct_type calib_level obs_collection ... intentType obsid objID + ---------------- ----------- -------------- ... ---------- ---------- ---------- + image 1 HST ... science 2003520267 2023816094 + image 1 HST ... science 2003520266 2023816134 + image 1 HST ... science 2003520268 2025756935 + ... + >>> obs_table = Observations.query_criteria(filters=["*UV","Kepler"],objectname="M101") + >>> print(obs_table) # doctest: +IGNORE_OUTPUT + dataproduct_type calib_level obs_collection ... objID1 distance + ---------------- ----------- -------------- ... ---------- ------------------ + image 2 GALEX ... 1000045952 0.0 + image 2 GALEX ... 1000001327 371.71837196246395 + image 2 GALEX ... 1000016641 0.0 + image 2 GALEX ... 1000016644 229.81061601101433 + image 2 GALEX ... 1000001326 0.0 + image 2 GALEX ... 1000004203 0.0 + image 2 GALEX ... 1000004937 3.8329068532314046 + image 2 GALEX ... 1000045953 371.71837196246395 + image 2 GALEX ... 1000048357 0.0 + image 2 GALEX ... 1000048943 3.8329068532314046 + image 2 GALEX ... 1000055044 0.0 + image 2 GALEX ... 1000055047 229.81061601101433 Getting Observation Counts @@ -122,63 +118,60 @@ Getting Observation Counts To get the number of observations and not the observations themselves, query_counts functions are available. This can be useful if trying to decide whether the available memory is sufficient for the number of observations. -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.mast import Observations - - >>> print(Observations.query_region_count("322.49324 12.16683")) - 1804 - - >>> print(Observations.query_object_count("M8",radius=".02 deg")) - 196 - - >>> print(Observations.query_criteria_count(dataproduct_type="image", - ... filters=["NUV","FUV"], - ... t_max=[52264.4586,54452.8914])) - 59033 + >>> from astroquery.mast import Observations + ... + >>> print(Observations.query_region_count("322.49324 12.16683")) # doctest: +IGNORE_OUTPUT + 2364 + ... + >>> print(Observations.query_object_count("M8",radius=".02 deg")) # doctest: +IGNORE_OUTPUT + 469 + ... + >>> print(Observations.query_criteria_count(dataproduct_type="image", + ... filters=["NUV","FUV"], + ... t_max=[52264.4586,54452.8914])) # doctest: +IGNORE_OUTPUT + 59033 Metadata Queries ---------------- -To list data missions archived by MAST and avaiable through `astroquery.mast`, use the `~astroquery.mast.ObservationsClass.list_missions` function. - -.. code-block:: python +To list data missions archived by MAST and avaiable through `astroquery.mast`, +use the `~astroquery.mast.ObservationsClass.list_missions` function. - >>> from astroquery.mast import Observations - - >>> print(Observations.list_missions()) - ['IUE', 'Kepler', 'K2FFI', 'EUVE', 'HLA', 'KeplerFFI','FUSE', - 'K2', 'HST', 'WUPPE', 'BEFS', 'GALEX', 'TUES','HUT', 'SWIFT'] +.. doctest-remote-data:: + >>> from astroquery.mast import Observations + ... + >>> print(Observations.list_missions()) + ['BEFS', 'EUVE', 'FUSE', 'GALEX', 'HLA', 'HLSP', 'HST', 'HUT', 'IUE', 'JWST', 'K2', 'K2FFI', 'Kepler', 'KeplerFFI', 'OPO', 'PS1', 'SPITZER_SHA', 'SWIFT', 'TESS', 'TUES', 'WUPPE'] To get a table of metadata associated with observation or product lists use the `~astroquery.mast.ObservationsClass.get_metadata` function. -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> meta_table = Observations.get_metadata("observations") - >>> print(meta_table[:5]) - Column Name Column Label ... Examples/Valid Values - ----------------- ------------ ... --------------------------------- - obs_collection Mission ... E.g. SWIFT, PS1, HST, IUE - instrument_name Instrument ... E.g. WFPC2/WFC, UVOT, STIS/CCD - project Project ... E.g. HST, HLA, EUVE, hlsp_legus - filters Filters ... F469N, NUV, FUV, LOW DISP, MIRROR - wavelength_region Waveband ... EUV, XRAY, OPTICAL - - >>> meta_table = Observations.get_metadata("products") - >>> print(meta_table[:3]) - - Column Name Column Label ... Examples/Valid Values - -------------- ---------------- ... ------------------------------------- - obs_id Observation ID ... U24Z0101T, N4QF18030 - obsID Product Group ID ... Long integer, e.g. 2007590987 - obs_collection Mission ... HST, HLA, SWIFT, GALEX, Kepler, K2... - +.. doctest-remote-data:: + + >>> from astroquery.mast import Observations + ... + >>> meta_table = Observations.get_metadata("observations") + >>> print(meta_table[:5]) # doctest: +IGNORE_OUTPUT + Column Name Column Label ... Examples/Valid Values + --------------- ---------------- ... ---------------------------------- + intentType Observation Type ... Valid values: science, calibration + obs_collection Mission ... E.g. SWIFT, PS1, HST, IUE + provenance_name Provenance Name ... E.g. TASOC, CALSTIS, PS1 + instrument_name Instrument ... E.g. WFPC2/WFC, UVOT, STIS/CCD + project Project ... E.g. HST, HLA, EUVE, hlsp_legus + ... + >>> meta_table = Observations.get_metadata("products") + >>> print(meta_table[:3]) # doctest: +IGNORE_OUTPUT + Column Name Column Label ... Examples/Valid Values + -------------- ---------------- ... ------------------------------------- + obs_id Observation ID ... U24Z0101T, N4QF18030 + obsID Product Group ID ... Long integer, e.g. 2007590987 + obs_collection Mission ... HST, HLA, SWIFT, GALEX, Kepler, K2... Downloading Data @@ -193,96 +186,70 @@ Given one or more observations or observation ids ("obsid") a `~astropy.table.Table` containing the associated data products. The product fields are documented `here `__. -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> obs_table = Observations.query_object("M8",radius=".02 deg") - >>> data_products_by_obs = Observations.get_product_list(obs_table[0:2]) - >>> print(data_products_by_obs) - - obsID obs_collection ... productFilename size - ---------- -------------- ... ---------------------------------- -------- - 3000007760 IUE ... lwp13058.elbll.gz 185727 - 3000007760 IUE ... lwp13058.elbls.gz 183350 - 3000007760 IUE ... lwp13058.lilo.gz 612715 - 3000007760 IUE ... lwp13058.melol.gz 12416 - 3000007760 IUE ... lwp13058.melos.gz 12064 - 3000007760 IUE ... lwp13058.raw.gz 410846 - 3000007760 IUE ... lwp13058.rilo.gz 416435 - 3000007760 IUE ... lwp13058.silo.gz 100682 - 3000007760 IUE ... lwp13058.gif 8971 - 3000007760 IUE ... lwp13058.mxlo.gz 18206 - 3000007760 IUE ... lwp13058mxlo_vo.fits 48960 - 3000007760 IUE ... lwp13058.gif 3967 - 9500243833 K2 ... k2-tpf-only-target_bw_large.png 9009 - 9500243833 K2 ... ktwo200071160-c91_lpd-targ.fits.gz 39930404 - 9500243833 K2 ... ktwo200071160-c92_lpd-targ.fits.gz 62213068 - 9500243833 K2 ... k2-tpf-only-target_bw_thumb.png 1301 - - >>> obsids = obs_table[0:2]['obsid'] - >>> data_products_by_id = Observations.get_product_list(obsids) - >>> print(data_products_by_id) - - obsID obs_collection ... productFilename size - ---------- -------------- ... ---------------------------------- -------- - 3000007760 IUE ... lwp13058.elbll.gz 185727 - 3000007760 IUE ... lwp13058.elbls.gz 183350 - 3000007760 IUE ... lwp13058.lilo.gz 612715 - 3000007760 IUE ... lwp13058.melol.gz 12416 - 3000007760 IUE ... lwp13058.melos.gz 12064 - 3000007760 IUE ... lwp13058.raw.gz 410846 - 3000007760 IUE ... lwp13058.rilo.gz 416435 - 3000007760 IUE ... lwp13058.silo.gz 100682 - 3000007760 IUE ... lwp13058.gif 8971 - 3000007760 IUE ... lwp13058.mxlo.gz 18206 - 3000007760 IUE ... lwp13058mxlo_vo.fits 48960 - 3000007760 IUE ... lwp13058.gif 3967 - 9500243833 K2 ... k2-tpf-only-target_bw_large.png 9009 - 9500243833 K2 ... ktwo200071160-c91_lpd-targ.fits.gz 39930404 - 9500243833 K2 ... ktwo200071160-c92_lpd-targ.fits.gz 62213068 - 9500243833 K2 ... k2-tpf-only-target_bw_thumb.png 1301 - - >>> print((data_products_by_obs == data_products_by_id).all()) - True - - - - - -Downloading Data Products -------------------------- - -Products can be downloaded by using `~astroquery.mast.ObservationsClass.download_products`, -with a `~astropy.table.Table` of data products, or a list (or single) obsid as the argument. - -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> obsid = '3000007760' - >>> data_products = Observations.get_product_list(obsid) - >>> manifest = Observations.download_products(data_products) - Downloading URL http://archive.stsci.edu/pub/iue/data/lwp/13000/lwp13058.mxlo.gz to ./mastDownload/IUE/lwp13058/lwp13058.mxlo.gz ... [Done] - Downloading URL http://archive.stsci.edu/pub/vospectra/iue2/lwp13058mxlo_vo.fits to ./mastDownload/IUE/lwp13058/lwp13058mxlo_vo.fits ... [Done] - >>> print(manifest) - - Local Path Status Message URL - ------------------------------------------------ -------- ------- ---- - ./mastDownload/IUE/lwp13058/lwp13058.mxlo.gz COMPLETE None None - ./mastDownload/IUE/lwp13058/lwp13058mxlo_vo.fits COMPLETE None None - -​As an alternative to downloading the data files now, the curl_flag can be used instead to instead get a curl script that can be used to download the files at a later time. - -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> Observations.download_products('2003839997', - ... productType="SCIENCE", - ... curl_flag=True) - - Downloading URL https://mast.stsci.edu/portal/Download/stage/anonymous/public/514cfaa9-fdc1-4799-b043-4488b811db4f/mastDownload_20170629162916.sh to ./mastDownload_20170629162916.sh ... [Done] +.. doctest-remote-data:: + + >>> from astroquery.mast import Observations + ... + >>> obs_table = Observations.query_object("M8",radius=".02 deg") + >>> data_products_by_obs = Observations.get_product_list(obs_table[0:2]) + >>> print(data_products_by_obs) # doctest: +IGNORE_OUTPUT + obsID obs_collection dataproduct_type ... size parent_obsid + ----------- -------------- ---------------- ... ------- ------------ + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + ... ... ... ... ... ... + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 8648640 19000016510 + Length = 1153 rows + ... + >>> obsids = obs_table[0:2]['obsid'] + >>> data_products_by_id = Observations.get_product_list(obsids) + >>> print(data_products_by_id) # doctest: +IGNORE_OUTPUT + obsID obs_collection dataproduct_type ... size parent_obsid + ----------- -------------- ---------------- ... ------- ------------ + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + 19000016510 SPITZER_SHA image ... 316800 19000016510 + ... ... ... ... ... ... + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 57600 19000016510 + 19000016510 SPITZER_SHA image ... 8648640 19000016510 + Length = 1153 rows + ... + >>> print((data_products_by_obs == data_products_by_id).all()) + True Filtering @@ -292,157 +259,328 @@ Filter keyword arguments can be applied to download only data products that meet Available filters are "mrp_only" (Minimum Recommended Products), "extension" (file extension), and all products fields listed `here `_. -The ‘AND' operation is performed for a list of filters, and the ‘OR' operation is performed within a filter set. -The below example illustrates downloading all product files with the extension "fits" that are either "RAW" or "UNCAL." +The ‘AND' operation is performed for a list of filters, and the ‘OR' operation is performed within a +filter set. The below example illustrates downloading all product files with the extension "fits" that +are either "RAW" or "UNCAL." + +.. doctest-remote-data:: + + >>> from astroquery.mast import Observations + ... + >>> Observations.download_products('25119363', + ... productType=["SCIENCE", "PREVIEW"], + ... extension="fits") # doctest: +IGNORE_OUTPUT +
+ Local Path Status Message URL + str47 str8 object object + ----------------------------------------------- -------- ------- ------ + ./mastDownload/HST/fa2f0101m/fa2f0101m_a1f.fits COMPLETE None None + ./mastDownload/HST/fa2f0101m/fa2f0101m_a2f.fits COMPLETE None None + ./mastDownload/HST/fa2f0101m/fa2f0101m_a3f.fits COMPLETE None None -.. code-block:: python - - >>> from astroquery.mast import Observations +Product filtering can also be applied directly to a table of products without proceeding to the download step. - >>> Observations.download_products('2003839997', - ... productSubGroupDescription=["RAW", "UNCAL"], - ... extension="fits") - Downloading URL https://mast.stsci.edu/api/v0/download/file/HST/product/ib3p11p7q_raw.fits to ./mastDownload/HST/IB3P11P7Q/ib3p11p7q_raw.fits ... [Done] - Downloading URL https://mast.stsci.edu/api/v0/download/file/HST/product/ib3p11p8q_raw.fits to ./mastDownload/HST/IB3P11P8Q/ib3p11p8q_raw.fits ... [Done] - Downloading URL https://mast.stsci.edu/api/v0/download/file/HST/product/ib3p11phq_raw.fits to ./mastDownload/HST/IB3P11PHQ/ib3p11phq_raw.fits ... [Done] - Downloading URL https://mast.stsci.edu/api/v0/download/file/HST/product/ib3p11q9q_raw.fits to ./mastDownload/HST/IB3P11Q9Q/ib3p11q9q_raw.fits ... [Done] +.. doctest-remote-data:: + >>> from astroquery.mast import Observations + ... + >>> data_products = Observations.get_product_list('25588063') + >>> print(len(data_products)) + 27 + >>> products = Observations.filter_products(data_products, + ... productType=["SCIENCE", "PREVIEW"], + ... extension="fits") + >>> print(len(products)) + 8 -Product filtering can also be applied directly to a table of products without proceeding to the download step. -.. code-block:: python +Downloading Data Products +------------------------- - >>> from astroquery.mast import Observations +Products can be downloaded by using `~astroquery.mast.ObservationsClass.download_products`, +with a `~astropy.table.Table` of data products, or a list (or single) obsid as the argument. - >>> products = Observations.get_product_list('2003839997') - >>> print(len(products)) - 31 +.. doctest-skip:: + + >>> from astroquery.mast import Observations + ... + >>> single_obs = Observations.query_criteria(obs_collection="IUE",obs_id="lwp13058") + >>> data_products = Observations.get_product_list(single_obs) + ... + >>> manifest = Observations.download_products(data_products, productType="SCIENCE") + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=http://archive.stsci.edu/pub/iue/data/lwp/13000/lwp13058.mxlo.gz to ./mastDownload/IUE/lwp13058/lwp13058.mxlo.gz ... [Done] + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=http://archive.stsci.edu/pub/vospectra/iue2/lwp13058mxlo_vo.fits to ./mastDownload/IUE/lwp13058/lwp13058mxlo_vo.fits ... [Done] + ... + >>> print(manifest) + Local Path Status Message URL + ------------------------------------------------ -------- ------- ---- + ./mastDownload/IUE/lwp13058/lwp13058.mxlo.gz COMPLETE None None + ./mastDownload/IUE/lwp13058/lwp13058mxlo_vo.fits COMPLETE None None + +​As an alternative to downloading the data files now, the ``curl_flag`` can be used instead to instead get a +curl script that can be used to download the files at a later time. + +.. doctest-remote-data:: + + >>> from astroquery.mast import Observations + ... + >>> single_obs = Observations.query_criteria(obs_collection="IUE", obs_id="lwp13058") + >>> data_products = Observations.get_product_list(single_obs) + ... + >>> table = Observations.download_products(data_products, productType="SCIENCE", curl_flag=True) # doctest: +IGNORE_OUTPUT + Downloading URL https://mast.stsci.edu/portal/Download/stage/anonymous/public/514cfaa9-fdc1-4799-b043-4488b811db4f/mastDownload_20170629162916.sh to ./mastDownload_20170629162916.sh ... [Done] - >>> products = Observations.filter_products(data_products, - ... productSubGroupDescription=["RAW", "UNCAL"], - ... extension="fits") - >>> print(len(products)) - 4 Downloading a Single File ------------------------- -You can download a single data product file using the `~astroquery.mast.ObservationsClass.download_file` method, and passing in -a MAST dataURL. The default is to download the file the current working directory, which can be changed with -the *local_path* keyword argument. - -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> product = 'mast:IUE/url/pub/iue/data/lwp/13000/lwp13058.elbll.gz' - >>> result = Observations.download_file(product) - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:IUE/url/pub/iue/data/lwp/13000/lwp13058.elbll.gz to ./lwp13058.elbll.gz ... [Done] - >>> print(result) - ('COMPLETE', None, None) +You can download a single data product file using the `~astroquery.mast.ObservationsClass.download_file` +method, and passing in a MAST Data URI. The default is to download the file the current working directory, +which can be changed with the ``local_path`` keyword argument. + +.. doctest-remote-data:: + + >>> from astroquery.mast import Observations + ... + >>> single_obs = Observations.query_criteria(obs_collection="IUE",obs_id="lwp13058") + >>> data_products = Observations.get_product_list(single_obs) + ... + >>> product = data_products[0]["dataURI"] + >>> print(product) + mast:IUE/url/pub/iue/data/lwp/13000/lwp13058.elbll.gz + >>> result = Observations.download_file(product) # doctest: +IGNORE_OUTPUT + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:IUE/url/pub/iue/data/lwp/13000/lwp13058.elbll.gz to ./lwp13058.elbll.gz ... [Done] + ... + >>> print(result) + ('COMPLETE', None, None) Cloud Data Access ------------------ Public datasets from the Hubble, Kepler and TESS telescopes are also available for free on Amazon Web Services in `public S3 buckets `__. -Using AWS resources to process public data no longer requires an AWS account for all AWS regions. To enable cloud data access for the Hubble, Kepler, and TESS missions, follow the steps below: +Using AWS resources to process public data no longer requires an AWS account for all AWS regions. +To enable cloud data access for the Hubble, Kepler, TESS, and GALEX missions, follow the steps below: -You can enable cloud data access via the `~astroquery.mast.ObservationsClass.enable_cloud_dataset` function, which sets AWS to become the preferred source for data access as opposed to on-premise MAST until it is disabled with `~astroquery.mast.ObservationsClass.disable_cloud_dataset`. +You can enable cloud data access via the `~astroquery.mast.ObservationsClass.enable_cloud_dataset` +function, which sets AWS to become the preferred source for data access as opposed to on-premise +MAST until it is disabled with `~astroquery.mast.ObservationsClass.disable_cloud_dataset`. -To directly access a list of cloud URIs for a given dataset, use the `~astroquery.mast.ObservationsClass.get_cloud_uris` +To directly access a list of cloud URIs for a given dataset, use the +`~astroquery.mast.ObservationsClass.get_cloud_uris` function (Python will prompt you to enable cloud access if you haven't already). When cloud access is enabled, the standard download function -`~astroquery.mast.ObservationsClass.download_products` preferentially pulls files from AWS when they are available. When set to `True`, the ``cloud_only`` parameter in `~astroquery.mast.ObservationsClass.download_products` skips all data products not available in the cloud. +`~astroquery.mast.ObservationsClass.download_products` preferentially pulls files from AWS when they +are available. When set to `True`, the ``cloud_only`` parameter in +`~astroquery.mast.ObservationsClass.download_products` skips all data products not available in the cloud. Getting a list of S3 URIs: -.. code-block:: python - - >>> import os - >>> from astroquery.mast import Observations +.. doctest-skip:: + + >>> import os + >>> from astroquery.mast import Observations + ... + >>> # Simply call the `enable_cloud_dataset` method from `Observations`. The default provider is `AWS`, but we will write it in manually for this example: + >>> Observations.enable_cloud_dataset(provider='AWS') + INFO: Using the S3 STScI public dataset [astroquery.mast.core] + ... + >>> # Getting the cloud URIs + >>> obs_table = Observations.query_criteria(obs_collection='HST', + ... filters='F606W', + ... instrument_name='ACS/WFC', + ... proposal_id=['12062'], + ... dataRights='PUBLIC') + >>> products = Observations.get_product_list(obs_table) + >>> filtered = Observations.filter_products(products, + ... productSubGroupDescription='DRZ') + >>> s3_uris = Observations.get_cloud_uris(filtered) + >>> print(s3_uris) + ['s3://stpubdata/hst/public/jbev/jbeveo010/jbeveo010_drz.fits', 's3://stpubdata/hst/public/jbev/jbeveo010/jbeveo010_drz.fits', 's3://stpubdata/hst/public/jbev/jbevet010/jbevet010_drz.fits', 's3://stpubdata/hst/public/jbev/jbevet010/jbevet010_drz.fits'] + ... + >>> Observations.disable_cloud_dataset() - >>> # Simply call the `enable_cloud_dataset` method from `Observations`. The default provider is `AWS`, but we will write it in manually for this example: - >>> Observations.enable_cloud_dataset(provider='AWS') - INFO: Using the S3 STScI public dataset [astroquery.mast.core] - >>> # Getting the cloud URIs - >>> obs_table = Observations.query_criteria(obs_collection='HST', - ... filters='F606W', - ... instrument_name='ACS/WFC', - ... proposal_id=['12062'], - ... dataRights='PUBLIC') - >>> products = Observations.get_product_list(obs_table) - >>> filtered = Observations.filter_products(products, - ... productSubGroupDescription='DRZ') - >>> s3_uris = Observations.get_cloud_uris(filtered) - >>> print(s3_uris) - ['s3://stpubdata/hst/public/jbev/jbeveo010/jbeveo010_drz.fits', 's3://stpubdata/hst/public/jbev/jbeveo010/jbeveo010_drz.fits', 's3://stpubdata/hst/public/jbev/jbevet010/jbevet010_drz.fits', 's3://stpubdata/hst/public/jbev/jbevet010/jbevet010_drz.fits'] +Downloading data products from S3: +.. doctest-skip:: + + >>> import os + >>> from astroquery.mast import Observations + ... + >>> # Simply call the `enable_cloud_dataset` method from `Observations`. The default provider is `AWS`, but we will write it in manually for this example: + >>> Observations.enable_cloud_dataset(provider='AWS') + INFO: Using the S3 STScI public dataset [astroquery.mast.core] + ... + >>> # Downloading from the cloud + >>> obs_table = Observations.query_criteria(obs_collection=['Kepler'], + ... objectname="Kepler 12b", radius=0) + >>> products = Observations.get_product_list(obs_table[0]) + >>> manifest = Observations.download_products(products[:10], cloud_only=True) + manifestDownloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/dv_files/0118/011804465/kplr011804465-01-20160209194854_dvs.pdf to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-01-20160209194854_dvs.pdf ... + |==========================================| 1.5M/1.5M (100.00%) 0s + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/dv_files/0118/011804465/kplr011804465-20160128150956_dvt.fits to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-20160128150956_dvt.fits ... + |==========================================| 17M/ 17M (100.00%) 1s + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/dv_files/0118/011804465/kplr011804465-20160209194854_dvr.pdf to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-20160209194854_dvr.pdf ... + |==========================================| 5.8M/5.8M (100.00%) 0s + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/dv_files/0118/011804465/kplr011804465_q1_q17_dr25_obs_tcert.pdf to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465_q1_q17_dr25_obs_tcert.pdf ... + |==========================================| 2.2M/2.2M (100.00%) 0s + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/previews/0118/011804465/kplr011804465-2013011073258_llc_bw_large.png to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-2013011073258_llc_bw_large.png ... + |==========================================| 24k/ 24k (100.00%) 0s + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/target_pixel_files/0118/011804465/kplr011804465_tpf_lc_Q111111110111011101.tar to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465_tpf_lc_Q111111110111011101.tar ... + |==========================================| 43M/ 43M (100.00%) 4s + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/lightcurves/0118/011804465/kplr011804465_lc_Q111111110111011101.tar to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465_lc_Q111111110111011101.tar ... + |==========================================| 5.9M/5.9M (100.00%) 0s + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/lightcurves/0118/011804465/kplr011804465-2009131105131_llc.fits to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-2009131105131_llc.fits ... + |==========================================| 77k/ 77k (100.00%) 0s + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/lightcurves/0118/011804465/kplr011804465-2009166043257_llc.fits to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-2009166043257_llc.fits ... + |==========================================| 192k/192k (100.00%) 0s + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/lightcurves/0118/011804465/kplr011804465-2009259160929_llc.fits to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-2009259160929_llc.fits ... + |==========================================| 466k/466k (100.00%) 0s + ... + >>> print(manifest["Status"]) + Status + -------- + COMPLETE + COMPLETE + COMPLETE + COMPLETE + COMPLETE + COMPLETE + COMPLETE + COMPLETE + COMPLETE + COMPLETE + ... + >>> Observations.disable_cloud_dataset() + + + +Mission Searches +================ - >>> Observations.disable_cloud_dataset() +Mission-Specific Search Queries +------------------------------- + +These queries allow for searches based on mission-specific metadata for a given +data collection. Currently it provides access to a broad set of Hubble Space +Telescope (HST) metadata, including header keywords, proposal information, and +observational parameters. The available metadata includes all information that +was previously available in the original HST web search form, and are present in +the current `Mission Search interface `__. + +**Note:** this API interface does not yet support data product download, only +metadata earch access. + +An object of MastMissions class is instantiated with a default mission of 'hst' and +default service set to 'search'. + +.. doctest-remote-data:: + + >>> from astroquery.mast import MastMissions + >>> missions = MastMissions() + >>> missions.mission + 'hst' + >>> missions.service + 'search' + +The missions object can be used to search metadata using by sky position, or other criteria. +The keyword arguments can be used to specify output characteristics like selec_cols and +sort_by and conditions that filter on values like proposal id, pi last name etc. +The available column names for a mission are returned by the +`~astroquery.mast.MastMissionsClass.get_column_list` function. + +.. doctest-remote-data:: + + >>> from astroquery.mast import MastMissions + >>> missions = MastMissions(mission='hst') + >>> columns = missions.get_column_list() + +For positional searches, the columns "ang_sep", "sci_data_set_name", "search_key" and "search_position" +will always be included, in addition to any columns specified using "select_cols". For non-positional +searches, "search_key" and "sci_data_set_name" will always be included, in addition to any columns +specified using "select_cols". + +For a non positional search, ``select_cols`` would always include ``'search_key'`` and ``'sci_data_set_name'``. + +.. doctest-remote-data:: + + >>> from astroquery.mast import MastMissions + >>> from astropy.coordinates import SkyCoord + >>> missions = MastMissions(mission='hst') + >>> regionCoords = SkyCoord(210.80227, 54.34895, unit=('deg', 'deg')) + >>> results = missions.query_region(regionCoords, radius=3, sci_pep_id=12556, + ... select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"], + ... sort_by=['sci_targname']) + >>> results[:5] # doctest: +IGNORE_OUTPUT +
+ sci_status sci_targname sci_data_set_name ang_sep sci_pep_id search_pos sci_pi_last_name search_key + str6 str16 str9 str20 int64 str18 str6 str27 + ---------- ---------------- ----------------- -------------------- ---------- ------------------ ---------------- --------------------------- + PUBLIC NUCLEUS+HODGE602 OBQU010H0 0.017460048037303017 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU010H0 + PUBLIC NUCLEUS+HODGE602 OBQU01050 0.017460048037303017 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU01050 + PUBLIC NUCLEUS+HODGE602 OBQU01030 0.022143836477276503 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU01030 + PUBLIC NUCLEUS+HODGE602 OBQU010F0 0.022143836477276503 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU010F0 + PUBLIC NUCLEUS+HODGE602 OBQU010J0 0.04381046755938432 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU010J0 + +for paging through the results, offset and limit can be used to specify the starting record and the number +of returned records. the default values for offset and limit is 0 and 5000 respectively. + +.. doctest-remote-data:: + + >>> from astroquery.mast import MastMissions + >>> from astropy.coordinates import SkyCoord + >>> missions = MastMissions() + >>> results = missions.query_criteria(sci_start_time=">=2021-01-01 00:00:00", + ... select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status", "sci_pep_id"], + ... sort_by=['sci_pep_id'], limit=1000, offset=1000) # doctest: +IGNORE_WARNINGS + ... # MaxResultsWarning('Maximum results returned, may not include all sources within radius.') + >>> len(results) + 1000 + +Metadata queries can also be performed using object names with the +~astroquery.mast.MastMissionsClass.query_object function. + +.. doctest-remote-data:: + + >>> results = missions.query_object('M101', radius=3, select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"], + ... sort_by=['sci_targname']) + >>> results[:5] # doctest: +IGNORE_OUTPUT +
+ ang_sep search_pos sci_status search_key sci_stop_time sci_targname sci_start_time sci_data_set_name + str20 str18 str6 str27 str26 str16 str26 str9 + ------------------ ------------------ ---------- --------------------------- -------------------------- ------------ -------------------------- ----------------- + 2.751140575012458 210.80227 54.34895 PUBLIC 210.80227 54.34895LDJI01010 2019-02-19T05:52:40.020000 +164.6+9.9 2019-02-19T00:49:58.010000 LDJI01010 + 0.8000626246647815 210.80227 54.34895 PUBLIC 210.80227 54.34895J8OB02011 2003-08-27T08:27:34.513000 ANY 2003-08-27T07:44:47.417000 J8OB02011 + 1.1261718338567348 210.80227 54.34895 PUBLIC 210.80227 54.34895J8D711J1Q 2003-01-17T00:50:22.250000 ANY 2003-01-17T00:42:06.993000 J8D711J1Q + 1.1454431087675097 210.80227 54.34895 PUBLIC 210.80227 54.34895JD6V01012 2017-06-15T18:33:25.983000 ANY 2017-06-15T18:10:12.037000 JD6V01012 + 1.1457795862361977 210.80227 54.34895 PUBLIC 210.80227 54.34895JD6V01013 2017-06-15T20:08:44.063000 ANY 2017-06-15T19:45:30.023000 JD6V01013 + +Metadata queries can also be performed using non-positional parameters with the +`~astroquery.mast.MastMissionsClass.query_criteria` function. + +.. doctest-remote-data:: + + >>> results = missions.query_criteria(sci_data_set_name="Z06G0101T", sci_pep_id="1455", + ... select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"], + ... sort_by=['sci_targname']) + >>> results[:5] # doctest: +IGNORE_OUTPUT +
+ search_key sci_stop_time sci_data_set_name sci_start_time sci_targname sci_status + str9 str26 str9 str26 str19 str6 + ---------- -------------------------- ----------------- -------------------------- ------------ ---------- + Z06G0101T 1990-05-13T11:02:34.567000 Z06G0101T 1990-05-13T10:38:09.193000 -- PUBLIC -Downloading data products from S3: -.. code-block:: python - - >>> import os - >>> from astroquery.mast import Observations - - >>> # Simply call the `enable_cloud_dataset` method from `Observations`. The default provider is `AWS`, but we will write it in manually for this example: - >>> Observations.enable_cloud_dataset(provider='AWS') - INFO: Using the S3 STScI public dataset [astroquery.mast.core] - - >>> # Downloading from the cloud - >>> obs_table = Observations.query_criteria(obs_collection=['Kepler'], - ... objectname="Kepler 12b", radius=0) - >>> products = Observations.get_product_list(obs_table[0]) - >>> manifest = Observations.download_products(products[:10], cloud_only=True) - - manifestDownloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/dv_files/0118/011804465/kplr011804465-01-20160209194854_dvs.pdf to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-01-20160209194854_dvs.pdf ... - |==========================================| 1.5M/1.5M (100.00%) 0s - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/dv_files/0118/011804465/kplr011804465-20160128150956_dvt.fits to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-20160128150956_dvt.fits ... - |==========================================| 17M/ 17M (100.00%) 1s - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/dv_files/0118/011804465/kplr011804465-20160209194854_dvr.pdf to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-20160209194854_dvr.pdf ... - |==========================================| 5.8M/5.8M (100.00%) 0s - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/dv_files/0118/011804465/kplr011804465_q1_q17_dr25_obs_tcert.pdf to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465_q1_q17_dr25_obs_tcert.pdf ... - |==========================================| 2.2M/2.2M (100.00%) 0s - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/previews/0118/011804465/kplr011804465-2013011073258_llc_bw_large.png to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-2013011073258_llc_bw_large.png ... - |==========================================| 24k/ 24k (100.00%) 0s - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/target_pixel_files/0118/011804465/kplr011804465_tpf_lc_Q111111110111011101.tar to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465_tpf_lc_Q111111110111011101.tar ... - |==========================================| 43M/ 43M (100.00%) 4s - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/lightcurves/0118/011804465/kplr011804465_lc_Q111111110111011101.tar to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465_lc_Q111111110111011101.tar ... - |==========================================| 5.9M/5.9M (100.00%) 0s - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/lightcurves/0118/011804465/kplr011804465-2009131105131_llc.fits to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-2009131105131_llc.fits ... - |==========================================| 77k/ 77k (100.00%) 0s - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/lightcurves/0118/011804465/kplr011804465-2009166043257_llc.fits to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-2009166043257_llc.fits ... - |==========================================| 192k/192k (100.00%) 0s - Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:KEPLER/url/missions/kepler/lightcurves/0118/011804465/kplr011804465-2009259160929_llc.fits to ./mastDownload/Kepler/kplr011804465_lc_Q111111110111011101/kplr011804465-2009259160929_llc.fits ... - |==========================================| 466k/466k (100.00%) 0s - - >>> print(manifest["Status"]) - Status - -------- - COMPLETE - COMPLETE - COMPLETE - COMPLETE - COMPLETE - COMPLETE - COMPLETE - COMPLETE - COMPLETE - COMPLETE - - >>> Observations.disable_cloud_dataset() Catalog Queries =============== -The Catalogs class provides access to a subset of the astronomical catalogs stored at MAST. The catalogs currently available through this interface are: +The Catalogs class provides access to a subset of the astronomical catalogs stored at MAST. +The catalogs currently available through this interface are: - The Hubble Source Catalog (HSC) - The GALEX Catalog (V2 and V3) @@ -450,184 +588,188 @@ The Catalogs class provides access to a subset of the astronomical catalogs stor - The TESS Input Catalog (TIC) - The TESS Candidate Target List (CTL) - The Disk Detective Catalog -- PanSTARRS (DR1, DR2) +- The PanSTARRS Catalog (DR1 and DR2) +- The All-Sky PLATO Input Catalog (DR1) Positional Queries ------------------ Positional queries can be based on a sky position or a target name. -The returned fields vary by catalog, find the field documentation for specific catalogs `here `__. If no catalog is specified, the Hubble Source Catalog will be queried. - -.. code-block:: python - - >>> from astroquery.mast import Catalogs - - >>> catalog_data = Catalogs.query_object("158.47924 -7.30962", catalog="Galex") - >>> print(catalog_data[:10]) - - distance_arcmin objID survey ... fuv_flux_aper_7 fuv_artifact - --------------- ------------------- ------ ... --------------- ------------ - 0.349380250633 6382034098673685038 AIS ... 0.047751952 0 - 0.76154224886 6382034098672634783 AIS ... -- 0 - 0.924332936617 6382034098672634656 AIS ... -- 0 - 1.16261573926 6382034098672634662 AIS ... -- 0 - 1.26708912875 6382034098672634735 AIS ... -- 0 - 1.4921733955 6382034098674731780 AIS ... 0.0611195639 0 - 1.60512357572 6382034098672634645 AIS ... -- 0 - 1.70541854139 6382034098672634716 AIS ... -- 0 - 1.74637211002 6382034098672634619 AIS ... -- 0 - 1.75244231529 6382034098672634846 AIS ... -- 0 +The returned fields vary by catalog, find the field documentation for specific catalogs +`here `__. +If no catalog is specified, the Hubble Source Catalog will be queried. + +.. doctest-remote-data:: + + >>> from astroquery.mast import Catalogs + ... + >>> catalog_data = Catalogs.query_object("158.47924 -7.30962", catalog="Galex") + >>> print(catalog_data[:10]) + distance_arcmin objID survey ... fuv_flux_aper_7 fuv_artifact + ------------------ ------------------- ------ ... --------------- ------------ + 0.3493802506329695 6382034098673685038 AIS ... 0.047751952 0 + 0.7615422488595471 6382034098672634783 AIS ... -- 0 + 0.9243329366166956 6382034098672634656 AIS ... -- 0 + 1.162615739258038 6382034098672634662 AIS ... -- 0 + 1.2670891287503308 6382034098672634735 AIS ... -- 0 + 1.492173395497916 6382034098674731780 AIS ... 0.0611195639 0 + 1.6051235757244107 6382034098672634645 AIS ... -- 0 + 1.705418541388336 6382034098672634716 AIS ... -- 0 + 1.7463721100195875 6382034098672634619 AIS ... -- 0 + 1.7524423152919317 6382034098672634846 AIS ... -- 0 Some catalogs have a maximum number of results they will return. -If a query results in this maximum number of results a warning will be displayed to alert the user that they might be getting a subset of the true result set. - -.. code-block:: python - - >>> from astroquery.mast import Catalogs - - >>> catalog_data = Catalogs.query_region("322.49324 12.16683", catalog="HSC", magtype=2) - - WARNING: MaxResultsWarning: Maximum catalog results returned, may not include all - sources within radius. [astroquery.mast.core] - - >>> print(catalog_data[:10]) - - MatchID Distance MatchRA ... W3_F160W W3_F160W_Sigma W3_F160W_N - -------- ---------------- ------------- ... -------- -------------- ---------- - 82371983 0.00445549943203 322.493181974 ... -- -- 0 - 82603024 0.006890683763 322.493352058 ... -- -- 0 - 82374767 0.00838818765315 322.49337203 ... -- -- 0 - 82368728 0.0088064912074 322.493272691 ... -- -- 0 - 82371509 0.0104348577531 322.493354352 ... -- -- 0 - 82372543 0.0106808683543 322.493397455 ... -- -- 0 - 82371076 0.0126535758873 322.493089416 ... -- -- 0 - 82367288 0.0130150558411 322.493247548 ... -- -- 0 - 82371086 0.0135993945732 322.493248703 ... -- -- 0 - 82368622 0.0140289292301 322.493101406 ... -- -- 0 - +If a query results in this maximum number of results a warning will be displayed to alert +the user that they might be getting a subset of the true result set. + +.. doctest-remote-data:: + + >>> from astroquery.mast import Catalogs + ... + >>> catalog_data = Catalogs.query_region("322.49324 12.16683", catalog="HSC", magtype=2) # doctest: +SHOW_WARNINGS + InputWarning: Coordinate string is being interpreted as an ICRS coordinate provided in degrees. + MaxResultsWarning: Maximum catalog results returned, may not include all sources within radius. + >>> print(catalog_data[:10]) + MatchID Distance MatchRA ... W3_F160W_MAD W3_F160W_N + --------- -------------------- ------------------ ... ------------ ---------- + 50180585 0.003984902849540913 322.4931746094701 ... nan 0 + 8150896 0.006357935819940561 322.49334740450234 ... nan 0 + 100906349 0.00808206428937523 322.4932839715549 ... nan 0 + 105434103 0.011947078376104195 322.49324000530777 ... nan 0 + 103116183 0.01274757103013683 322.4934207202404 ... nan 0 + 45593349 0.013026569623011767 322.4933878707698 ... nan 0 + 103700905 0.01306760650244682 322.4932769229944 ... nan 0 + 102470085 0.014611879195009472 322.49311034430366 ... nan 0 + 93722307 0.01476438046135455 322.49348351134466 ... nan 0 + 24781941 0.015234351867433582 322.49300148743345 ... nan 0 Radius is an optional parameter and the default is 0.2 degrees. -.. code-block:: python - - >>> from astroquery.mast import Catalogs - - >>> catalog_data = Catalogs.query_object("M10", radius=.02, catalog="TIC") - >>> print(catalog_data[:10]) - - ID ra dec ... duplicate_id priority dstArcSec - --------- ------------- -------------- ... ------------ -------- ------------- - 189844423 254.287989 -4.099644 ... -- -- 2.21043178558 - 189844434 254.286301884 -4.09872352783 ... -- -- 4.69684511346 - 189844449 254.288157 -4.097959 ... -- -- 5.53390173242 - 189844403 254.286864 -4.101237 ... -- -- 7.19103845641 - 189844459 254.286798163 -4.0973143956 ... -- -- 7.63543964382 - 189844400 254.285379 -4.100856 ... -- -- 9.27452417927 - 189844461 254.285647884 -4.09722647575 ... -- -- 9.98427869106 - 189844385 254.289725042 -4.10156744653 ... -- -- 11.4468393777 - 189844419 254.290767 -4.099757 ... -- -- 11.9738216615 - 189844454 254.290349435 -4.09754191392 ... -- -- 12.2100186781 +.. doctest-remote-data:: + + >>> from astroquery.mast import Catalogs + ... + >>> catalog_data = Catalogs.query_object("M10", radius=.02, catalog="TIC") + >>> print(catalog_data[:10]) # doctest: +IGNORE_OUTPUT + ID ra dec ... wdflag dstArcSec + ---------- ---------------- ----------------- ... ------ ------------------ + 510188144 254.287449269816 -4.09954224264168 ... -1 0.7650443624931581 + 510188143 254.28717785824 -4.09908635292493 ... -1 1.3400566638148848 + 189844423 254.287799703996 -4.0994998249247 ... 0 1.3644407138867785 + 1305764031 254.287147439535 -4.09866105132406 ... -1 2.656905409847388 + 1305763882 254.286696117371 -4.09925522448626 ... -1 2.7561196688252894 + 510188145 254.287431890823 -4.10017293344746 ... -1 3.036238557555728 + 1305763844 254.286675148545 -4.09971617257086 ... 0 3.1424781549696217 + 1305764030 254.287249718516 -4.09841883152995 ... -1 3.365991083435227 + 1305764097 254.287599269103 -4.09837925361712 ... -1 3.4590276863989 + 1305764215 254.28820865799 -4.09859677020253 ... -1 3.7675526728257034 The Hubble Source Catalog, the Gaia Catalog, and the PanSTARRS Catalog have multiple versions. An optional version parameter allows you to select which version you want, the default is the highest version. -.. code-block:: python +.. doctest-remote-data:: - >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", radius=0.1, - ... catalog="Gaia", version=2) - >>> print("Number of results:",len(catalog_data)) - >>> print(catalog_data[:4]) - - Number of results: 111 - solution_id designation ... distance - ------------------- ---------------------------- ... ------------------ - 1635721458409799680 Gaia DR2 3774902350511581696 ... 0.6327882551927051 - 1635721458409799680 Gaia DR2 3774901427093274112 ... 0.8438875783827048 - 1635721458409799680 Gaia DR2 3774902148648277248 ... 0.9198397322382648 - 1635721458409799680 Gaia DR2 3774902453590798208 ... 1.3578882400285217 + >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", radius=0.1, + ... catalog="Gaia", version=2) + >>> print("Number of results:",len(catalog_data)) + Number of results: 111 + >>> print(catalog_data[:4]) + solution_id designation ... distance + ------------------- ---------------------------- ... ------------------ + 1635721458409799680 Gaia DR2 3774902350511581696 ... 0.6326770410972467 + 1635721458409799680 Gaia DR2 3774901427093274112 ... 0.8440033390947586 + 1635721458409799680 Gaia DR2 3774902148648277248 ... 0.9199206487344911 + 1635721458409799680 Gaia DR2 3774902453590798208 ... 1.3578181104319944 The PanSTARRS Catalog has multiple data releases as well as multiple queryable tables. An optional data release parameter allows you to select which data release is desired, with the default being the latest version (dr2). The table to query is a required parameter. -.. code-block:: python - - >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", radius=0.1, - >>> catalog="Panstarrs", data_release="dr1", table="mean") - >>> print("Number of results:",len(catalog_data)) - >>> print(catalog_data[:10]) - - Number of results: 7007 - objName objAltName1 objAltName2 ... yMeanApMagNpt yFlags distance - --------------------- ----------- ----------- ... ------------- ------ -------- - PSO J158.4130-07.2557 -999 -999 ... 0 0 0 - PSO J158.4133-07.2564 -999 -999 ... 0 0 0 - PSO J158.4136-07.2571 -999 -999 ... 0 114720 0 - PSO J158.4156-07.2530 -999 -999 ... 0 0 0 - PSO J158.4157-07.2511 -999 -999 ... 0 0 0 - PSO J158.4159-07.2535 -999 -999 ... 0 0 0 - PSO J158.4159-07.2554 -999 -999 ... 0 114720 0 - PSO J158.4160-07.2534 -999 -999 ... 0 114720 0 - PSO J158.4164-07.2568 -999 -999 ... 0 0 0 - PSO J158.4175-07.2574 -999 -999 ... 0 16416 0 +.. doctest-remote-data:: + + >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", radius=0.1, + ... catalog="Panstarrs", data_release="dr1", table="mean") + >>> print("Number of results:",len(catalog_data)) + Number of results: 7007 + >>> print(catalog_data[:10]) # doctest: +IGNORE_OUTPUT + ObjName objAltName1 ... yFlags distance + -------------------------- ----------- ... ------ -------------------- + PSO J103359.653-071622.382 -999 ... 16416 0.04140441098310487 + PSO J103359.605-071622.873 -999 ... 0 0.04121935961328582 + PSO J103359.691-071640.232 -999 ... 0 0.03718729257758985 + PSO J103400.268-071639.192 -999 ... 0 0.03870112803784765 + PSO J103400.073-071637.358 -999 ... 0 0.03867536827891155 + PSO J103359.789-071632.606 -999 ... 0 0.03921557769883566 + PSO J103359.192-071654.790 -999 ... 0 0.03266232705300051 + PSO J103359.959-071655.155 -999 ... 0 0.034361022297827955 + PSO J103359.847-071655.610 -999 ... 0 0.033986082329893995 + PSO J103400.586-071656.646 -999 ... 0 0.035645179491121386 Catalog Criteria Queries ------------------------ The TESS Input Catalog (TIC), Disk Detective Catalog, and PanSTARRS Catalog can also be queried based on non-positional criteria. -.. code-block:: python - - >>> from astroquery.mast import Catalogs - - >>> catalog_data = Catalogs.query_criteria(catalog="Tic",Bmag=[30,50],objType="STAR") - >>> print(catalog_data) - - ID version HIP TYC ... disposition duplicate_id priority objID - --------- -------- --- --- ... ----------- ------------ -------- --------- - 81609218 20171221 -- -- ... -- -- -- 217917514 - 23868624 20171221 -- -- ... -- -- -- 296973171 - 406300991 20171221 -- -- ... -- -- -- 400575018 - - -.. code-block:: python - - >>> from astroquery.mast import Catalogs - - >>> catalog_data = Catalogs.query_criteria(catalog="Ctl", - ... objectname='M101', radius=1, Tmag=[10.75,11]) - >>> print(catalog_data) - ID version HIP TYC ... wdflag ctlPriority objID - --------- -------- --- ------------ ... ------ -------------------- --------- - 441639577 20190415 -- 3852-00429-1 ... 0 0.00138923974233085 150848150 - 441662028 20190415 -- 3855-00941-1 ... 0 0.00100773800289492 151174508 - 233458861 20190415 -- 3852-01407-1 ... 0 0.000843468567169446 151169732 - 441658008 20190415 -- 3852-00116-1 ... 0 0.000337697695047815 151025336 - 154258521 20190415 -- 3852-01403-1 ... 0 0.000791883530388075 151060938 - 441658179 20190415 -- 3855-00816-1 ... 0 0.000933466312394693 151025457 - 441659970 20190415 -- 3852-00505-1 ... 0 0.000894696498704202 151075682 - 441660006 20190415 -- 3852-00341-1 ... 0 0.000600037898043061 151075713 - - -.. code-block:: python - - >>> from astroquery.mast import Catalogs - - >>> catalog_data = Catalogs.query_criteria(catalog="DiskDetective", - ... objectname="M10",radius=2,state="complete") - >>> print(catalog_data) - - designation ... ZooniverseURL - ------------------- ... ---------------------------------------------------- - J165628.40-054630.8 ... https://talk.diskdetective.org/#/subjects/AWI0005cka - J165748.96-054915.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ckd - J165427.11-022700.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ck5 - J165749.79-040315.1 ... https://talk.diskdetective.org/#/subjects/AWI0005cke - J165327.01-042546.2 ... https://talk.diskdetective.org/#/subjects/AWI0005ck3 - J165949.90-054300.7 ... https://talk.diskdetective.org/#/subjects/AWI0005ckk - J170314.11-035210.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ckv +.. doctest-remote-data:: + + >>> from astroquery.mast import Catalogs + ... + >>> catalog_data = Catalogs.query_criteria(catalog="Tic",Bmag=[30,50],objType="STAR") + >>> print(catalog_data) # doctest: +IGNORE_OUTPUT + ID version HIP TYC ... e_Dec_orig raddflag wdflag objID + --------- -------- --- --- ... ------------------ -------- ------ ---------- + 125413929 20190415 -- -- ... 0.293682765259495 1 0 579825059 + 261459129 20190415 -- -- ... 0.200397148604244 1 0 1701625107 + 64575709 20190415 -- -- ... 0.21969663115091 1 0 595775997 + 94322581 20190415 -- -- ... 0.205286802302475 1 0 606092549 + 125414201 20190415 -- -- ... 0.22398993783274 1 0 579825329 + 463721073 20190415 -- -- ... 0.489828592248652 -1 1 710312391 + 81609218 20190415 -- -- ... 0.146788572369267 1 0 630541794 + 282024596 20190415 -- -- ... 0.548806522539047 1 0 573765450 + 23868624 20190415 -- -- ... 355.949 -- 0 916384285 + 282391528 20190415 -- -- ... 0.47766300834538 0 0 574723760 + 123585000 20190415 -- -- ... 0.618316068787371 0 0 574511442 + 260216294 20190415 -- -- ... 0.187170498094167 1 0 683390717 + 406300991 20190415 -- -- ... 0.0518318978617112 0 0 1411465651 + + +.. doctest-remote-data:: + + >>> from astroquery.mast import Catalogs + ... + >>> catalog_data = Catalogs.query_criteria(catalog="Ctl", + ... objectname='M101', radius=1, Tmag=[10.75,11]) + >>> print(catalog_data) + ID version HIP TYC ... raddflag wdflag objID + --------- -------- --- ------------ ... -------- ------ --------- + 441639577 20190415 -- 3852-00429-1 ... 1 0 150070672 + 441658179 20190415 -- 3855-00816-1 ... 1 0 150246482 + 441658008 20190415 -- 3852-00116-1 ... 1 0 150246361 + 154258521 20190415 -- 3852-01403-1 ... 1 0 150281963 + 441659970 20190415 -- 3852-00505-1 ... 1 0 150296707 + 441660006 20190415 -- 3852-00341-1 ... 1 0 150296738 + 233458861 20190415 -- 3852-01407-1 ... 1 0 150390757 + 441662028 20190415 -- 3855-00941-1 ... 1 0 150395533 + + +.. doctest-remote-data:: + + >>> from astroquery.mast import Catalogs + ... + >>> catalog_data = Catalogs.query_criteria(catalog="DiskDetective", + ... objectname="M10",radius=2,state="complete") + >>> print(catalog_data) # doctest: +IGNORE_OUTPUT + designation ... ZooniverseURL + ------------------- ... ---------------------------------------------------- + J165628.40-054630.8 ... https://talk.diskdetective.org/#/subjects/AWI0005cka + J165748.96-054915.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ckd + J165427.11-022700.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ck5 + J165749.79-040315.1 ... https://talk.diskdetective.org/#/subjects/AWI0005cke + J165327.01-042546.2 ... https://talk.diskdetective.org/#/subjects/AWI0005ck3 + J165949.90-054300.7 ... https://talk.diskdetective.org/#/subjects/AWI0005ckk + J170314.11-035210.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ckv The PanSTARRS catalog also accepts additional parameters to allow for query refinement. These options include column selection, @@ -644,27 +786,26 @@ column name criteria, and the 'OR' operation is performed within column name cri parameter, criteria may consist of either a value or a list. The list may consist of a mix of values and tuples of criteria decorator (min, gte, gt, max, lte, lt, like, contains) and value. -.. code-block:: python - - >>> catalog_data = Catalogs.query_criteria(coordinates="5.97754 32.53617", radius=0.01, - ... catalog="PANSTARRS", table="mean", data_release="dr2", - ... nStackDetections=[("gte", 2)], - ... columns=["objName", "objID", "nStackDetections", "distance"], - ... sort_by=[("desc", "distance")], pagesize=15) - >>> print(catalog_data[:10]) - - objName objID nStackDetections distance - --------------------- ------------------ ---------------- --------------------- - PSO J005.9812+32.5270 147030059812483022 5 0.009651200148871086 - PSO J005.9726+32.5278 147030059727583992 2 0.0093857181370567 - PSO J005.9787+32.5453 147050059787164914 4 0.009179045509852305 - PSO J005.9722+32.5418 147050059721440704 4 0.007171813230776031 - PSO J005.9857+32.5377 147040059855825725 4 0.007058815429178634 - PSO J005.9810+32.5424 147050059809651427 2 0.006835678269917365 - PSO J005.9697+32.5368 147040059697224794 2 0.006654002479439699 - PSO J005.9712+32.5330 147040059711340087 4 0.006212461367287632 - PSO J005.9747+32.5413 147050059747400181 5 0.0056515210592035965 - PSO J005.9775+32.5314 147030059774678271 3 0.004739286624336443 +.. doctest-remote-data:: + + >>> catalog_data = Catalogs.query_criteria(coordinates="5.97754 32.53617", radius=0.01, + ... catalog="PANSTARRS", table="mean", data_release="dr2", + ... nStackDetections=[("gte", 2)], + ... columns=["objName", "objID", "nStackDetections", "distance"], + ... sort_by=[("desc", "distance")], pagesize=15) + >>> print(catalog_data[:10]) # doctest: +IGNORE_OUTPUT + objName objID nStackDetections distance + --------------------- ------------------ ---------------- --------------------- + PSO J005.9812+32.5270 147030059812483022 5 0.009651200148871086 + PSO J005.9726+32.5278 147030059727583992 2 0.0093857181370567 + PSO J005.9787+32.5453 147050059787164914 4 0.009179045509852305 + PSO J005.9722+32.5418 147050059721440704 4 0.007171813230776031 + PSO J005.9857+32.5377 147040059855825725 4 0.007058815429178634 + PSO J005.9810+32.5424 147050059809651427 2 0.006835678269917365 + PSO J005.9697+32.5368 147040059697224794 2 0.006654002479439699 + PSO J005.9712+32.5330 147040059711340087 4 0.006212461367287632 + PSO J005.9747+32.5413 147050059747400181 5 0.0056515210592035965 + PSO J005.9775+32.5314 147030059774678271 3 0.004739286624336443 Hubble Source Catalog (HSC) specific queries @@ -672,76 +813,71 @@ Hubble Source Catalog (HSC) specific queries Given an HSC Match ID, return all catalog results. -.. code-block:: python - - >>> from astroquery.mast import Catalogs - - >>> catalog_data = Catalogs.query_object("M10", radius=.02, catalog="HSC") - >>> matchid = catalog_data[0]["MatchID"] - >>> print(matchid) - - 17554326 - - >>> matches = Catalogs.query_hsc_matchid(matchid) - >>> print(matches) - - CatID MatchID ... cd_matrix - --------- -------- ... ------------------------------------------------------ - 303940283 17554326 ... -1.10059e-005 6.90694e-010 6.90694e-010 1.10059e-005 - 303936256 17554326 ... -1.10059e-005 6.90694e-010 6.90694e-010 1.10059e-005 - 303938261 17554326 ... -1.10059e-005 6.90694e-010 6.90694e-010 1.10059e-005 - 301986299 17554326 ... -1.10049e-005 -1.6278e-010 -1.6278e-010 1.10049e-005 - 301988274 17554326 ... -1.10049e-005 -1.6278e-010 -1.6278e-010 1.10049e-005 - 301990418 17554326 ... -1.10049e-005 -1.6278e-010 -1.6278e-010 1.10049e-005 - 206511399 17554326 ... -1.38889e-005 -1.36001e-009 -1.36001e-009 1.38889e-005 - 206507082 17554326 ... -1.38889e-005 -1.36001e-009 -1.36001e-009 1.38889e-005 - - -HSC spectra accessed through this class as well. `~astroquery.mast.CatalogsClass.get_hsc_spectra` does not take any arguments, and simply loads all HSC spectra. - -.. code-block:: python - - >>> from astroquery.mast import Catalogs - - >>> all_spectra = Catalogs.get_hsc_spectra() - >>> print(all_spectra[:10]) - - ObjID DatasetName MatchID ... PropID HSCMatch - ----- -------------------------------------------- -------- ... ------ -------- - 20010 HAG_J072655.67+691648.9_J8HPAXAEQ_V01.SPEC1D 19657846 ... 9482 Y - 20011 HAG_J072655.69+691648.9_J8HPAOZMQ_V01.SPEC1D 19657846 ... 9482 Y - 20012 HAG_J072655.76+691729.7_J8HPAOZMQ_V01.SPEC1D 19659745 ... 9482 Y - 20013 HAG_J072655.82+691620.0_J8HPAOZMQ_V01.SPEC1D 19659417 ... 9482 Y - 20014 HAG_J072656.34+691704.7_J8HPAXAEQ_V01.SPEC1D 19660230 ... 9482 Y - 20015 HAG_J072656.36+691704.7_J8HPAOZMQ_V01.SPEC1D 19660230 ... 9482 Y - 20016 HAG_J072656.36+691744.9_J8HPAOZMQ_V01.SPEC1D 19658847 ... 9482 Y - 20017 HAG_J072656.37+691630.2_J8HPAXAEQ_V01.SPEC1D 19660827 ... 9482 Y - 20018 HAG_J072656.39+691630.2_J8HPAOZMQ_V01.SPEC1D 19660827 ... 9482 Y - 20019 HAG_J072656.41+691734.9_J8HPAOZMQ_V01.SPEC1D 19656620 ... 9482 Y - - -Individual or ranges of spectra can be downloaded using the `~astroquery.mast.CatalogsClass.download_hsc_spectra` function. - -.. code-block:: python - - >>> from astroquery.mast import Catalogs - - >>> all_spectra = Catalogs.get_hsc_spectra() - >>> manifest = Catalogs.download_hsc_spectra(all_spectra[100:104]) - - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - - >>> print(manifest) - - Local Path ... URL - -------------------------------------------------------------------- ... ---- - ./mastDownload/HSC/HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits ... None - ./mastDownload/HSC/HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits ... None - ./mastDownload/HSC/HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits ... None - ./mastDownload/HSC/HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits ... None +.. doctest-remote-data:: + + >>> from astroquery.mast import Catalogs + ... + >>> catalog_data = Catalogs.query_object("M10", radius=.02, catalog="HSC") + >>> matchid = catalog_data[0]["MatchID"] + >>> print(matchid) + 63980492 + >>> matches = Catalogs.query_hsc_matchid(matchid) + >>> print(matches) + CatID MatchID ... cd_matrix + --------- -------- ... ------------------------------------------------------ + 257195287 63980492 ... -1.38889e-005 -5.26157e-010 -5.26157e-010 1.38889e-005 + 257440119 63980492 ... -1.38889e-005 -5.26157e-010 -5.26157e-010 1.38889e-005 + 428373428 63980492 ... -1.10056e-005 5.65193e-010 5.65193e-010 1.10056e-005 + 428373427 63980492 ... -1.10056e-005 5.65193e-010 5.65193e-010 1.10056e-005 + 428373429 63980492 ... -1.10056e-005 5.65193e-010 5.65193e-010 1.10056e-005 + 410574499 63980492 ... -1.10056e-005 1.56577e-009 1.56577e-009 1.10056e-005 + 410574498 63980492 ... -1.10056e-005 1.56577e-009 1.56577e-009 1.10056e-005 + 410574497 63980492 ... -1.10056e-005 1.56577e-009 1.56577e-009 1.10056e-005 + +HSC spectra accessed through this class as well. `~astroquery.mast.CatalogsClass.get_hsc_spectra` +does not take any arguments, and simply loads all HSC spectra. + +.. doctest-remote-data:: + + >>> from astroquery.mast import Catalogs + ... + >>> all_spectra = Catalogs.get_hsc_spectra() + >>> print(all_spectra[:10]) + ObjID DatasetName MatchID ... PropID HSCMatch + ----- -------------------------------------------- -------- ... ------ -------- + 20010 HAG_J072655.67+691648.9_J8HPAXAEQ_V01.SPEC1D 19657846 ... 9482 Y + 20011 HAG_J072655.69+691648.9_J8HPAOZMQ_V01.SPEC1D 19657846 ... 9482 Y + 20012 HAG_J072655.76+691729.7_J8HPAOZMQ_V01.SPEC1D 19659745 ... 9482 Y + 20013 HAG_J072655.82+691620.0_J8HPAOZMQ_V01.SPEC1D 19659417 ... 9482 Y + 20014 HAG_J072656.34+691704.7_J8HPAXAEQ_V01.SPEC1D 19660230 ... 9482 Y + 20015 HAG_J072656.36+691704.7_J8HPAOZMQ_V01.SPEC1D 19660230 ... 9482 Y + 20016 HAG_J072656.36+691744.9_J8HPAOZMQ_V01.SPEC1D 19658847 ... 9482 Y + 20017 HAG_J072656.37+691630.2_J8HPAXAEQ_V01.SPEC1D 19660827 ... 9482 Y + 20018 HAG_J072656.39+691630.2_J8HPAOZMQ_V01.SPEC1D 19660827 ... 9482 Y + 20019 HAG_J072656.41+691734.9_J8HPAOZMQ_V01.SPEC1D 19656620 ... 9482 Y + + +Individual or ranges of spectra can be downloaded using the +`~astroquery.mast.CatalogsClass.download_hsc_spectra` function. + +.. doctest-remote-data:: + + >>> from astroquery.mast import Catalogs + ... + >>> all_spectra = Catalogs.get_hsc_spectra() + >>> manifest = Catalogs.download_hsc_spectra(all_spectra[100:104]) # doctest: +IGNORE_OUTPUT + Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] + Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] + Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] + Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] + ... + >>> print(manifest) # doctest: +IGNORE_OUTPUT + Local Path ... URL + -------------------------------------------------------------------- ... ---- + ./mastDownload/HSC/HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits ... None + ./mastDownload/HSC/HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits ... None + ./mastDownload/HSC/HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits ... None + ./mastDownload/HSC/HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits ... None TESSCut @@ -763,93 +899,136 @@ https://ui.adsabs.harvard.edu/abs/2019ascl.soft05007B/abstract Cutouts ------- -The `~astroquery.mast.TesscutClass.get_cutouts` function takes a coordinate or object name -(such as "M104" or "TIC 32449963") and cutout size (in pixels or an angular quantity) and -returns the cutout target pixel file(s) as a list of `~astropy.io.fits.HDUList` objects. +The `~astroquery.mast.TesscutClass.get_cutouts` function takes a coordinate, object name +(i.e. "M104" or "TIC 32449963"), or moving target (i.e. "Eleonora") and cutout size +(in pixels or an angular quantity) and returns the cutout target pixel file(s) as a +list of `~astropy.io.fits.HDUList` objects. If the given coordinate/object location appears in more than one TESS sector a target pixel file will be produced for each sector. If the cutout area overlaps more than one camera or ccd a target pixel file will be produced for each one. -.. code-block:: python - - >>> from astroquery.mast import Tesscut - >>> from astropy.coordinates import SkyCoord - - >>> cutout_coord = SkyCoord(107.18696, -70.50919, unit="deg") - >>> hdulist = Tesscut.get_cutouts(coordinates=cutout_coord, size=5) - >>> hdulist[0].info() - Filename: - No. Name Ver Type Cards Dimensions Format - 0 PRIMARY 1 PrimaryHDU 55 () - 1 PIXELS 1 BinTableHDU 279 1282R x 12C [D, E, J, 25J, 25E, 25E, 25E, 25E, J, E, E, 38A] - 2 APERTURE 1 ImageHDU 79 (5, 5) int32 - +Requesting a cutout by coordinate or objectname accesses the +`MAST TESScut API `__ +and returns a target pixel file, with format described +`here `__. + +.. doctest-remote-data:: + + >>> from astroquery.mast import Tesscut + >>> from astropy.coordinates import SkyCoord + ... + >>> cutout_coord = SkyCoord(107.18696, -70.50919, unit="deg") + >>> hdulist = Tesscut.get_cutouts(coordinates=cutout_coord, size=5) + >>> hdulist[0].info() # doctest: +IGNORE_OUTPUT + Filename: + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 56 () + 1 PIXELS 1 BinTableHDU 280 1196R x 12C [D, E, J, 25J, 25E, 25E, 25E, 25E, J, E, E, 38A] + 2 APERTURE 1 ImageHDU 81 (5, 5) int32 + + +.. doctest-remote-data:: + + >>> from astroquery.mast import Tesscut + ... + >>> hdulist = Tesscut.get_cutouts(objectname="TIC 32449963", size=5) + >>> hdulist[0].info() # doctest: +IGNORE_OUTPUT + Filename: + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 56 () + 1 PIXELS 1 BinTableHDU 280 3477R x 12C [D, E, J, 25J, 25E, 25E, 25E, 25E, J, E, E, 38A] + 2 APERTURE 1 ImageHDU 81 (5, 5) int32 + + +Requesting a cutout by moving_target accesses the +`MAST Moving Target TESScut API `__ +and returns a target pixel file, with format described +`here `__. +The moving_target is an optional bool argument where `True` signifies that the accompanying ``objectname`` +input is the object name or ID understood by the +`JPL Horizon ephemerades interface `__. +The default value for moving_target is set to False. Therefore, a non-moving target can be input +simply with either the objectname or coordinates. + +.. doctest-remote-data:: + + >>> from astroquery.mast import Tesscut + ... + >>> hdulist = Tesscut.get_cutouts(objectname="Eleonora", moving_target=True, size=5, sector=6) + >>> hdulist[0].info() # doctest: +IGNORE_OUTPUT + Filename: + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 54 () + 1 PIXELS 1 BinTableHDU 150 355R x 16C [D, E, J, 25J, 25E, 25E, 25E, 25E, J, E, E, 38A, D, D, D, D] + 2 APERTURE 1 ImageHDU 97 (2136, 2078) int32 + + +The `~astroquery.mast.TesscutClass.download_cutouts` function takes a coordinate, cutout size +(in pixels or an angular quantity), or object name (i.e. "M104" or "TIC 32449963") and moving target +(True or False). It uses these parameters to download the cutout target pixel file(s). + +If a given coordinate/object/moving target appears in more than one TESS sector, a target pixel file +will be produced for each sector. If the cutout area overlaps more than one camera or ccd, a target +pixel file will be produced for each one. + +.. doctest-remote-data:: + + >>> from astroquery.mast import Tesscut + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + ... + >>> cutout_coord = SkyCoord(107.18696, -70.50919, unit="deg") + >>> manifest = Tesscut.download_cutouts(coordinates=cutout_coord, size=[5, 7]*u.arcmin, sector=9) # doctest: +IGNORE_OUTPUT + Downloading URL https://mast.stsci.edu/tesscut/api/v0.1/astrocut?ra=107.18696&dec=-70.50919&y=0.08333333333333333&x=0.11666666666666667&units=d§or=9 to ./tesscut_20210716150026.zip ... [Done] + Inflating... + ... + >>> print(manifest) # doctest: +IGNORE_OUTPUT + Local Path + ---------------------------------------------------------- + ./tess-s0009-4-1_107.186960_-70.509190_21x15_astrocut.fits -.. code-block:: python - - >>> from astroquery.mast import Tesscut - - >>> hdulist = Tesscut.get_cutouts(objectname="TIC 32449963", size=5) - >>> hdulist[0].info() - Filename: - No. Name Ver Type Cards Dimensions Format - 0 PRIMARY 1 PrimaryHDU 56 () - 1 PIXELS 1 BinTableHDU 280 1211R x 12C [D, E, J, 25J, 25E, 25E, 25E, 25E, J, E, E, 38A] - 2 APERTURE 1 ImageHDU 80 (5, 5) int32 - - -The `~astroquery.mast.TesscutClass.download_cutouts` function takes a coordinate or object name -(such as "M104" or "TIC 32449963") and cutout size (in pixels or an angular quantity) and -downloads the cutout target pixel file(s). - -If a given coordinate appears in more than one TESS sector a target pixel file will be -produced for each sector. If the cutout area overlaps more than one camera or ccd -a target pixel file will be produced for each one. - -.. code-block:: python - - >>> from astroquery.mast import Tesscut - >>> from astropy.coordinates import SkyCoord - >>> import astropy.units as u - - >>> cutout_coord = SkyCoord(107.18696, -70.50919, unit="deg") - >>> manifest = Tesscut.download_cutouts(coordinates=cutout_coord, size=[5, 7]*u.arcmin) - Downloading URL https://mast.stsci.edu/tesscut/api/v0.1/astrocut?ra=107.18696&dec=-70.50919&y=0.08333333333333333&x=0.11666666666666667&units=d§or=1 to ./tesscut_20181102104719.zip ... [Done] - Inflating... - - >>> print(manifest) - local_file - ------------------------------------------------------ - ./tess-s0001-4-3_107.18696_-70.50919_14x21_astrocut.fits Sector information ------------------ -To access sector information at a particular location there is `~astroquery.mast.TesscutClass.get_sectors`. +To access sector information for a particular coordinate, object, or moving target there is +`~astroquery.mast.TesscutClass.get_sectors`. -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.mast import Tesscut - >>> from astropy.coordinates import SkyCoord + >>> from astroquery.mast import Tesscut + >>> from astropy.coordinates import SkyCoord + ... + >>> coord = SkyCoord(324.24368, -27.01029,unit="deg") + >>> sector_table = Tesscut.get_sectors(coordinates=coord) + >>> print(sector_table) # doctest: +IGNORE_OUTPUT + sectorName sector camera ccd + -------------- ------ ------ --- + tess-s0028-1-4 28 1 4 - >>> coord = SkyCoord(324.24368, -27.01029,unit="deg") - >>> sector_table = Tesscut.get_sectors(coordinates=coord) - >>> print(sector_table) - sectorName sector camera ccd - -------------- ------ ------ --- - tess-s0001-1-3 1 1 3 +.. doctest-remote-data:: + >>> from astroquery.mast import Tesscut + ... + >>> sector_table = Tesscut.get_sectors(objectname="TIC 32449963") + >>> print(sector_table) # doctest: +IGNORE_OUTPUT + sectorName sector camera ccd + -------------- ------ ------ --- + tess-s0010-1-4 10 1 4 -.. code-block:: python - >>> from astroquery.mast import Tesscut +.. doctest-remote-data:: - >>> sector_table = Tesscut.get_sectors(objectname="TIC 32449963") - >>> print(sector_table) - sectorName sector camera ccd - -------------- ------ ------ --- - tess-s0010-1-4 10 1 4 + >>> from astroquery.mast import Tesscut + ... + >>> sector_table = Tesscut.get_sectors(objectname="Ceres", moving_target=True) + >>> print(sector_table) + sectorName sector camera ccd + -------------- ------ ------ --- + tess-s0029-1-4 29 1 4 + tess-s0043-3-3 43 3 3 + tess-s0044-2-4 44 2 4 Zcut @@ -870,20 +1049,20 @@ an angular quantity) and returns the cutout FITS file(s) as a list of ~astropy.i If the given coordinate appears in more than one Zcut survey, a FITS file will be produced for each survey. -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.mast import Zcut - >>> from astropy.coordinates import SkyCoord - - >>> cutout_coord = SkyCoord(189.49206, 62.20615, unit="deg") - >>> hdulist = Zcut.get_cutouts(coordinates=cutout_coord, size=5) - >>> hdulist[0].info() - Filename: - No. Name Ver Type Cards Dimensions Format - 0 PRIMARY 1 PrimaryHDU 11 () - 1 CUTOUT 1 ImageHDU 177 (5, 5) float32 - 2 CUTOUT 1 ImageHDU 177 (5, 5) float32 - 3 CUTOUT 1 ImageHDU 177 (5, 5) float32 + >>> from astroquery.mast import Zcut + >>> from astropy.coordinates import SkyCoord + ... + >>> cutout_coord = SkyCoord(189.49206, 62.20615, unit="deg") + >>> hdulist = Zcut.get_cutouts(coordinates=cutout_coord, size=5) + >>> hdulist[0].info() # doctest: +IGNORE_OUTPUT + Filename: + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 11 () + 1 CUTOUT 1 ImageHDU 177 (5, 5) float32 + 2 CUTOUT 1 ImageHDU 177 (5, 5) float32 + 3 CUTOUT 1 ImageHDU 177 (5, 5) float32 The `~astroquery.mast.ZcutClass.download_cutouts` function takes a coordinate and cutout size (in pixels or @@ -892,36 +1071,37 @@ files. If a given coordinate appears in more than one Zcut survey, a cutout will be produced for each survey. -.. code-block:: python - - >>> from astroquery.mast import Zcut - >>> from astropy.coordinates import SkyCoord - - >>> cutout_coord = SkyCoord(189.49206, 62.20615, unit="deg") - >>> manifest = Zcut.download_cutouts(coordinates=cutout_coord, size=[200, 300], units="px") - Downloading URL https://mast.stsci.edu/zcut/api/v0.1/astrocut?ra=189.49206&dec=62.20615&y=200&x=300&units=px&format=fits to ./zcut_20201202132247.zip ... [Done] - - >>> print(manifest) - Local Path - ------------------------------------------------------------------------- - ./candels_gn_30mas_189.492060_62.206150_300.0pix-x-200.0pix_astrocut.fits - - -.. code-block:: python - - >>> from astroquery.mast import Zcut - >>> from astropy.coordinates import SkyCoord - - >>> cutout_coord = SkyCoord(189.49206, 62.20615, unit="deg") - >>> manifest = Zcut.download_cutouts(coordinates=cutout_coord, size=[200, 300], units="px", form="jpg") - Downloading URL https://mast.stsci.edu/zcut/api/v0.1/astrocut?ra=189.49206&dec=62.20615&y=200&x=300&units=px&format=jpg to ./zcut_20201202132453.zip ... [Done] - - >>> print(manifest) - Local Path - --------------------------------------------------------------------------------------------------------- - ./hlsp_candels_hst_acs_gn-tot-30mas_f606w_v1.0_drz_189.492060_62.206150_300.0pix-x-200.0pix_astrocut.jpg - ./hlsp_candels_hst_acs_gn-tot-30mas_f814w_v1.0_drz_189.492060_62.206150_300.0pix-x-200.0pix_astrocut.jpg - ./hlsp_candels_hst_acs_gn-tot-30mas_f850lp_v1.0_drz_189.492060_62.206150_300.0pix-x-200.0pix_astrocut.jpg +.. doctest-remote-data:: + + >>> from astroquery.mast import Zcut + >>> from astropy.coordinates import SkyCoord + ... + >>> cutout_coord = SkyCoord(189.49206, 62.20615, unit="deg") + >>> manifest = Zcut.download_cutouts(coordinates=cutout_coord, size=[200, 300], units="px") # doctest: +IGNORE_OUTPUT + Downloading URL https://mast.stsci.edu/zcut/api/v0.1/astrocut?ra=189.49206&dec=62.20615&y=200&x=300&units=px&format=fits to ./zcut_20210125155545.zip ... [Done] + Inflating... + ... + >>> print(manifest) # doctest: +IGNORE_OUTPUT + Local Path + ------------------------------------------------------------------------- + ./candels_gn_30mas_189.492060_62.206150_300.0pix-x-200.0pix_astrocut.fits + + +.. doctest-remote-data:: + + >>> from astroquery.mast import Zcut + >>> from astropy.coordinates import SkyCoord + ... + >>> cutout_coord = SkyCoord(189.49206, 62.20615, unit="deg") + >>> manifest = Zcut.download_cutouts(coordinates=cutout_coord, size=[200, 300], units="px", form="jpg") # doctest: +IGNORE_OUTPUT + Downloading URL https://mast.stsci.edu/zcut/api/v0.1/astrocut?ra=189.49206&dec=62.20615&y=200&x=300&units=px&format=jpg to ./zcut_20201202132453.zip ... [Done] + ... + >>> print(manifest) # doctest: +IGNORE_OUTPUT + Local Path + --------------------------------------------------------------------------------------------------------- + ./hlsp_candels_hst_acs_gn-tot-30mas_f606w_v1.0_drz_189.492060_62.206150_300.0pix-x-200.0pix_astrocut.jpg + ./hlsp_candels_hst_acs_gn-tot-30mas_f814w_v1.0_drz_189.492060_62.206150_300.0pix-x-200.0pix_astrocut.jpg + ./hlsp_candels_hst_acs_gn-tot-30mas_f850lp_v1.0_drz_189.492060_62.206150_300.0pix-x-200.0pix_astrocut.jpg Survey information @@ -929,15 +1109,15 @@ Survey information To list the available deep field surveys at a particular location there is `~astroquery.mast.ZcutClass.get_surveys`. -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.mast import Zcut - >>> from astropy.coordinates import SkyCoord - - >>> coord = SkyCoord(189.49206, 62.20615, unit="deg") - >>> survey_list = Zcut.get_surveys(coordinates=coord) - >>> print(survey_list) - ['candels_gn_60mas', 'candels_gn_30mas', 'goods_north'] + >>> from astroquery.mast import Zcut + >>> from astropy.coordinates import SkyCoord + ... + >>> coord = SkyCoord(189.49206, 62.20615, unit="deg") + >>> survey_list = Zcut.get_surveys(coordinates=coord) + >>> print(survey_list) # doctest: +IGNORE_OUTPUT + ['candels_gn_60mas', 'candels_gn_30mas', 'goods_north'] Accessing Proprietary Data @@ -952,32 +1132,28 @@ If a token is not supplied, the user will be prompted to enter one. To view tokens accessible through your account, visit https://auth.mast.stsci.edu -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> Observations.login(token="12348r9w0sa2392ff94as841") - - INFO: MAST API token accepted, welcome User Name [astroquery.mast.core] - - >>> sessioninfo = Observations.session_info() - - eppn: user_name@stsci.edu - ezid: uname - ... - -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> my_session = Observations(token="12348r9w0sa2392ff94as841") - - INFO: MAST API token accepted, welcome User Name [astroquery.mast.core] - - >>> sessioninfo = Observations.session_info() - - eppn: user_name@stsci.edu - ezid: uname +.. doctest-skip:: + + >>> from astroquery.mast import Observations + ... + >>> Observations.login(token="12348r9w0sa2392ff94as841") + INFO: MAST API token accepted, welcome User Name [astroquery.mast.core] + ... + >>> sessioninfo = Observations.session_info() + eppn: user_name@stsci.edu + ezid: uname + ... + +.. doctest-skip:: + + >>> from astroquery.mast import Observations + ... + >>> my_session = Observations(token="12348r9w0sa2392ff94as841") + INFO: MAST API token accepted, welcome User Name [astroquery.mast.core] + ... + >>> sessioninfo = Observations.session_info() + eppn: user_name@stsci.edu + ezid: uname ... \* For security tokens should not be typed into a terminal or Jupyter notebook @@ -1004,55 +1180,68 @@ astroquery, this class does allow access. See the `MAST api documentation The basic MAST query function returns query results as an `~astropy.table.Table`. -.. code-block:: python - - >>> from astroquery.mast import Mast - - >>> service = 'Mast.Caom.Cone' - >>> params = {'ra':184.3, - ... 'dec':54.5, - ... 'radius':0.2} - - >>> observations = Mast.service_request(service, params) - >>> print(observations) - - dataproduct_type obs_collection instrument_name ... distance _selected_ - ---------------- -------------- --------------- ... ------------- ---------- - image GALEX GALEX ... 0.0 False - image GALEX GALEX ... 0.0 False - image GALEX GALEX ... 0.0 False - image GALEX GALEX ... 0.0 False - image GALEX GALEX ... 0.0 False - image GALEX GALEX ... 302.405835798 False - image GALEX GALEX ... 302.405835798 False +.. doctest-remote-data:: + + >>> from astroquery.mast import Mast + ... + >>> service = 'Mast.Caom.Cone' + >>> params = {'ra':184.3, + ... 'dec':54.5, + ... 'radius':0.2} + >>> observations = Mast.service_request(service, params) + >>> print(observations) # doctest: +IGNORE_OUTPUT + intentType obs_collection provenance_name ... obsid distance + ---------- -------------- --------------- ... ----------- ------------------ + science TESS SPOC ... 17001016097 0.0 + science TESS SPOC ... 17000855562 0.0 + science TESS SPOC ... 17000815577 203.70471189751947 + science TESS SPOC ... 17000981417 325.4085155315165 + science TESS SPOC ... 17000821493 325.4085155315165 + science PS1 3PI ... 16000864847 0.0 + science PS1 3PI ... 16000864848 0.0 + science PS1 3PI ... 16000864849 0.0 + science PS1 3PI ... 16000864850 0.0 + science PS1 3PI ... 16000864851 0.0 + ... ... ... ... ... ... + science HLSP QLP ... 18013987996 637.806560287869 + science HLSP QLP ... 18007518640 637.806560287869 + science HLSP TESS-SPOC ... 18013510950 637.806560287869 + science HLSP TESS-SPOC ... 18007364076 637.806560287869 + science GALEX MIS ... 1000007123 0.0 + science GALEX AIS ... 1000016562 0.0 + science GALEX AIS ... 1000016562 0.0 + science GALEX AIS ... 1000016563 0.0 + science GALEX AIS ... 1000016563 0.0 + science GALEX AIS ... 1000016556 302.4058357983673 + science GALEX AIS ... 1000016556 302.4058357983673 + Length = 77 rows If the output is not the MAST json result type it cannot be properly parsed into a `~astropy.table.Table`. In this case, the async method should be used to get the raw http response, which can then be manually parsed. -.. code-block:: python - - >>> from astroquery.mast import Mast - - >>> service = 'Mast.Name.Lookup' - >>> params ={'input':"M8", - ... 'format':'json'} - - >>> response = Mast.service_request_async(service,params) - >>> result = response[0].json() - >>> print(result) - - {'resolvedCoordinate': [{'cacheDate': 'Apr 12, 2017 9:28:24 PM', - 'cached': True, - 'canonicalName': 'MESSIER 008', - 'decl': -24.38017, - 'objectType': 'Neb', - 'ra': 270.92194, - 'resolver': 'NED', - 'resolverTime': 113, - 'searchRadius': -1.0, - 'searchString': 'm8'}], - 'status': ''} +.. doctest-remote-data:: + + >>> from astroquery.mast import Mast + ... + >>> service = 'Mast.Name.Lookup' + >>> params ={'input':"M8", + ... 'format':'json'} + ... + >>> response = Mast.service_request_async(service,params) + >>> result = response[0].json() + >>> print(result) # doctest: +IGNORE_OUTPUT + {'resolvedCoordinate': [{'cacheDate': 'Apr 12, 2017 9:28:24 PM', + 'cached': True, + 'canonicalName': 'MESSIER 008', + 'decl': -24.38017, + 'objectType': 'Neb', + 'ra': 270.92194, + 'resolver': 'NED', + 'resolverTime': 113, + 'searchRadius': -1.0, + 'searchString': 'm8'}], + 'status': ''} diff --git a/docs/mpc/mpc.rst b/docs/mpc/mpc.rst index 7eee7adee8..0a5cb16b09 100644 --- a/docs/mpc/mpc.rst +++ b/docs/mpc/mpc.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.mpc: ************************************************************************* @@ -21,86 +19,94 @@ Planet Center (MPC). Three services are available: - `Minor Planet Center Observations Database `_ for obtaining observations of asteroids and comets reported to the MPC - + In addition, the module provides access to the MPC's hosted list of `IAU Observatory Codes `__. To return the orbit of Ceres and an ephemeris for the next 20 days: -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.mpc import MPC >>> from pprint import pprint >>> result = MPC.query_object('asteroid', name='ceres') - >>> pprint(result) - - [{'absolute_magnitude': '3.34', - 'aphelion_distance': '2.976', - 'arc_length': 79346, - 'argument_of_perihelion': '73.11528', - 'ascending_node': '80.309916', + >>> pprint(result) # doctest: +IGNORE_OUTPUT + [{'absolute_magnitude': '3.52', + 'aphelion_distance': '2.982', + 'arc_length': 80185, + 'argument_of_perihelion': '73.73165', + 'ascending_node': '80.2869866', 'critical_list_numbered_object': False, 'delta_v': 10.5, 'designation': None, - 'earth_moid': 1.59353, - 'eccentricity': '0.0755347', - 'epoch': '2018-03-23.0', - 'epoch_jd': '2458200.5', - 'first_observation_date_used': '1801-01-31.0', + 'earth_moid': 1.58537, + 'eccentricity': '0.0775571', + 'epoch': '2020-05-31.0', + 'epoch_jd': '2459000.5', + 'first_observation_date_used': '1801-01-01.0', 'first_opposition_used': '1801', - 'inclination': '10.59351', - 'jupiter_moid': 2.09509, + 'inclination': '10.58862', + 'jupiter_moid': 2.09127, 'km_neo': False, - 'last_observation_date_used': '2018-04-30.0', - 'last_opposition_used': '2018', - 'mars_moid': 0.939285, - 'mean_anomaly': '352.23053', - 'mean_daily_motion': '0.2141308', - 'mercury_moid': 2.18454, + 'last_observation_date_used': '2020-07-17.0', + 'last_opposition_used': '2020', + 'mars_moid': 0.93178, + 'mean_anomaly': '162.68618', + 'mean_daily_motion': '0.21406', + 'mercury_moid': 2.1761, 'name': 'Ceres', 'neo': False, 'number': 1, - 'observations': 6714, - 'oppositions': 114, + 'observations': 7663, + 'oppositions': 119, 'orbit_type': 0, 'orbit_uncertainty': '0', - 'p_vector_x': '-0.87827464', - 'p_vector_y': '0.33795667', - 'p_vector_z': '0.33825869', - 'perihelion_date': '2018-04-28.28377', - 'perihelion_date_jd': '2458236.78377', - 'perihelion_distance': '2.5580384', + 'p_vector_x': '-0.88282454', + 'p_vector_y': '0.3292319', + 'p_vector_z': '0.33500327', + 'perihelion_date': '2018-05-01.99722', + 'perihelion_date_jd': '2458240.49722', + 'perihelion_distance': '2.5530055', 'period': '4.6', 'pha': False, - 'phase_slope': '0.12', - 'q_vector_x': '-0.44248619', - 'q_vector_y': '-0.84255513', - 'q_vector_z': '-0.30709418', - 'residual_rms': '0.6', - 'saturn_moid': 6.38856, - 'semimajor_axis': '2.7670463', + 'phase_slope': '0.15', + 'q_vector_x': '-0.43337703', + 'q_vector_y': '-0.84597284', + 'q_vector_z': '-0.3106675', + 'residual_rms': '0.51', + 'saturn_moid': 6.37764, + 'semimajor_axis': '2.7676568', 'tisserand_jupiter': 3.3, - 'updated_at': '2018-05-31T01:07:39Z', - 'uranus_moid': 15.6642, - 'venus_moid': 1.84632}] - + 'updated_at': '2021-01-16T13:32:57Z', + 'uranus_moid': 15.8216, + 'venus_moid': 1.8382}] >>> eph = MPC.get_ephemeris('ceres') - >>> print(eph) - - Date RA Dec Delta r Elongation Phase V Proper motion Direction Uncertainty 3sig Unc. P.A. - deg deg AU AU deg deg mag arcsec / h deg arcsec deg - ----------------------- ------------------ ----------------- ----- ----- ---------- ----- --- ------------- --------- ---------------- --------- - 2018-08-23 15:56:35.000 177.25874999999996 9.57 3.466 2.581 24.6 9.4 8.7 66.18 115.9 -- -- - 2018-08-24 15:56:35.000 177.66125 9.377222222222223 3.471 2.581 24.1 9.2 8.7 66.24 115.9 -- -- - 2018-08-25 15:56:35.000 178.06416666666667 9.184166666666666 3.476 2.582 23.6 9.0 8.7 66.3 115.9 -- -- - 2018-08-26 15:56:35.000 178.4670833333333 8.99111111111111 3.481 2.582 23.1 8.8 8.7 66.36 115.9 -- -- - ... ... ... ... ... ... ... ... ... ... ... ... - 2018-09-09 15:56:35.000 184.13 6.287222222222222 3.539 2.588 16.3 6.3 8.7 67.08 115.5 -- -- - 2018-09-10 15:56:35.000 184.53625 6.094444444444444 3.542 2.588 15.9 6.1 8.6 67.12 115.5 -- -- - 2018-09-11 15:56:35.000 184.94249999999997 5.901944444444445 3.545 2.589 15.4 5.9 8.6 67.15 115.5 -- -- - 2018-09-12 15:56:35.000 185.34874999999997 5.709444444444444 3.548 2.589 14.9 5.8 8.6 67.18 115.4 -- -- - Length = 21 rows + >>> print(eph) # doctest: +IGNORE_OUTPUT + Date RA ... Uncertainty 3sig Unc. P.A. + deg ... arcsec deg + ----------------------- ------------------- ... ---------------- --------- + 2021-01-29 03:16:34.000 355.2416666666667 ... -- -- + 2021-01-30 03:16:34.000 355.56125 ... -- -- + 2021-01-31 03:16:34.000 355.8812499999999 ... -- -- + 2021-02-01 03:16:34.000 356.2029166666666 ... -- -- + 2021-02-02 03:16:34.000 356.5254166666666 ... -- -- + 2021-02-03 03:16:34.000 356.8491666666667 ... -- -- + 2021-02-04 03:16:34.000 357.17375 ... -- -- + 2021-02-05 03:16:34.000 357.4995833333333 ... -- -- + 2021-02-06 03:16:34.000 357.82625 ... -- -- + 2021-02-07 03:16:34.000 358.15374999999995 ... -- -- + 2021-02-08 03:16:34.000 358.48249999999996 ... -- -- + 2021-02-09 03:16:34.000 358.8120833333333 ... -- -- + 2021-02-10 03:16:34.000 359.1429166666666 ... -- -- + 2021-02-11 03:16:34.000 359.47416666666663 ... -- -- + 2021-02-12 03:16:34.000 359.8066666666666 ... -- -- + 2021-02-13 03:16:34.000 0.13999999999999999 ... -- -- + 2021-02-14 03:16:34.000 0.4741666666666666 ... -- -- + 2021-02-15 03:16:34.000 0.80875 ... -- -- + 2021-02-16 03:16:34.000 1.1445833333333333 ... -- -- + 2021-02-17 03:16:34.000 1.4812499999999997 ... -- -- + 2021-02-18 03:16:34.000 1.8183333333333331 ... -- -- @@ -120,19 +126,28 @@ queried in three manners: An example of an exact match: -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.mpc import MPC >>> result = MPC.query_object('asteroid', name='ceres') - >>> print(result) - - [{'absolute_magnitude': '3.34', 'aphelion_distance': '2.976', 'arc_length': 79247, 'argument_of_perihelion': '73.11528', 'ascending_node': '80.3099167', 'critical_list_numbered_object': False, 'delta_v': 10.5, 'designation': None, 'earth_moid': 1.59353, 'eccentricity': '0.0755347', 'epoch': '2018-03-23.0', 'epoch_jd': '2458200.5', 'first_observation_date_used': '1801-01-31.0', 'first_opposition_used': '1801', 'inclination': '10.59351', 'jupiter_moid': 2.09509, 'km_neo': False, 'last_observation_date_used': '2018-01-20.0', 'last_opposition_used': '2018', 'mars_moid': 0.939285, 'mean_anomaly': '352.23052', 'mean_daily_motion': '0.2141308', 'mercury_moid': 2.18454, 'name': 'Ceres', 'neo': False, 'number': 1, 'observations': 6689, 'oppositions': 114, 'orbit_type': 0, 'orbit_uncertainty': '0', 'p_vector_x': '-0.87827466', 'p_vector_y': '0.33795664', 'p_vector_z': '0.33825868', 'perihelion_date': '2018-04-28.28378', 'perihelion_date_jd': '2458236.78378', 'perihelion_distance': '2.5580384', 'period': '4.6', 'pha': False, 'phase_slope': '0.12', 'q_vector_x': '-0.44248615', 'q_vector_y': '-0.84255514', 'q_vector_z': '-0.30709419', 'residual_rms': '0.6', 'saturn_moid': 6.38856, 'semimajor_axis': '2.7670463', 'tisserand_jupiter': 3.3, 'updated_at': '2018-02-26T17:29:46Z', 'uranus_moid': 15.6642, 'venus_moid': 1.84632}] + >>> print(result) # doctest: +IGNORE_OUTPUT + [{'absolute_magnitude': '3.52', 'aphelion_distance': '2.982', 'arc_length': 80185, + 'argument_of_perihelion': '73.73165', 'ascending_node': '80.2869866', 'critical_list_numbered_object': False, 'delta_v': 10.5, + 'designation': None, 'earth_moid': 1.58537, 'eccentricity': '0.0775571', 'epoch': '2020-05-31.0', 'epoch_jd': '2459000.5', + 'first_observation_date_used': '1801-01-01.0', 'first_opposition_used': '1801', 'inclination': '10.58862', 'jupiter_moid': 2.09127, + 'km_neo': False, 'last_observation_date_used': '2020-07-17.0', 'last_opposition_used': '2020', 'mars_moid': 0.93178, + 'mean_anomaly': '162.68618', 'mean_daily_motion': '0.21406', 'mercury_moid': 2.1761, 'name': 'Ceres', 'neo': False, + 'number': 1, 'observations': 7663, 'oppositions': 119, 'orbit_type': 0, 'orbit_uncertainty': '0', 'p_vector_x': '-0.88282454', + 'p_vector_y': '0.3292319', 'p_vector_z': '0.33500327', 'perihelion_date': '2018-05-01.99722', 'perihelion_date_jd': '2458240.49722', + 'perihelion_distance': '2.5530055', 'period': '4.6', 'pha': False, 'phase_slope': '0.15', 'q_vector_x': '-0.43337703', + 'q_vector_y': '-0.84597284', 'q_vector_z': '-0.3106675', 'residual_rms': '0.51', 'saturn_moid': 6.37764, 'semimajor_axis': '2.7676568', + 'tisserand_jupiter': 3.3, 'updated_at': '2021-01-16T13:32:57Z', 'uranus_moid': 15.8216, 'venus_moid': 1.8382}] A minimum value: .. code-block:: python - >>> result = MPC.query_objects('asteroid', inclination_min=170) + >>> result = MPC.query_objects('asteroid', inclination_min=170) # doctest: +REMOTE_DATA which will get all asteroids with an inclination of greater than or equal to 170. @@ -141,8 +156,8 @@ A maximum value: .. code-block:: python - >>> result = MPC.query_objects('asteroid', inclination_max=1.0) - + >>> result = MPC.query_objects('asteroid', inclination_max=1.0) # doctest: +REMOTE_DATA + which will get all asteroids with an inclination of less than or equal to 1. There is another parameter that can be used, ```is_not_null```. This @@ -150,7 +165,7 @@ can be used in the following fashion: .. code-block:: python - >>> result = MPC.query_objects('asteroid', name="is_not_null") + >>> result = MPC.query_objects('asteroid', name="is_not_null") # doctest: +REMOTE_DATA This will, predictably, find all named objects in the MPC database--but that would take a while! @@ -169,7 +184,7 @@ options. .. code-block:: python - >>> result = MPC.query_objects('asteroid', order_by_desc="semimajor_axis", limit=10) + >>> result = MPC.query_objects('asteroid', order_by_desc="semimajor_axis", limit=10) # doctest: +REMOTE_DATA This will return the 10 furthest asteroids. @@ -179,7 +194,7 @@ Customizing return fields If a consumer isn't interested in some return fields, they can use the MPC to limit the fields they're interested in. -.. code-block:: python +.. doctest-remote-data:: >>> result = MPC.query_object('asteroid', name="ceres", return_fields="name,number") >>> print(result) @@ -206,13 +221,14 @@ Dates and intervals For the ephemeris of asteroid (24) Themis, starting today with the default time step (1 day) and location (geocenter): -.. code-block:: python + +.. doctest-remote-data:: >>> from astroquery.mpc import MPC >>> eph = MPC.get_ephemeris('24') - >>> print(eph) + >>> print(eph) # doctest: +IGNORE_OUTPUT Date RA Dec Delta r Elongation Phase V Proper motion Direction Uncertainty 3sig Unc. P.A. - deg deg AU AU deg deg mag arcsec / h deg arcsec deg + deg deg AU AU deg deg mag arcsec / h deg arcsec deg ----------------------- ------------------ ------------------ ----- ----- ---------- ----- ---- ------------- --------- ---------------- --------- 2018-08-16 14:34:53.000 96.46708333333333 23.749722222222225 3.502 2.916 47.5 14.8 12.9 53.08 92.1 -- -- 2018-08-17 14:34:53.000 96.85291666666666 23.73638888888889 3.491 2.915 48.1 15.0 12.9 52.91 92.3 -- -- @@ -227,12 +243,13 @@ default time step (1 day) and location (geocenter): Step sizes are parsed with Astropy's `~astropy.units.Quantity`. For a time step of 1 hour: -.. code-block:: python + +.. doctest-remote-data:: >>> eph = MPC.get_ephemeris('24', step='1h') - >>> print(eph) + >>> print(eph) # doctest: +IGNORE_OUTPUT Date RA Dec Delta r Elongation Phase V Proper motion Direction Uncertainty 3sig Unc. P.A. - deg deg AU AU deg deg mag arcsec / h deg arcsec deg + deg deg AU AU deg deg mag arcsec / h deg arcsec deg ----------------------- ----------------- ------------------ ----- ----- ---------- ----- ---- ------------- --------- ---------------- --------- 2018-08-16 14:00:00.000 96.45791666666666 23.75 3.503 2.916 47.5 14.8 12.9 53.09 92.1 -- -- 2018-08-16 15:00:00.000 96.47374999999998 23.749444444444446 3.502 2.916 47.5 14.8 12.9 53.08 92.1 -- -- @@ -248,12 +265,13 @@ Step sizes are parsed with Astropy's `~astropy.units.Quantity`. For a time step Start dates are parsed with Astropy's `~astropy.time.Time`. For a weekly ephemeris in 2020: -.. code-block:: python + +.. doctest-remote-data:: >>> eph = MPC.get_ephemeris('24', start='2020-01-01', step='7d', number=52) - >>> print(eph) + >>> print(eph) # doctest: +IGNORE_OUTPUT Date RA Dec Delta r Elongation Phase V Proper motion Direction Uncertainty 3sig Unc. P.A. - deg deg AU AU deg deg mag arcsec / h deg arcsec deg + deg deg AU AU deg deg mag arcsec / h deg arcsec deg ----------------------- ------------------ ------------------- ----- ----- ---------- ----- ---- ------------- --------- ---------------- --------- 2020-01-01 00:00:00.000 209.16749999999996 -11.63361111111111 3.066 2.856 68.5 18.7 12.7 45.15 110.6 -- -- 2020-01-08 00:00:00.000 211.11999999999995 -12.342500000000001 2.98 2.863 73.6 19.2 12.7 42.09 110.2 -- -- @@ -274,12 +292,13 @@ Ephemerides may be calculated for Earth-based observers. To calculate Makemake's ephemeris for the Discovery Channel Telescope (IAU observatory code G37): -.. code-block:: python + +.. doctest-remote-data:: >>> eph = MPC.get_ephemeris('Makemake', location='G37') - >>> print(eph) + >>> print(eph) # doctest: +IGNORE_OUTPUT Date RA Dec Delta r Elongation Phase V Proper motion Direction Azimuth Altitude Sun altitude Moon phase Moon distance Moon altitude Uncertainty 3sig Unc. P.A. - deg deg AU AU deg deg mag arcsec / h deg deg deg deg deg deg arcsec deg + deg deg AU AU deg deg mag arcsec / h deg deg deg deg deg deg arcsec deg ----------------------- ------------------ ------------------ ------ ------ ---------- ----- ---- ------------- --------- ------- -------- ------------ ---------- ------------- ------------- ---------------- --------- 2018-08-16 14:42:27.000 194.66791666666663 24.109722222222224 53.211 52.528 47.2 0.8 17.2 2.62 134.2 53 -9 23 0.33 36 -43 -- -- 2018-08-17 14:42:27.000 194.68166666666664 24.09722222222222 53.22 52.528 46.5 0.8 17.2 2.65 133.7 53 -8 23 0.43 46 -54 -- -- @@ -300,7 +319,8 @@ an array of longitude (east), latitude, and altitude (parsed with `~astropy.coordinates.EarthLocation`. For example, to compute Encke's parallax between Mauna Kea and Botswana: -.. code-block:: python + +.. doctest-remote-data:: >>> from astropy.table import Table >>> from astropy.coordinates import SkyCoord @@ -310,9 +330,9 @@ Encke's parallax between Mauna Kea and Botswana: >>> bw = SkyCoord.guess_from_table(eph) >>> mu = mko.separation(bw) >>> tab = Table(data=(eph['Date'], mu), names=('Date', 'Parallax')) - >>> print(tab) - Date Parallax - deg + >>> print(tab) # doctest: +IGNORE_OUTPUT + Date Parallax + deg ----------------------- --------------------- 2003-11-01 00:00:00.000 0.005050002777840046 2003-11-02 00:00:00.000 0.005439170027971742 @@ -325,7 +345,7 @@ Encke's parallax between Mauna Kea and Botswana: 2003-11-21 00:00:00.000 0.0075389478267517745 Length = 21 rows - + Working with ephemeris tables ----------------------------- @@ -334,7 +354,8 @@ Convert the columns to Astropy quantities using the ``.quantity`` attribute. To find comet Hyakutake's peak proper motion in the sky in degrees per hour: -.. code-block:: python + +.. doctest-remote-data:: >>> eph = MPC.get_ephemeris('C/1996 B2', start='1996-03-01', step='1h', number=30 * 24) >>> print(eph['Proper motion'].quantity.to('deg/h').max()) @@ -346,12 +367,13 @@ strings using the ``ra_format`` and ``dec_format`` keyword arguments (see ``Angle``'s `~astropy.coordinates.Angle.to_string` for formatting options): -.. code-block:: python + +.. doctest-remote-data:: >>> eph = MPC.get_ephemeris('2P', ra_format={'sep': ':', 'unit': 'hourangle', 'precision': 1}, dec_format={'sep': ':', 'precision': 0}) - >>> print(eph) + >>> print(eph) # doctest: +IGNORE_OUTPUT Date RA Dec Delta r Elongation Phase V Proper motion Direction - hourangle deg AU AU deg deg mag arcsec / h deg + hourangle deg AU AU deg deg mag arcsec / h deg ----------------------- ---------- -------- ----- ----- ---------- ----- ---- ------------- --------- 2018-08-16 14:12:18.000 22:52:30.5 -6:18:57 3.076 4.048 161.4 4.6 22.4 36.34 250.9 2018-08-17 14:12:18.000 22:51:35.0 -6:23:43 3.072 4.049 162.6 4.3 22.4 36.67 250.9 @@ -371,11 +393,12 @@ IAU Observatory Codes and Locations Two methods are available for working with the MPC's observatory list. To retrieve a list of all observatories: -.. code-block:: python + +.. doctest-remote-data:: >>> obs = MPC.get_observatory_codes() - >>> print(obs) - Code Longitude cos sin Name + >>> print(obs) # doctest: +IGNORE_OUTPUT + Code Longitude cos sin Name ---- --------- -------- -------- ---------------------------------------- 000 0.0 0.62411 0.77873 Greenwich 001 0.1542 0.62992 0.77411 Crowborough @@ -397,14 +420,14 @@ The results are cached by default. To update the cache, use the .. code-block:: python - >>> obs = MPC.get_observatory_codes(cache=False) + >>> obs = MPC.get_observatory_codes(cache=False) # doctest: +REMOTE_DATA To get the location (longitude, parallax constants, and name) of a single observatory: .. code-block:: python - >>> print(MPC.get_observatory_location('371')) + >>> print(MPC.get_observatory_location('371')) # doctest: +REMOTE_DATA (, 0.82433, 0.56431, 'Tokyo-Okayama') The parallax constants are ``rho * cos(phi)`` and ``rho * sin(phi)`` where @@ -418,12 +441,12 @@ Observations The following code snippet queries all reported observations for asteroid 12893: -.. code-block:: python +.. doctest-remote-data:: >>> obs = MPC.get_observations(12893) - >>> print(obs) + >>> print(obs) # doctest: +IGNORE_OUTPUT number desig discovery note1 ... DEC mag band observatory - ... deg mag + ... deg mag ------ --------- --------- ----- ... ------------------- ----- ---- ----------- 12893 1998 QS55 -- -- ... -15.78888888888889 0.0 -- 413 12893 1998 QS55 -- -- ... -15.788944444444445 0.0 -- 413 diff --git a/docs/nist/nist.rst b/docs/nist/nist.rst index fd0310c7d9..c9c0a96b22 100644 --- a/docs/nist/nist.rst +++ b/docs/nist/nist.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.nist: ******************************** @@ -23,46 +21,37 @@ parameters you can also specify. For instance use the ``linename`` parameter to specify the spectrum you wish to fetch. By default this is set to "H I", but you can set it to several other values like "Na;Mg", etc. Lets now see a simple example. -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.nist import Nist >>> import astropy.units as u >>> table = Nist.query(4000 * u.AA, 7000 * u.AA, linename="H I") >>> print(table) - - Observed Ritz Rel. Aki Acc. ... Lower level Upper level Type TP Line - ------------- ------------- ------ ---------- ---- ... ------------------- ------------------- ---- ----- -------- - -- 4102.85985516 -- 4287700.0 AAA ... -- -- -- -- -- - -- 4102.86191086 -- 245010.0 AAA ... 2p | 2P* | 1/2 6s | 2S | 1/2 -- T8637 -- - -- 4102.8632 -- -- -- ... | | | | -- -- c57 - 4102.86503481 4102.86503481 -- -- -- ... 2s | 2S | 1/2 6d | 2D | 5/2 E2 -- L11759 - -- 4102.86579134 -- 2858300.0 AAA ... -- -- -- -- -- - 4102.86785074 4102.86785074 -- -- -- ... 2s | 2S | 1/2 6s | 2S | 1/2 M1 -- L11759 - -- 4102.86807252 -- 2858400.0 AAA ... -- -- -- -- -- - 4102.892 4102.8991 70000 973200.0 AAA ... 2 | | 6 | | -- T8637 L7436c29 - -- 4102.8922 -- -- -- ... | | | | -- -- c58 - -- 4102.92068748 -- 5145000.0 AAA ... 2p | 2P* | 3/2 6d | 2D | 5/2 -- T8637 -- - -- 4102.9208 -- -- -- ... | | | | -- -- c59 - -- 4102.92144772 -- 857480.0 AAA ... -- -- -- -- -- - -- 4102.92350348 -- 490060.0 AAA ... 2p | 2P* | 3/2 6s | 2S | 1/2 -- T8637 -- - -- 4341.647191 -- 7854800.0 AAA ... 2p | 2P* | 1/2 5d | 2D | 3/2 -- T8637 -- - -- 4341.6512 -- -- -- ... | | | | -- -- c60 - ... ... ... ... ... ... ... ... ... ... ... - 6564.522552 6564.522555 -- 53877000.0 AAA ... 2p | 2P* | 1/2 3d | 2D | 3/2 -- T8637 L2752 - -- 6564.527 -- -- -- ... | | | | -- -- c67 - -- 6564.535 -- -- -- ... | | | | -- -- c68 - 6564.537684 6564.537684 -- 22448000.0 AAA ... 2s | 2S | 1/2 3p | 2P* | 3/2 -- T8637 L6891c38 - -- 6564.564672 -- 2104600.0 AAA ... 2p | 2P* | 1/2 3s | 2S | 1/2 -- T8637 -- - -- 6564.579878 -- -- -- ... 2s | 2S | 1/2 3s | 2S | 1/2 M1 -- -- - -- 6564.583 -- -- -- ... | | | | -- -- c66 - 6564.584404 6564.584403 -- 22449000.0 AAA ... 2s | 2S | 1/2 3p | 2P* | 1/2 -- T8637 L6891c38 - 6564.6 6564.632 500000 44101000.0 AAA ... 2 | | 3 | | -- T8637 L7400c29 - -- 6564.608 -- -- -- ... | | | | -- -- c69 - 6564.66464 6564.66466 -- 64651000.0 AAA ... 2p | 2P* | 3/2 3d | 2D | 5/2 -- T8637 L2752 - -- 6564.6662 -- -- -- ... | | | | -- -- c71 - -- 6564.667 -- -- -- ... | | | | -- -- c70 - -- 6564.680232 -- 10775000.0 AAA ... 2p | 2P* | 3/2 3d | 2D | 3/2 -- T8637 -- - -- 6564.722349 -- 4209700.0 AAA ... 2p | 2P* | 3/2 3s | 2S | 1/2 -- T8637 -- + Observed Ritz Transition Rel. ... Type TP Line + ------------- ------------- ------------- ------ ... ---- ----- -------- + -- 4102.85985517 24373.2429403 -- ... -- T8637 -- + -- 4102.86191087 24373.2307283 -- ... -- T8637 -- + -- 4102.8632 24373.223 -- ... -- -- c57 + 4102.86503481 4102.86503481 24373.2121704 -- ... E2 -- L11759 + -- 4102.86579132 24373.2076763 -- ... -- T8637 -- + 4102.86785074 4102.86785074 24373.1954423 -- ... M1 -- L11759 + -- 4102.8680725 24373.1941249 -- ... -- T8637 -- + 4102.892 4102.8991 24373.05 70000 ... -- T8637 L7436c29 + -- 4102.8922 24373.051 -- ... -- -- c58 + -- 4102.92068748 24372.8815683 -- ... -- T8637 -- + ... ... ... ... ... ... ... ... + -- 6564.564672 15233.302588 -- ... -- T8637 -- + -- 6564.579878 15233.267302 -- ... M1 -- -- + -- 6564.583 15233.26 -- ... -- -- c66 + 6564.584404 6564.584403 15233.256799 -- ... -- T8637 L6891c38 + 6564.6 6564.632 15233.21 500000 ... -- T8637 L7400c29 + -- 6564.608 15233.202 -- ... -- -- c69 + 6564.66464 6564.66466 15233.07061 -- ... -- T8637 L2752 + -- 6564.6662 15233.067 -- ... -- -- c71 + -- 6564.667 15233.065 -- ... -- -- c70 + -- 6564.680232 15233.034432 -- ... -- T8637 -- + -- 6564.722349 15232.9367 -- ... -- T8637 -- + Length = 53 rows Note that using a different unit will result in different output units in the @@ -77,35 +66,38 @@ Similarly you can set the ``output_order`` to any one of 'wavelength' or 'multiplet'. A final parameter you may also set is the ``wavelength_type`` to one of 'vacuum' or 'vac+air'. Here is an example with all these parameters. -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.nist import Nist >>> table = Nist.query(4000 * u.nm, 7000 * u.nm, 'H I', ... energy_level_unit='eV', output_order='wavelength', ... wavelength_type='vacuum') >>> print(table) - - Observed Ritz Rel. Aki ... Upper level Type TP Line - -------- ----------- ----- --------- ... ------------------- ---- ----- ---- - -- 4020.871 (200) 5526.5 ... -- -- -- -- - -- 4052.18664 -- 1238100.0 ... 5d | 2D | 3/2 -- T8637 -- - -- 4052.19376 -- 737160.0 ... 5p | 2P* | 3/2 -- T8637 -- - -- 4052.22121 -- 215030.0 ... -- -- -- -- - -- 4052.23222 -- 737210.0 ... 5p | 2P* | 1/2 -- T8637 -- - -- 4052.248747 -- 2412100.0 ... -- -- -- -- - -- 4052.24892 -- 1485800.0 ... -- -- -- -- - -- 4052.26147 -- 18846.0 ... 5p | 2P* | 3/2 -- T8637 -- - ... ... ... ... ... ... ... ... ... - -- 5525.19 (150) 2470.9 ... -- -- -- -- - -- 5711.464 (180) 3515.8 ... -- -- -- -- - -- 5908.22 (540) 70652.0 ... 9 | | -- T8637 -- - -- 5956.845 (210) 5156.2 ... -- -- -- -- - -- 6291.918 (250) 7845.7 ... -- -- -- -- - -- 6771.993 (300) 12503.0 ... 12 | | -- T8637 -- - -- 6946.756 -- 688.58 ... -- -- -- -- - - - + Observed Ritz Transition Rel. ... Upper level Type TP Line + -------- ----------- ----------- ----- ... ------------------- ---- ----- ----- + -- 4020.871 2487.024 (200) ... 14 | | -- T8637 -- + -- 4052.18664 2467.803411 -- ... 5d | 2D | 3/2 -- T8637 -- + -- 4052.19376 2467.79907 -- ... 5p | 2P* | 3/2 -- T8637 -- + -- 4052.22121 2467.78236 -- ... 5s | 2S | 1/2 -- T8637 -- + -- 4052.23222 2467.77565 -- ... 5p | 2P* | 1/2 -- T8637 -- + -- 4052.248747 2467.765585 -- ... 5f | 2F* | 5/2 -- T8637 -- + -- 4052.24892 2467.765479 -- ... 5d | 2D | 5/2 -- T8637 -- + -- 4052.26147 2467.75784 -- ... 5p | 2P* | 3/2 -- T8637 -- + -- 4052.26174 2467.757676 -- ... 5d | 2D | 3/2 -- T8637 -- + -- 4052.26738 2467.75424 -- ... 5g | 2G | 7/2 -- T8637 -- + ... ... ... ... ... ... ... ... ... + 5128.65 5128.662 1949.83 (450) ... 10 | | -- T8637 L7452 + -- 5169.282 1934.5047 -- ... 19 | | -- T8637 -- + -- 5263.685 1899.8096 -- ... 18 | | -- T8637 -- + -- 5379.776 1858.8134 -- ... 17 | | -- T8637 -- + -- 5525.19 1809.8925 (150) ... 16 | | -- T8637 -- + -- 5711.464 1750.8646 (180) ... 15 | | -- T8637 -- + -- 5908.22 1692.5572 (540) ... 9 | | -- T8637 -- + -- 5956.845 1678.7409 (210) ... 14 | | -- T8637 -- + -- 6291.918 1589.3405 (250) ... 13 | | -- T8637 -- + -- 6771.993 1476.6701 (300) ... 12 | | -- T8637 -- + -- 6946.756 1439.5208 -- ... 20 | | -- T8637 -- + Length = 37 rows Reference/API ============= diff --git a/docs/sdss/sdss.rst b/docs/sdss/sdss.rst index ec61bb9136..bcfcc760e9 100644 --- a/docs/sdss/sdss.rst +++ b/docs/sdss/sdss.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.sdss: ******************************** @@ -15,16 +13,16 @@ arcsecond radius for optical counterparts in SDSS. Note use of the keyword argument spectro, which requires matches to have spectroscopy, not just photometry: -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.sdss import SDSS >>> from astropy import coordinates as coords >>> pos = coords.SkyCoord('0h8m05.63s +14d50m23.3s', frame='icrs') >>> xid = SDSS.query_region(pos, spectro=True) >>> print(xid) - ra dec objid run rerun camcol field z plate mjd fiberID specobjid specClass - ---------- ----------- ------------------ ---- ----- ------ ----- ------- ----- ----- ------- ------------------ --------- - 2.02344483 14.83982059 587727221951234166 1739 40 3 315 0.04541 751 52251 160 211612124516974592 3 + ra dec ... specobjid run2d + ---------------- ---------------- ... ------------------ ----- + 2.02344596573482 14.8398237551311 ... 845594848269461504 26 The result is an astropy.Table. @@ -33,7 +31,7 @@ Downloading data If we'd like to download spectra and/or images for our match, we have all the information we need in the elements of "xid" from the above example. -.. code-block:: python +.. doctest-remote-data:: >>> sp = SDSS.get_spectra(matches=xid) >>> im = SDSS.get_images(matches=xid, band='g') @@ -58,16 +56,21 @@ Spectral templates It is also possible to download spectral templates from SDSS. To see what is available, do -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.sdss import SDSS - >>> print(SDSS.AVAILABLE_TEMPLATES) + >>> print(SDSS.AVAILABLE_TEMPLATES) # doctest: +IGNORE_OUTPUT + {'star_O': 0, 'star_OB': 1, 'star_B': 2, 'star_A': [3, 4], 'star_FA': 5, + 'star_F': [6, 7], 'star_G': [8, 9], 'star_K': 10, 'star_M1': 11, 'star_M3': 12, + 'star_M5': 13, 'star_M8': 14, 'star_L1': 15, 'star_wd': [16, 20, 21], 'star_carbon': [17, 18, 19], + 'star_Ksubdwarf': 22, 'galaxy_early': 23, 'galaxy': [24, 25, 26], 'galaxy_late': 27, 'galaxy_lrg': 28, + 'qso': 29, 'qso_bal': [30, 31], 'qso_bright': 32} Then, to download your favorite template, do something like .. code-block:: python - >>> template = SDSS.get_spectral_template('qso') + >>> template = SDSS.get_spectral_template('qso') # doctest: +REMOTE_DATA The variable "template" is a list of `~astropy.io.fits.HDUList` objects (same object as "sp" in the above example). In this case there is only one diff --git a/docs/solarsystem/solarsystem.rst b/docs/solarsystem/solarsystem.rst index a2e940e864..013f8e6139 100644 --- a/docs/solarsystem/solarsystem.rst +++ b/docs/solarsystem/solarsystem.rst @@ -16,8 +16,8 @@ The currently available service providers and services are: .. toctree:: :maxdepth: 1 - jpl/jpl.rst imcce/imcce.rst + jpl/jpl.rst mpc/mpc.rst Reference/API diff --git a/docs/testing.rst b/docs/testing.rst index 58409ec5a7..45c22273d8 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -85,7 +85,7 @@ the ``test_module.py`` file. ------------------------- The remote tests are much easier. Just decorate the test class or test -functions with ``astropy.tests.helper.remote_data``. +functions with ``@pytest.mark.remote_data``. ``setup_package.py`` -------------------- diff --git a/docs/utils/tap.rst b/docs/utils/tap.rst index 2ef92f34d6..50ee41c8de 100644 --- a/docs/utils/tap.rst +++ b/docs/utils/tap.rst @@ -757,7 +757,25 @@ This service provides links to data: >>> ids="1000103304040175360,1000117258390464896" >>> results = gaia.get_datalink(ids=ids) +2.8. Rename Table +~~~~~~~~~~~~~~~~~~ +This query allows the user to rename a table and/or its columns names. For this method it is only mandatory +to provide the old name of the table and at least one of the following parameters: new table name and/or +new columns names. Here is an example: + +.. code-block:: python + + >>> from astroquery.gaia import Gaia, GaiaClass + >>> from astroquery.utils.tap.model.tapcolumn import TapColumn + >>> from astroquery.utils.tap.core import TapPlus, TAP_CLIENT_ID + >>> from astroquery.utils.tap import taputils + >>> gaia = GaiaClass(gaia_tap_server='https://gea.esac.esa.int/', gaia_data_server='https://gea.esac.esa.int/') + >>> gaia.login() + >>> tableName = 'user_.my_old_table_name' + >>> newTableName = 'user_.my_new_table_name' + >>> newColumnNames = {'old_col_name1': 'new_col_name1', 'old_col_name2': 'new_col_name2'} + >>> tap.rename_table(table_name=tableName, new_table_name=newTableName, new_column_names_dict=newColumnNames,verbose=True) 3. Using TAP+ to connect other TAP services ------------------------------------------- diff --git a/setup.cfg b/setup.cfg index 71f8016780..e766b20dea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = astroquery -version = 0.4.5.dev +version = 0.4.7.dev description = Functions and classes to access online astronomical data resources # FIXME long_description = author = The Astroquery Developers @@ -37,6 +37,7 @@ filterwarnings = error ignore: Experimental:UserWarning: # This is a temporary measure, all of these should be fixed: + ignore:distutils Version classes are deprecated:DeprecationWarning ignore::pytest.PytestUnraisableExceptionWarning ignore::numpy.VisibleDeprecationWarning ignore: unclosed file:ResourceWarning @@ -57,6 +58,8 @@ filterwarnings = # Upstream, remove when fixed, PRs have been opened ignore::DeprecationWarning:pyvo ignore::DeprecationWarning:regions +# Should ignore these for astropy<5.0 + ignore:getName|currentThread:DeprecationWarning:astropy # This should be cleared once we requre astropy>=4.1 ignore:tostring\(\) is deprecated. Use tobytes:DeprecationWarning:astropy markers = @@ -172,12 +175,5 @@ all= astropy-healpix boto3 regions -# aplpy is not py39 compatible (it requires shapely that doesn't compile -# pyregion is not py39 compatible -all_lt_39= - mocpy>=0.5.2 - regions pyregion - astropy-healpix aplpy - boto3 diff --git a/tox.ini b/tox.ini index 461dc4aae8..5ca194cb7b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{37,38,39}-test{,-alldeps,-oldestdeps}{,-devastropy}{,-cov} + py{37,38,39,310}-test{,-alldeps,-oldestdeps}{,-devastropy}{,-cov} codestyle build_docs requires = @@ -41,8 +41,8 @@ deps = extras = test - !py39-alldeps: all_lt_39 - py39-alldeps: all + alldeps: all + commands = pip freeze