diff --git a/docs/fbs-output-schema.rst b/docs/fbs-output-schema.rst index b7f57e8f..e3e50918 100644 --- a/docs/fbs-output-schema.rst +++ b/docs/fbs-output-schema.rst @@ -101,18 +101,12 @@ All values are for the center of the field of view (e.g., airmass, altitude, etc * - sunAlt - degrees - Altitude of the Sun. - * - note - - string - - DEPRECATED. Do not use. * - scheduler_note - string - Descriptive comment about how the observations were scheduled, for use by the FBS. * - target_name - string - Descriptive name for the target. Only used for DDFs, ToOs, or other special targets. Should translate to target_name in the headers/ConsDB. - * - block_id - - integer - - Identification ID of the block (used by some survey objects). * - observationStartLST - degrees - Local Sidereal Time at the start of the observation. @@ -152,6 +146,9 @@ All values are for the center of the field of view (e.g., airmass, altitude, etc * - cummTelAz - degrees - Cumulative azimuth of the telescope mount, tracks cable wrap. + * - target_id + - integer + - Integer added by the `CoreScheduler`. * - observation_reason - string - The reason for the observation. Identifier for DM. Translates to observation_reason in the headers/consdb. diff --git a/rubin_scheduler/scheduler/schedulers/core_scheduler.py b/rubin_scheduler/scheduler/schedulers/core_scheduler.py index 2ff995b8..d41436d3 100644 --- a/rubin_scheduler/scheduler/schedulers/core_scheduler.py +++ b/rubin_scheduler/scheduler/schedulers/core_scheduler.py @@ -41,6 +41,10 @@ class CoreScheduler: telescope : `str` Which telescope model to use for rotTelPos/rotSkyPos conversions. Default "rubin". + target_id_counter : int + Starting value for the target_id. If restarting observations, could + be useful to set to whatever value the scheduler was at previously. + Default 0. """ def __init__( @@ -51,6 +55,7 @@ def __init__( log=None, keep_rewards=False, telescope="rubin", + target_id_counter=0, ): self.keep_rewards = keep_rewards # Use integer ns just to be sure there are no rounding issues. @@ -86,9 +91,12 @@ def __init__( self.rc = rotation_converter(telescope=telescope) - # keep track of how many observations get flushed from the queue + # Keep track of how many observations get flushed from the queue self.flushed = 0 + # Counter for observations added to the queue + self.target_id_counter = target_id_counter + def flush_queue(self): """Like it sounds, clear any currently queued desired observations.""" self.queue = [] @@ -127,6 +135,9 @@ def add_observations_array(self, obs): for survey in surveys: survey.add_observations_array(obs, obs_array_hpid) + if np.max(obs["target_id"]) >= self.target_id_counter: + self.target_id_counter = np.max(obs["target_id"]) + 1 + def add_observation(self, observation): """ Record a completed observation and update features accordingly. @@ -154,6 +165,9 @@ def add_observation(self, observation): for survey in surveys: survey.add_observation(observation, indx=indx) + if np.max(observation["target_id"]) >= self.target_id_counter: + self.target_id_counter = np.max(observation["target_id"]) + 1 + def update_conditions(self, conditions_in): """ Parameters @@ -302,14 +316,16 @@ def _fill_queue(self): # Take a min here, so the surveys will be executed in the order # they are entered if there is a tie. self.survey_index[1] = np.min(np.where(rewards == np.nanmax(rewards))) - # Survey returns ObservationArray. Convert to list. - result = ( - self.survey_lists[self.survey_index[0]][self.survey_index[1]] - .generate_observations(self.conditions) - .tolist() + # Survey returns ObservationArray + result = self.survey_lists[self.survey_index[0]][self.survey_index[1]].generate_observations( + self.conditions ) + # Tag with a unique target_id + result["target_id"] = np.arange(self.target_id_counter, self.target_id_counter + result.size) + self.target_id_counter += result.size - self.queue = result + # Convert to a list for the queue + self.queue = result.tolist() self.queue_filled = self.conditions.mjd if len(self.queue) == 0: diff --git a/rubin_scheduler/scheduler/surveys/surveys.py b/rubin_scheduler/scheduler/surveys/surveys.py index da72df3a..668a29cd 100644 --- a/rubin_scheduler/scheduler/surveys/surveys.py +++ b/rubin_scheduler/scheduler/surveys/surveys.py @@ -510,7 +510,6 @@ def generate_observations_rough(self, conditions): observations["nexp"] = self.nexp_dict[self.filtername1] observations["exptime"] = self.exptime observations["scheduler_note"] = self.scheduler_note - observations["block_id"] = self.counter observations["flush_by_mjd"] = flush_time return observations diff --git a/rubin_scheduler/scheduler/utils/__init__.py b/rubin_scheduler/scheduler/utils/__init__.py index f0672826..7dfdd815 100644 --- a/rubin_scheduler/scheduler/utils/__init__.py +++ b/rubin_scheduler/scheduler/utils/__init__.py @@ -1,5 +1,6 @@ from .comcam_tessellate import * from .footprints import * +from .observation_array import * from .sky_area import * from .tsp import * from .utils import * diff --git a/rubin_scheduler/scheduler/utils/observation_array.py b/rubin_scheduler/scheduler/utils/observation_array.py new file mode 100644 index 00000000..3e1d7fd0 --- /dev/null +++ b/rubin_scheduler/scheduler/utils/observation_array.py @@ -0,0 +1,262 @@ +__all__ = ( + "obsarray_concat", + "ObservationArray", + "ScheduledObservationArray", +) + +import numpy as np + + +class ObservationArray(np.ndarray): + """Class to work as an array of observations + + Parameters + ---------- + n : `int` + Size of array to return. Default 1. + + The numpy fields have the following labels. + + RA : `float` + The Right Acension of the observation (center of the field) + (Radians) + dec : `float` + Declination of the observation (Radians) + mjd : `float` + Modified Julian Date at the start of the observation + (time shutter opens) + exptime : `float` + Total exposure time of the visit (seconds) + filter : `str` + The filter used. Should be one of u, g, r, i, z, y. + rotSkyPos : `float` + The rotation angle of the camera relative to the sky E of N + (Radians). Will be ignored if rotTelPos is finite. + If rotSkyPos is set to NaN, rotSkyPos_desired is used. + rotTelPos : `float` + The rotation angle of the camera relative to the telescope + (radians). Set to np.nan to force rotSkyPos to be used. + rotSkyPos_desired : `float` + If both rotSkyPos and rotTelPos are None/NaN, then + rotSkyPos_desired (radians) is used. If rotSkyPos_desired + results in a valid rotTelPos, rotSkyPos is set to + rotSkyPos_desired. If rotSkyPos and rotTelPos are both NaN, + and rotSkyPos_desired results in an out of range value for the + camera rotator, then rotTelPos_backup is used. + rotTelPos_backup : `float` + Rotation angle of the camera relative to the telescope (radians). + Only used as a last resort if rotSkyPos and rotTelPos are set + to NaN and rotSkyPos_desired results in an out of range rotator + value. + nexp : `int` + Number of exposures in the visit. + flush_by_mjd : `float` + If we hit this MJD, we should flush the queue and refill it. + scheduler_note : `str` (optional) + Usually good to set the note field so one knows which survey + object generated the observation. + target_name : `str` (optional) + A note about what target is being observed. + This maps to target_name in the ConsDB. + Generally would be used to identify DD, ToO or special targets. + science_program : `str` (optional) + Science program being executed. + This maps to science_program in the ConsDB, although can + be overwritten in JSON BLOCK. + Generally would be used to identify a particular program for DM. + observation_reason : `str` (optional) + General 'reason' for observation, for DM purposes. + (for scheduler purposes, use `scheduler_note`). + This maps to observation_reason in the ConsDB, although could + be overwritten in JSON BLOCK. + Most likely this is just "science" or "FBS" when using the FBS. + + Notes + ----- + + On the camera rotator angle. Order of priority goes: + rotTelPos > rotSkyPos > rotSkyPos_desired > rotTelPos_backup + where if rotTelPos is NaN, it checks rotSkyPos. If rotSkyPos is set, + but not at an accessible rotTelPos, the observation will fail. + If rotSkyPos is NaN, then rotSkyPos_desired is used. If + rotSkyPos_desired is at an inaccessbile rotTelPos, the observation + does not fail, but falls back to the value in rotTelPos_backup. + + Lots of additional fields that get filled in by the model observatory + when the observation is completed. + See documentation at: + https://rubin-scheduler.lsst.io/output_schema.html + + """ + + def __new__(cls, n=1): + dtypes = [ + ("ID", int), + ("RA", float), + ("dec", float), + ("mjd", float), + ("flush_by_mjd", float), + ("exptime", float), + ("filter", "U40"), + ("rotSkyPos", float), + ("rotSkyPos_desired", float), + ("nexp", int), + ("airmass", float), + ("FWHM_500", float), + ("FWHMeff", float), + ("FWHM_geometric", float), + ("skybrightness", float), + ("night", int), + ("slewtime", float), + ("visittime", float), + ("slewdist", float), + ("fivesigmadepth", float), + ("alt", float), + ("az", float), + ("pa", float), + ("pseudo_pa", float), + ("clouds", float), + ("moonAlt", float), + ("sunAlt", float), + ("scheduler_note", "U40"), + ("target_name", "U40"), + ("target_id", int), + ("lmst", float), + ("rotTelPos", float), + ("rotTelPos_backup", float), + ("moonAz", float), + ("sunAz", float), + ("sunRA", float), + ("sunDec", float), + ("moonRA", float), + ("moonDec", float), + ("moonDist", float), + ("solarElong", float), + ("moonPhase", float), + ("cummTelAz", float), + ("observation_reason", "U40"), + ("science_program", "U40"), + ] + obj = np.zeros(n, dtype=dtypes).view(cls) + return obj + + def tolist(self): + """Convert to a list of 1-element arrays""" + obs_list = [] + for obs in self: + new_obs = self.__class__(n=1) + new_obs[0] = obs + obs_list.append(new_obs) + + return obs_list + + +class ScheduledObservationArray(ObservationArray): + """Make an array to hold pre-scheduling observations + + Note + ---- + mjd_tol : `float` + The tolerance on how early an observation can execute (days). + Observation will be considered valid to attempt + when mjd-mjd_tol < current MJD < flush_by_mjd (and other + conditions below pass) + dist_tol : `float` + The angular distance an observation can be away from the + specified RA,Dec and still count as completing the observation + (radians). + alt_min : `float` + The minimum altitude to consider executing the observation + (radians). + alt_max : `float` + The maximuim altitude to try observing (radians). + HA_max : `float` + Hour angle limit. Constraint is such that for hour angle + running from 0 to 24 hours, the target RA,Dec must be greather + than HA_max and less than HA_min. Set HA_max to 0 for no + limit. (hours) + HA_min : `float` + Hour angle limit. Constraint is such that for hour angle + running from 0 to 24 hours, the target RA,Dec must be greather + than HA_max and less than HA_min. Set HA_min to 24 for + no limit. (hours) + sun_alt_max : `float` + The sun must be below sun_alt_max to execute. (radians) + moon_min_distance : `float` + The minimum distance to demand the moon should be away (radians) + observed : `bool` + If set to True, scheduler will probably consider this a + completed observation and never attempt it. + + """ + + def __new__(cls, n=1): + # Standard things from the usual observations + dtypes1 = [ + ("ID", int), + ("RA", float), + ("dec", float), + ("mjd", float), + ("flush_by_mjd", float), + ("exptime", float), + ("filter", "U1"), + ("rotSkyPos", float), + ("rotTelPos", float), + ("rotTelPos_backup", float), + ("rotSkyPos_desired", float), + ("nexp", int), + ("scheduler_note", "U40"), + ("target_name", "U40"), + ("science_program", "U40"), + ("observation_reason", "U40"), + ] + + # New things not in standard ObservationArray + dtype2 = [ + ("mjd_tol", float), + ("dist_tol", float), + ("alt_min", float), + ("alt_max", float), + ("HA_max", float), + ("HA_min", float), + ("sun_alt_max", float), + ("moon_min_distance", float), + ("observed", bool), + ] + + obj = np.zeros(n, dtype=dtypes1 + dtype2).view(cls) + return obj + + def to_observation_array(self): + """Convert the scheduled observation to a + Regular ObservationArray + """ + result = ObservationArray(n=self.size) + in_common = np.intersect1d(self.dtype.names, result.dtype.names) + for key in in_common: + result[key] = self[key] + return result + + +def obsarray_concat(in_arrays): + """Concatenate ObservationArray objects. + + Can't use np.concatenate because it will no longer + be an array subclass + + Parameters + ---------- + in_arrays : `list` of `ObservationArray` or `ScheduledObservationArray` + """ + # Check if we have ScheduledObservationArray + array_class = ObservationArray + if "observed" in in_arrays[0].dtype.names: + array_class = ScheduledObservationArray + + size = 0 + for arr in in_arrays: + size += arr.size + # Init empty array of proper class + # to hold output. + out_arr = array_class(n=size) + return np.concatenate(in_arrays, out=out_arr) diff --git a/rubin_scheduler/scheduler/utils/utils.py b/rubin_scheduler/scheduler/utils/utils.py index 66274b23..9fedcb7c 100644 --- a/rubin_scheduler/scheduler/utils/utils.py +++ b/rubin_scheduler/scheduler/utils/utils.py @@ -11,8 +11,6 @@ "SimTargetooServer", "restore_scheduler", "warm_start", - "ObservationArray", - "ScheduledObservationArray", "gnomonic_project_toxy", "gnomonic_project_tosky", "raster_sort", @@ -24,7 +22,6 @@ "xyz2thetaphi", "mean_azimuth", "wrap_ra_dec", - "obsarray_concat", ) import datetime @@ -39,6 +36,7 @@ import pandas as pd import rubin_scheduler.version as rsVersion +from rubin_scheduler.scheduler.utils.observation_array import ObservationArray from rubin_scheduler.utils import ( DEFAULT_NSIDE, _build_tree, @@ -572,262 +570,6 @@ def opsim2obs(self, filename): return self.opsimdf2obs(df) -class ObservationArray(np.ndarray): - """Class to work as an array of observations - - Parameters - ---------- - n : `int` - Size of array to return. Default 1. - - The numpy fields have the following labels. - - RA : `float` - The Right Acension of the observation (center of the field) - (Radians) - dec : `float` - Declination of the observation (Radians) - mjd : `float` - Modified Julian Date at the start of the observation - (time shutter opens) - exptime : `float` - Total exposure time of the visit (seconds) - filter : `str` - The filter used. Should be one of u, g, r, i, z, y. - rotSkyPos : `float` - The rotation angle of the camera relative to the sky E of N - (Radians). Will be ignored if rotTelPos is finite. - If rotSkyPos is set to NaN, rotSkyPos_desired is used. - rotTelPos : `float` - The rotation angle of the camera relative to the telescope - (radians). Set to np.nan to force rotSkyPos to be used. - rotSkyPos_desired : `float` - If both rotSkyPos and rotTelPos are None/NaN, then - rotSkyPos_desired (radians) is used. If rotSkyPos_desired - results in a valid rotTelPos, rotSkyPos is set to - rotSkyPos_desired. If rotSkyPos and rotTelPos are both NaN, - and rotSkyPos_desired results in an out of range value for the - camera rotator, then rotTelPos_backup is used. - rotTelPos_backup : `float` - Rotation angle of the camera relative to the telescope (radians). - Only used as a last resort if rotSkyPos and rotTelPos are set - to NaN and rotSkyPos_desired results in an out of range rotator - value. - nexp : `int` - Number of exposures in the visit. - flush_by_mjd : `float` - If we hit this MJD, we should flush the queue and refill it. - scheduler_note : `str` (optional) - Usually good to set the note field so one knows which survey - object generated the observation. - target_name : `str` (optional) - A note about what target is being observed. - This maps to target_name in the ConsDB. - Generally would be used to identify DD, ToO or special targets. - science_program : `str` (optional) - Science program being executed. - This maps to science_program in the ConsDB, although can - be overwritten in JSON BLOCK. - Generally would be used to identify a particular program for DM. - observation_reason : `str` (optional) - General 'reason' for observation, for DM purposes. - (for scheduler purposes, use `scheduler_note`). - This maps to observation_reason in the ConsDB, although could - be overwritten in JSON BLOCK. - Most likely this is just "science" or "FBS" when using the FBS. - - Notes - ----- - - On the camera rotator angle. Order of priority goes: - rotTelPos > rotSkyPos > rotSkyPos_desired > rotTelPos_backup - where if rotTelPos is NaN, it checks rotSkyPos. If rotSkyPos is set, - but not at an accessible rotTelPos, the observation will fail. - If rotSkyPos is NaN, then rotSkyPos_desired is used. If - rotSkyPos_desired is at an inaccessbile rotTelPos, the observation - does not fail, but falls back to the value in rotTelPos_backup. - - Lots of additional fields that get filled in by the model observatory - when the observation is completed. - See documentation at: - https://rubin-scheduler.lsst.io/output_schema.html - - """ - - def __new__(cls, n=1): - dtypes = [ - ("ID", int), - ("RA", float), - ("dec", float), - ("mjd", float), - ("flush_by_mjd", float), - ("exptime", float), - ("filter", "U40"), - ("rotSkyPos", float), - ("rotSkyPos_desired", float), - ("nexp", int), - ("airmass", float), - ("FWHM_500", float), - ("FWHMeff", float), - ("FWHM_geometric", float), - ("skybrightness", float), - ("night", int), - ("slewtime", float), - ("visittime", float), - ("slewdist", float), - ("fivesigmadepth", float), - ("alt", float), - ("az", float), - ("pa", float), - ("pseudo_pa", float), - ("clouds", float), - ("moonAlt", float), - ("sunAlt", float), - ("note", "U40"), - ("scheduler_note", "U40"), - ("target_name", "U40"), - ("block_id", int), - ("lmst", float), - ("rotTelPos", float), - ("rotTelPos_backup", float), - ("moonAz", float), - ("sunAz", float), - ("sunRA", float), - ("sunDec", float), - ("moonRA", float), - ("moonDec", float), - ("moonDist", float), - ("solarElong", float), - ("moonPhase", float), - ("cummTelAz", float), - ("observation_reason", "U40"), - ("science_program", "U40"), - ] - obj = np.zeros(n, dtype=dtypes).view(cls) - return obj - - def tolist(self): - """Convert to a list of 1-element arrays""" - obs_list = [] - for obs in self: - new_obs = self.__class__(n=1) - new_obs[0] = obs - obs_list.append(new_obs) - - return obs_list - - -class ScheduledObservationArray(ObservationArray): - """Make an array to hold pre-scheduling observations - - Note - ---- - mjd_tol : `float` - The tolerance on how early an observation can execute (days). - Observation will be considered valid to attempt - when mjd-mjd_tol < current MJD < flush_by_mjd (and other - conditions below pass) - dist_tol : `float` - The angular distance an observation can be away from the - specified RA,Dec and still count as completing the observation - (radians). - alt_min : `float` - The minimum altitude to consider executing the observation - (radians). - alt_max : `float` - The maximuim altitude to try observing (radians). - HA_max : `float` - Hour angle limit. Constraint is such that for hour angle - running from 0 to 24 hours, the target RA,Dec must be greather - than HA_max and less than HA_min. Set HA_max to 0 for no - limit. (hours) - HA_min : `float` - Hour angle limit. Constraint is such that for hour angle - running from 0 to 24 hours, the target RA,Dec must be greather - than HA_max and less than HA_min. Set HA_min to 24 for - no limit. (hours) - sun_alt_max : `float` - The sun must be below sun_alt_max to execute. (radians) - moon_min_distance : `float` - The minimum distance to demand the moon should be away (radians) - observed : `bool` - If set to True, scheduler will probably consider this a - completed observation and never attempt it. - - """ - - def __new__(cls, n=1): - # Standard things from the usual observations - dtypes1 = [ - ("ID", int), - ("RA", float), - ("dec", float), - ("mjd", float), - ("flush_by_mjd", float), - ("exptime", float), - ("filter", "U1"), - ("rotSkyPos", float), - ("rotTelPos", float), - ("rotTelPos_backup", float), - ("rotSkyPos_desired", float), - ("nexp", int), - ("scheduler_note", "U40"), - ("target_name", "U40"), - ("science_program", "U40"), - ("observation_reason", "U40"), - ] - - # New things not in standard ObservationArray - dtype2 = [ - ("mjd_tol", float), - ("dist_tol", float), - ("alt_min", float), - ("alt_max", float), - ("HA_max", float), - ("HA_min", float), - ("sun_alt_max", float), - ("moon_min_distance", float), - ("observed", bool), - ] - - obj = np.zeros(n, dtype=dtypes1 + dtype2).view(cls) - return obj - - def to_observation_array(self): - """Convert the scheduled observation to a - Regular ObservationArray - """ - result = ObservationArray(n=self.size) - in_common = np.intersect1d(self.dtype.names, result.dtype.names) - for key in in_common: - result[key] = self[key] - return result - - -def obsarray_concat(in_arrays): - """Concatenate ObservationArray objects. - - Can't use np.concatenate because it will no longer - be an array subclass - - Parameters - ---------- - in_arrays : `list` of `ObservationArray` or `ScheduledObservationArray` - """ - # Check if we have ScheduledObservationArray - array_class = ObservationArray - if "observed" in in_arrays[0].dtype.names: - array_class = ScheduledObservationArray - - size = 0 - for arr in in_arrays: - size += arr.size - # Init empty array of proper class - # to hold output. - out_arr = array_class(n=size) - return np.concatenate(in_arrays, out=out_arr) - - def hp_kd_tree(nside=DEFAULT_NSIDE, leafsize=100, scale=1e5): """ Generate a KD-tree of healpixel locations