Tickets/sp 1786: Adding target_id (#143)

docs/fbs-output-schema.rst

@@ -101,18 +101,12 @@ All values are for the center of the field of view (e.g., airmass, altitude, etc
    * - sunAlt
      - degrees
      - Altitude of the Sun.
-   * - note
-     - string
-     - DEPRECATED. Do not use.
    * - scheduler_note
      - string
      - Descriptive comment about how the observations were scheduled, for use by the FBS.
    * - target_name
      - string
      - Descriptive name for the target. Only used for DDFs, ToOs, or other special targets. Should translate to target_name in the headers/ConsDB.
-   * - block_id
-     - integer
-     - Identification ID of the block (used by some survey objects).
    * - observationStartLST
      - degrees
      - Local Sidereal Time at the start of the observation.
@@ -152,6 +146,9 @@ All values are for the center of the field of view (e.g., airmass, altitude, etc
    * - cummTelAz
      - degrees
      - Cumulative azimuth of the telescope mount, tracks cable wrap.
+   * - target_id
+     - integer
+     - Integer added by the `CoreScheduler`.
    * - observation_reason
      - string
      - The reason for the observation. Identifier for DM. Translates to observation_reason in the headers/consdb.

rubin_scheduler/scheduler/schedulers/core_scheduler.py

@@ -41,6 +41,10 @@ class CoreScheduler:
     telescope : `str`
         Which telescope model to use for rotTelPos/rotSkyPos conversions.
         Default "rubin".
+    target_id_counter : int
+        Starting value for the target_id. If restarting observations, could
+        be useful to set to whatever value the scheduler was at previously.
+        Default 0.
     """
 
     def __init__(
@@ -51,6 +55,7 @@ def __init__(
         log=None,
         keep_rewards=False,
         telescope="rubin",
+        target_id_counter=0,
     ):
         self.keep_rewards = keep_rewards
         # Use integer ns just to be sure there are no rounding issues.
@@ -86,9 +91,12 @@ def __init__(
 
         self.rc = rotation_converter(telescope=telescope)
 
-        # keep track of how many observations get flushed from the queue
+        # Keep track of how many observations get flushed from the queue
         self.flushed = 0
 
+        # Counter for observations added to the queue
+        self.target_id_counter = target_id_counter
+
     def flush_queue(self):
         """Like it sounds, clear any currently queued desired observations."""
         self.queue = []
@@ -127,6 +135,9 @@ def add_observations_array(self, obs):
             for survey in surveys:
                 survey.add_observations_array(obs, obs_array_hpid)
 
+        if np.max(obs["target_id"]) >= self.target_id_counter:
+            self.target_id_counter = np.max(obs["target_id"]) + 1
+
     def add_observation(self, observation):
         """
         Record a completed observation and update features accordingly.
@@ -154,6 +165,9 @@ def add_observation(self, observation):
             for survey in surveys:
                 survey.add_observation(observation, indx=indx)
 
+        if np.max(observation["target_id"]) >= self.target_id_counter:
+            self.target_id_counter = np.max(observation["target_id"]) + 1
+
     def update_conditions(self, conditions_in):
         """
         Parameters
@@ -302,14 +316,16 @@ def _fill_queue(self):
             # Take a min here, so the surveys will be executed in the order
             # they are entered if there is a tie.
             self.survey_index[1] = np.min(np.where(rewards == np.nanmax(rewards)))
-            # Survey returns ObservationArray. Convert to list.
-            result = (
-                self.survey_lists[self.survey_index[0]][self.survey_index[1]]
-                .generate_observations(self.conditions)
-                .tolist()
+            # Survey returns ObservationArray
+            result = self.survey_lists[self.survey_index[0]][self.survey_index[1]].generate_observations(
+                self.conditions
             )
+            # Tag with a unique target_id
+            result["target_id"] = np.arange(self.target_id_counter, self.target_id_counter + result.size)
+            self.target_id_counter += result.size
 
-            self.queue = result
+            # Convert to a list for the queue
+            self.queue = result.tolist()
             self.queue_filled = self.conditions.mjd
 
         if len(self.queue) == 0:

rubin_scheduler/scheduler/surveys/surveys.py

@@ -510,7 +510,6 @@ def generate_observations_rough(self, conditions):
             observations["nexp"] = self.nexp_dict[self.filtername1]
         observations["exptime"] = self.exptime
         observations["scheduler_note"] = self.scheduler_note
-        observations["block_id"] = self.counter
         observations["flush_by_mjd"] = flush_time
 
         return observations

rubin_scheduler/scheduler/utils/__init__.py

@@ -1,5 +1,6 @@
 from .comcam_tessellate import *
 from .footprints import *
+from .observation_array import *
 from .sky_area import *
 from .tsp import *
 from .utils import *

rubin_scheduler/scheduler/utils/observation_array.py

@@ -0,0 +1,262 @@
+__all__ = (
+    "obsarray_concat",
+    "ObservationArray",
+    "ScheduledObservationArray",
+)
+
+import numpy as np
+
+
+class ObservationArray(np.ndarray):
+    """Class to work as an array of observations
+
+    Parameters
+    ----------
+    n : `int`
+        Size of array to return. Default 1.
+
+    The numpy fields have the following labels.
+
+    RA : `float`
+       The Right Acension of the observation (center of the field)
+       (Radians)
+    dec : `float`
+       Declination of the observation (Radians)
+    mjd : `float`
+       Modified Julian Date at the start of the observation
+       (time shutter opens)
+    exptime : `float`
+       Total exposure time of the visit (seconds)
+    filter : `str`
+        The filter used. Should be one of u, g, r, i, z, y.
+    rotSkyPos : `float`
+        The rotation angle of the camera relative to the sky E of N
+        (Radians). Will be ignored if rotTelPos is finite.
+        If rotSkyPos is set to NaN, rotSkyPos_desired is used.
+    rotTelPos : `float`
+        The rotation angle of the camera relative to the telescope
+        (radians). Set to np.nan to force rotSkyPos to be used.
+    rotSkyPos_desired : `float`
+        If both rotSkyPos and rotTelPos are None/NaN, then
+        rotSkyPos_desired (radians) is used. If rotSkyPos_desired
+        results in a valid rotTelPos, rotSkyPos is set to
+        rotSkyPos_desired. If rotSkyPos and rotTelPos are both NaN,
+        and rotSkyPos_desired results in an out of range value for the
+        camera rotator, then rotTelPos_backup is used.
+    rotTelPos_backup : `float`
+        Rotation angle of the camera relative to the telescope (radians).
+        Only used as a last resort if rotSkyPos and rotTelPos are set
+        to NaN and rotSkyPos_desired results in an out of range rotator
+        value.
+    nexp : `int`
+        Number of exposures in the visit.
+    flush_by_mjd : `float`
+        If we hit this MJD, we should flush the queue and refill it.
+    scheduler_note : `str` (optional)
+        Usually good to set the note field so one knows which survey
+        object generated the observation.
+    target_name : `str` (optional)
+        A note about what target is being observed.
+        This maps to target_name in the ConsDB.
+        Generally would be used to identify DD, ToO or special targets.
+    science_program : `str` (optional)
+        Science program being executed.
+        This maps to science_program in the ConsDB, although can
+        be overwritten in JSON BLOCK.
+        Generally would be used to identify a particular program for DM.
+    observation_reason : `str` (optional)
+        General 'reason' for observation, for DM purposes.
+        (for scheduler purposes, use `scheduler_note`).
+        This maps to observation_reason in the ConsDB, although could
+        be overwritten in JSON BLOCK.
+        Most likely this is just "science" or "FBS" when using the FBS.
+
+    Notes
+    -----
+
+    On the camera rotator angle. Order of priority goes:
+    rotTelPos > rotSkyPos > rotSkyPos_desired > rotTelPos_backup
+    where if rotTelPos is NaN, it checks rotSkyPos. If rotSkyPos is set,
+    but not at an accessible rotTelPos, the observation will fail.
+    If rotSkyPos is NaN, then rotSkyPos_desired is used. If
+    rotSkyPos_desired is at an inaccessbile rotTelPos, the observation
+    does not fail, but falls back to the value in rotTelPos_backup.
+
+    Lots of additional fields that get filled in by the model observatory
+    when the observation is completed.
+    See documentation at:
+    https://rubin-scheduler.lsst.io/output_schema.html
+
+    """
+
+    def __new__(cls, n=1):
+        dtypes = [
+            ("ID", int),
+            ("RA", float),
+            ("dec", float),
+            ("mjd", float),
+            ("flush_by_mjd", float),
+            ("exptime", float),
+            ("filter", "U40"),
+            ("rotSkyPos", float),
+            ("rotSkyPos_desired", float),
+            ("nexp", int),
+            ("airmass", float),
+            ("FWHM_500", float),
+            ("FWHMeff", float),
+            ("FWHM_geometric", float),
+            ("skybrightness", float),
+            ("night", int),
+            ("slewtime", float),
+            ("visittime", float),
+            ("slewdist", float),
+            ("fivesigmadepth", float),
+            ("alt", float),
+            ("az", float),
+            ("pa", float),
+            ("pseudo_pa", float),
+            ("clouds", float),
+            ("moonAlt", float),
+            ("sunAlt", float),
+            ("scheduler_note", "U40"),
+            ("target_name", "U40"),
+            ("target_id", int),
+            ("lmst", float),
+            ("rotTelPos", float),
+            ("rotTelPos_backup", float),
+            ("moonAz", float),
+            ("sunAz", float),
+            ("sunRA", float),
+            ("sunDec", float),
+            ("moonRA", float),
+            ("moonDec", float),
+            ("moonDist", float),
+            ("solarElong", float),
+            ("moonPhase", float),
+            ("cummTelAz", float),
+            ("observation_reason", "U40"),
+            ("science_program", "U40"),
+        ]
+        obj = np.zeros(n, dtype=dtypes).view(cls)
+        return obj
+
+    def tolist(self):
+        """Convert to a list of 1-element arrays"""
+        obs_list = []
+        for obs in self:
+            new_obs = self.__class__(n=1)
+            new_obs[0] = obs
+            obs_list.append(new_obs)
+
+        return obs_list
+
+
+class ScheduledObservationArray(ObservationArray):
+    """Make an array to hold pre-scheduling observations
+
+    Note
+    ----
+    mjd_tol : `float`
+        The tolerance on how early an observation can execute (days).
+        Observation will be considered valid to attempt
+        when mjd-mjd_tol < current MJD < flush_by_mjd (and other
+        conditions below pass)
+    dist_tol : `float`
+        The angular distance an observation can be away from the
+        specified RA,Dec and still count as completing the observation
+        (radians).
+    alt_min : `float`
+        The minimum altitude to consider executing the observation
+        (radians).
+    alt_max : `float`
+        The maximuim altitude to try observing (radians).
+    HA_max : `float`
+        Hour angle limit. Constraint is such that for hour angle
+        running from 0 to 24 hours, the target RA,Dec must be greather
+        than HA_max and less than HA_min. Set HA_max to 0 for no
+        limit. (hours)
+    HA_min : `float`
+        Hour angle limit. Constraint is such that for hour angle
+        running from 0 to 24 hours, the target RA,Dec must be greather
+        than HA_max and less than HA_min. Set HA_min to 24 for
+        no limit. (hours)
+    sun_alt_max : `float`
+        The sun must be below sun_alt_max to execute. (radians)
+    moon_min_distance : `float`
+        The minimum distance to demand the moon should be away (radians)
+    observed : `bool`
+        If set to True, scheduler will probably consider this a
+        completed observation and never attempt it.
+
+    """
+
+    def __new__(cls, n=1):
+        # Standard things from the usual observations
+        dtypes1 = [
+            ("ID", int),
+            ("RA", float),
+            ("dec", float),
+            ("mjd", float),
+            ("flush_by_mjd", float),
+            ("exptime", float),
+            ("filter", "U1"),
+            ("rotSkyPos", float),
+            ("rotTelPos", float),
+            ("rotTelPos_backup", float),
+            ("rotSkyPos_desired", float),
+            ("nexp", int),
+            ("scheduler_note", "U40"),
+            ("target_name", "U40"),
+            ("science_program", "U40"),
+            ("observation_reason", "U40"),
+        ]
+
+        # New things not in standard ObservationArray
+        dtype2 = [
+            ("mjd_tol", float),
+            ("dist_tol", float),
+            ("alt_min", float),
+            ("alt_max", float),
+            ("HA_max", float),
+            ("HA_min", float),
+            ("sun_alt_max", float),
+            ("moon_min_distance", float),
+            ("observed", bool),
+        ]
+
+        obj = np.zeros(n, dtype=dtypes1 + dtype2).view(cls)
+        return obj
+
+    def to_observation_array(self):
+        """Convert the scheduled observation to a
+        Regular ObservationArray
+        """
+        result = ObservationArray(n=self.size)
+        in_common = np.intersect1d(self.dtype.names, result.dtype.names)
+        for key in in_common:
+            result[key] = self[key]
+        return result
+
+
+def obsarray_concat(in_arrays):
+    """Concatenate ObservationArray objects.
+
+    Can't use np.concatenate because it will no longer
+    be an array subclass
+
+    Parameters
+    ----------
+    in_arrays : `list` of `ObservationArray` or `ScheduledObservationArray`
+    """
+    # Check if we have ScheduledObservationArray
+    array_class = ObservationArray
+    if "observed" in in_arrays[0].dtype.names:
+        array_class = ScheduledObservationArray
+
+    size = 0
+    for arr in in_arrays:
+        size += arr.size
+    # Init empty array of proper class
+    # to hold output.
+    out_arr = array_class(n=size)
+    return np.concatenate(in_arrays, out=out_arr)