remove scripted_id (#131)

docs/fbs-output-schema.rst

@@ -152,9 +152,6 @@ All values are for the center of the field of view (e.g., airmass, altitude, etc
    * - cummTelAz
      - degrees
      - Cumulative azimuth of the telescope mount, tracks cable wrap.
-   * - scripted_id
-     - integer
-     - Links scheduled observations with their acquisition.
    * - observation_reason
      - string
      - The reason for the observation. Identifier for DM. Translates to observation_reason in the headers/consdb.

rubin_scheduler/scheduler/surveys/ddf_presched.py

@@ -515,6 +515,4 @@ def generate_ddf_scheduled_obs(
                     all_scheduled_obs.append(obs)
 
     result = np.concatenate(all_scheduled_obs)
-    # Put in the scripted ID so it's easier to track which ones fail.
-    result["scripted_id"] = np.arange(result.size)
     return result

rubin_scheduler/scheduler/surveys/long_gap_survey.py

@@ -198,11 +198,8 @@ def _schedule_obs(self, observations):
             # Don't let the desired rotSkyPos block the observation.
             sched_array["rotSkyPos_desired"] = sched_array["rotSkyPos"]
             sched_array["rotSkyPos"] = np.nan
-            # See if we need to append things to the scripted survey object
-            if self.scripted_survey.obs_wanted is not None:
-                sched_array = np.concatenate([self.scripted_survey.obs_wanted, sched_array])
 
-            self.scripted_survey.set_script(sched_array)
+            self.scripted_survey.set_script(sched_array, append=True)
 
     def add_observations_array(self, observations_array_in, observations_hpid_in):
         self._schedule_obs(observations_array_in)

rubin_scheduler/scheduler/surveys/scripted_surveys.py

@@ -6,7 +6,7 @@
 import numpy as np
 
 from rubin_scheduler.scheduler.surveys import BaseSurvey
-from rubin_scheduler.scheduler.utils import ObservationArray
+from rubin_scheduler.scheduler.utils import ObservationArray, ScheduledObservationArray
 from rubin_scheduler.utils import DEFAULT_NSIDE, _angular_separation, _approx_ra_dec2_alt_az
 
 log = logging.getLogger(__name__)
@@ -105,22 +105,16 @@ def add_observations_array(self, observations_array_in, observations_hpid_in):
             for detailer in self.detailers:
                 detailer.add_observations_array(observations_array, observations_hpid)
 
-            # If scripted_id, note, and filter match, then consider
-            # the observation completed.
-            completed = np.char.add(
-                observations_array["scripted_id"].astype(str),
-                observations_array["scheduler_note"],
-            )
-            completed = np.char.add(completed, observations_array["filter"])
+            if (self.obs_wanted is not None) & (np.size(self.obs_wanted) > 0):
+                full_note_in = np.char.add(
+                    observations_array_in["scheduler_note"], observations_array_in["filter"]
+                )
+                full_note_queue = np.char.add(self.obs_wanted["scheduler_note"], self.obs_wanted["filter"])
 
-            wanted = np.char.add(
-                self.obs_wanted["scripted_id"].astype(str), self.obs_wanted["scheduler_note"]
-            )
-            wanted = np.char.add(wanted, self.obs_wanted["filter"])
+                indx = np.in1d(full_note_queue, full_note_in)
 
-            indx = np.in1d(wanted, completed)
-            self.obs_wanted["observed"][indx] = True
-            self.scheduled_obs = self.obs_wanted["mjd"][~self.obs_wanted["observed"]]
+                self.obs_wanted["observed"][indx] = True
+                self.scheduled_obs = self.obs_wanted["mjd"][~self.obs_wanted["observed"]]
 
     def add_observation(self, observation, indx=None, **kwargs):
         """Check if observation matches a scripted observation"""
@@ -136,18 +130,14 @@ def add_observation(self, observation, indx=None, **kwargs):
                     detailer.add_observation(observation, **kwargs)
                 self.reward_checked = False
 
-                # Find the index
-                indx = np.where(self.obs_wanted["scripted_id"] == observation["scripted_id"])[0]
-                # If it matches scripted_id, note, and filter, mark it as
-                # observed and update scheduled observation list.
-                if indx.size > 0:
-                    if (
-                        (self.obs_wanted["scripted_id"][indx] == observation["scripted_id"])
-                        & (self.obs_wanted["scheduler_note"][indx] == observation["scheduler_note"])
-                        & (self.obs_wanted["filter"][indx] == observation["filter"])
-                    ):
-                        self.obs_wanted["observed"][indx] = True
-                        self.scheduled_obs = self.obs_wanted["mjd"][~self.obs_wanted["observed"]]
+                key = observation["scheduler_note"][0] + observation["filter"][0]
+                try:
+                    # Could add an additional check here for if the observation
+                    # is in the desired mjd window.
+                    self.obs_wanted["observed"][self.note_filter_dict[key]] = True
+                # If the key does not exist, nothing to do
+                except KeyError:
+                    return
 
     def calc_reward_function(self, conditions):
         """If there is an observation ready to go, execute it,
@@ -173,7 +163,6 @@ def _slice2obs(self, obs_row):
             "rotSkyPos",
             "rotTelPos",
             "flush_by_mjd",
-            "scripted_id",
         ]:
             observation[key] = obs_row[key]
         return observation
@@ -255,7 +244,7 @@ def _check_alts_ha(self, observation, conditions):
     def _check_list(self, conditions):
         """Check to see if the current mjd is good"""
         observations = None
-        if self.obs_wanted is not None:
+        if self.obs_wanted.size > 0:
             # Scheduled observations that are in the right time
             # window and have not been executed
             in_time_window = np.where(
@@ -303,11 +292,12 @@ def _check_list(self, conditions):
 
     def clear_script(self):
         """set an empty list to serve up"""
-        self.obs_wanted = None
+        self.obs_wanted = ScheduledObservationArray(n=0)
         self.mjd_start = None
         self.scheduled_obs = None
+        self.note_filter_dict = {}
 
-    def set_script(self, obs_wanted):
+    def set_script(self, obs_wanted, append=True, add_index=True):
         """
         Parameters
         ----------
@@ -321,22 +311,56 @@ def set_script(self, obs_wanted):
         mjd_tol : float (15.)
             The tolerance to consider an observation as still good to
             observe (min)
+        append : `bool`
+            Should the obs_wanted be appended to any existing scheduled
+            observations. Default True.
+        add_index : `bool`
+            Should the scheduler_note be modified to include a unique
+            index value. Default True.
         """
 
-        self.obs_wanted = obs_wanted
+        obs_wanted.sort(order=["mjd", "filter"])
 
-        self.obs_wanted.sort(order=["mjd", "filter"])
-        # Give each desired observation a unique "scripted ID". To be used for
-        # matching and logging later.
-        self.obs_wanted["scripted_id"] = np.arange(self.id_start, self.id_start + np.size(self.obs_wanted))
-        # Update so if we set the script again the IDs will not be reused.
-        self.id_start = np.max(self.obs_wanted["scripted_id"]) + 1
+        if add_index:
+            sep = [", "] * obs_wanted.size
+            indx = np.arange(self.id_start, self.id_start + obs_wanted.size, 1).astype(str)
+            new_note = np.char.add(obs_wanted["scheduler_note"], sep)
+            obs_wanted["scheduler_note"] = np.char.add(new_note, indx)
+            self.id_start += obs_wanted.size
+        else:
+            # go through and pull the index as an int
+            self.script_id_array = np.array(
+                [int(note.split(", ")[-1]) for note in self.obs_wanted["scheduler_note"]]
+            )
+            if np.size(self.script_id_array) > 0:
+                self.id_start = self.script_id_array.max() + 1
+
+        if append & (self.obs_wanted is not None):
+            self.obs_wanted = np.concatenate([self.obs_wanted, obs_wanted])
+            self.obs_wanted.sort(order=["mjd", "filter"])
+        else:
+            self.obs_wanted = obs_wanted
+
+        # check that we have valid unique keys for observations
+        potential_keys = np.char.add(self.obs_wanted["scheduler_note"], self.obs_wanted["filter"])
+        if np.size(np.unique(potential_keys)) < np.size(self.obs_wanted):
+            msg = (
+                "Scripted observations do not have unique scheduler_note "
+                "+ filter values. Consider setting add_index=True"
+            )
+            raise ValueError(msg)
 
         self.mjd_start = self.obs_wanted["mjd"] - self.obs_wanted["mjd_tol"]
         # Here is the attribute that core scheduler checks to
         # broadcast scheduled observations in the conditions object.
         self.scheduled_obs = self.obs_wanted["mjd"]
 
+        # Generate a dict so it can be fast to check if an observation matches
+        # the combination of scheduler_note and filter
+        self.note_filter_dict = {}
+        for i, obs in enumerate(self.obs_wanted):
+            self.note_filter_dict[obs["scheduler_note"] + obs["filter"]] = i
+
     def generate_observations_rough(self, conditions):
         # if we have already called for this mjd, no need to repeat.
         if self.last_mjd == conditions.mjd:

rubin_scheduler/scheduler/surveys/too_scripted_surveys.py

@@ -181,40 +181,10 @@ def __init__(
         # list to keep track of alerts we have already seen
         self.seen_alerts = []
 
-    # Need to make sure new set_script call doesn't clobber old script!
-    def set_script(self, obs_wanted, append=True):
-        """
-        Parameters
-        ----------
-        obs_wanted : rubin_scheduler.scheduler.utils.ScheduledObservationArray
-            The observations that should be executed.
-        append : bool
-            Should the obs_wanted be appended to any script already set?
-        """
-
-        obs_wanted.sort(order=["mjd", "filter"])
-        # Give each desired observation a unique "scripted ID". To be used for
-        # matching and logging later.
-        obs_wanted["scripted_id"] = np.arange(self.id_start, self.id_start + np.size(obs_wanted))
-        # Update so if we set the script again the IDs will not be reused.
-        self.id_start = np.max(obs_wanted["scripted_id"]) + 1
-
-        # If we already have a script and append
-        if append & (self.obs_wanted is not None):
-            self.obs_wanted = np.concatenate([self.obs_wanted, obs_wanted])
-            self.obs_wanted.sort(order=["mjd", "filter"])
-        else:
-            self.obs_wanted = obs_wanted
-
-        self.mjd_start = self.obs_wanted["mjd"] - self.obs_wanted["mjd_tol"]
-        # Here is the atribute that core scheduler checks to
-        # broadcast scheduled observations in the conditions object.
-        self.scheduled_obs = self.obs_wanted["mjd"]
-
     def _check_list(self, conditions):
         """Check to see if the current mjd is good"""
         observation = None
-        if self.obs_wanted is not None:
+        if self.obs_wanted.size > 0:
             # Scheduled observations that are in the right
             # time window and have not been executed
             in_time_window = np.where(
@@ -243,7 +213,7 @@ def flush_script(self, conditions):
 
             if np.size(still_relevant) > 0:
                 observations = self.obs_wanted[still_relevant]
-                self.set_script(observations, append=False)
+                self.set_script(observations, append=False, add_index=False)
             else:
                 self.clear_script()
 

rubin_scheduler/scheduler/utils/utils.py

@@ -669,7 +669,6 @@ def __new__(cls, n=1):
             ("solarElong", float),
             ("moonPhase", float),
             ("cummTelAz", float),
-            ("scripted_id", int),
             ("observation_reason", "U40"),
             ("science_program", "U40"),
         ]
@@ -748,7 +747,6 @@ def __new__(cls, n=1):
             ("sun_alt_max", float),
             ("moon_min_distance", float),
             ("observed", bool),
-            ("scripted_id", int),
         ]
 
         obj = np.zeros(n, dtype=dtypes1 + dtype2).view(cls)