diff --git a/frontend/__tests__/routes/_oh.test.tsx b/frontend/__tests__/routes/_oh.test.tsx
index 8897b5dd8bd3..e22d959446a8 100644
--- a/frontend/__tests__/routes/_oh.test.tsx
+++ b/frontend/__tests__/routes/_oh.test.tsx
@@ -39,12 +39,6 @@ describe("frontend/routes/_oh", () => {
     await screen.findByTestId("root-layout");
   });
 
-  it("should render the AI config modal if the user is authed", async () => {
-    // Our mock return value is true by default
-    renderWithProviders(<RouteStub />);
-    await screen.findByTestId("ai-config-modal");
-  });
-
   it("should render the AI config modal if settings are not up-to-date", async () => {
     settingsAreUpToDateMock.mockReturnValue(false);
     renderWithProviders(<RouteStub />);
diff --git a/frontend/src/routes/_oh/route.tsx b/frontend/src/routes/_oh/route.tsx
index 5055e24d2383..27058be55c94 100644
--- a/frontend/src/routes/_oh/route.tsx
+++ b/frontend/src/routes/_oh/route.tsx
@@ -45,12 +45,15 @@ export function ErrorBoundary() {
 
 export default function MainApp() {
   const { gitHubToken, clearToken } = useAuth();
-  const { settings } = useUserPrefs();
+  const { settings, settingsAreUpToDate } = useUserPrefs();
 
   const [consentFormIsOpen, setConsentFormIsOpen] = React.useState(
     !localStorage.getItem("analytics-consent"),
   );
 
+  const [aiConfigModalIsOpen, setAiConfigModalIsOpen] =
+    React.useState(!settingsAreUpToDate);
+
   const config = useConfig();
   const {
     data: isAuthed,
@@ -77,9 +80,6 @@ export default function MainApp() {
   const isInWaitlist =
     !isFetchingAuth && !isAuthed && config.data?.APP_MODE === "saas";
 
-  const { settingsAreUpToDate } = useUserPrefs();
-  const [showAIConfig, setShowAIConfig] = React.useState(true);
-
   return (
     <div
       data-testid="root-layout"
@@ -101,8 +101,11 @@ export default function MainApp() {
         />
       )}
 
-      {(isAuthed || !settingsAreUpToDate) && showAIConfig && (
-        <SettingsModal onClose={() => setShowAIConfig(false)} />
+      {aiConfigModalIsOpen && (
+        <SettingsModal
+          onClose={() => setAiConfigModalIsOpen(false)}
+          data-testid="ai-config-modal"
+        />
       )}
     </div>
   );