Skip to content

Commit

Permalink
Add cik annotation platform
Browse files Browse the repository at this point in the history
  • Loading branch information
@aldro61
aldro61 committed Jan 17, 2025
1 parent 985ba63 commit 84336d1
Show file tree
Hide file tree
Showing 3 changed files with 617 additions and 0 deletions.
94 changes: 94 additions & 0 deletions annotation/index.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>CIK Capability Annotations</title>
<link rel="stylesheet" href="styles.css">
</head>
<body>
<div id="container">
<h1 style="padding-bottom: 20px;">Context is Key Benchmark - Annotator Platform</h1>

<!-- Source URL and Fetch Button -->
<div class="source-wrapper">
<label for="page-url" class="source-label">Source:</label>
<input type="text" id="page-url" class="source-input" placeholder="Enter URL here...">
<button class="fetch-button" onclick="fetchTasks()">Reset</button>
</div>

<!-- Progress Bar and Download Button -->
<div id="progress-wrapper">
<div id="progress-container">
<div id="progress-bar"></div>
</div>
<span id="progress-info">0/0</span>
<button onclick="downloadJSON()">Download Results</button>
</div>

<!-- Annotation Instruction -->
<p class="progress-instruction">
Please inspect the task below and annotate it by selecting which capabilities are required to infer the ground truth time series based on the provided contextual information and history. Before proceeding, please read the <a href="#" class="definition-link" onclick="showDefinitions()">definition</a> of each capability carefully.
Once you are done, please download the results and send us the json file.
</p>

<!-- Content and Annotation Pane -->
<div class="content-wrapper">
<div id="rendered-content" style="display: none;"></div>
<div id="annotation-pane" style="display: none;">
<h3>Capabilities</h3>
<p class="annotation-instructions">
Please select all that apply.
</p>
<form id="capabilities-form">
<label class="capability-checkbox">
<input type="checkbox" value="instruction following"> Instruction Following
</label>
<label class="capability-checkbox">
<input type="checkbox" value="retrieval: context"> Retrieval: Context
</label>
<label class="capability-checkbox">
<input type="checkbox" value="retrieval: memory"> Retrieval: Memory
</label>
<label class="capability-checkbox">
<input type="checkbox" value="reasoning: analogy"> Reasoning: Analogy
</label>
<label class="capability-checkbox">
<input type="checkbox" value="reasoning: deduction"> Reasoning: Deduction
</label>
<label class="capability-checkbox">
<input type="checkbox" value="reasoning: math"> Reasoning: Math
</label>
<label class="capability-checkbox">
<input type="checkbox" value="reasoning: causal"> Reasoning: Causal
</label>
<div class="button-container">
<button type="button" class="next-button" onclick="saveAnnotationsAndNext()" disabled>Save</button>
<button type="button" class="skip-button" onclick="skipTask()">Skip</button>
</div>
</form>
</div>
</div>
</div>

<!-- Definitions Modal -->
<div id="definitions-modal" class="modal" style="display: none;">
<div class="modal-content">
<span class="close-button" onclick="closeDefinitions()">&times;</span>
<h2>Capability Definitions</h2>
<ul>
<li><strong>Instruction Following:</strong> Ability to execute tasks based on explicit instructions.</li>
<li><strong>Retrieval: Context:</strong> Extracting relevant information from external sources to contextualize the task.</li>
<li><strong>Retrieval: Memory:</strong> Leveraging stored or previously learned information to solve the task.</li>
<li><strong>Reasoning: Analogy:</strong> Drawing parallels between similar situations to deduce solutions.</li>
<li><strong>Reasoning: Deduction:</strong> Applying logical rules to derive conclusions from given premises.</li>
<li><strong>Reasoning: Math:</strong> Solving problems involving numerical and mathematical reasoning.</li>
<li><strong>Reasoning: Causal:</strong> Identifying and reasoning about cause-and-effect relationships.</li>
</ul>
</div>
</div>

<!-- Include Scripts -->
<script src="./scripts.js"></script>
</body>
</html>
252 changes: 252 additions & 0 deletions annotation/scripts.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,252 @@
// Show the definitions modal
function showDefinitions() {
const modal = document.getElementById("definitions-modal");
modal.style.display = "flex"; // Show the modal
}

// Close the definitions modal
function closeDefinitions() {
const modal = document.getElementById("definitions-modal");
modal.style.display = "none"; // Hide the modal
}

// Update the progress bar
function updateProgressBar() {
const dictionary = JSON.parse(localStorage.getItem("cik_capability_annotations")) || {};
const totalTasks = Object.keys(dictionary).length;
const annotatedTasks = Object.values(dictionary).filter(task => task.annotations.length > 0).length;

const progressPercentage = totalTasks > 0 ? (annotatedTasks / totalTasks) * 100 : 0;
document.getElementById("progress-bar").style.width = `${progressPercentage}%`;

// Update progress info
const progressInfo = `${annotatedTasks}/${totalTasks}`;
document.getElementById("progress-info").textContent = progressInfo;
}

// Fetch and render a task page
async function fetchAndRenderPage(taskUrl) {
try {
const response = await fetch(taskUrl);
if (!response.ok) {
throw new Error(`Failed to fetch page: ${response.statusText}`);
}

const htmlText = await response.text();

// Parse the HTML content
const parser = new DOMParser();
const doc = parser.parseFromString(htmlText, "text/html");

// Remove <p> elements containing "Capabilities:" or "Types of context:"
const paragraphs = doc.querySelectorAll("p");
paragraphs.forEach(p => {
if (p.textContent.includes("Capabilities:") || p.textContent.includes("Types of context:")) {
p.remove();
}
});

// Remove all buttons with class "back-button"
const backButtons = doc.querySelectorAll("button.back-button");
backButtons.forEach(button => button.remove());

// Copy stylesheets and inline styles
const stylesheets = Array.from(doc.querySelectorAll('link[rel="stylesheet"], style'));
const renderedContent = document.getElementById("rendered-content");
renderedContent.innerHTML = ""; // Clear previous content

stylesheets.forEach(style => {
const clonedStyle = style.cloneNode(true);
document.head.appendChild(clonedStyle);
});

// Render the modified content in the #rendered-content div
renderedContent.style.display = "block";
renderedContent.innerHTML = doc.body.innerHTML;

// Show the annotation pane
document.getElementById("annotation-pane").style.display = "block";

} catch (error) {
alert(`Error: ${error.message}`);
}
}

// Save annotations and move to the next task
function saveAnnotationsAndNext() {
const dictionary = JSON.parse(localStorage.getItem("cik_capability_annotations")) || {};
const currentTask = Object.keys(dictionary).find(task => dictionary[task].annotations.length === 0);

if (!currentTask) {
alert("All tasks are annotated!");
return;
}

// Get selected capabilities
const form = document.getElementById("capabilities-form");
const selectedCapabilities = Array.from(form.elements)
.filter(input => input.checked)
.map(input => input.value);

// Save annotations to the dictionary
dictionary[currentTask].annotations = selectedCapabilities;
localStorage.setItem("cik_capability_annotations", JSON.stringify(dictionary));

// Reset checkboxes
form.reset();
toggleNextButtonState();

// Update the progress bar
updateProgressBar();

// Proceed to the next task
openRandomUnannotatedTask();
}

// Skip the current task and load a random one
function skipTask() {
document.getElementById("capabilities-form").reset();
openRandomUnannotatedTask();
}

// Open a random unannotated task
function openRandomUnannotatedTask() {
const dictionary = JSON.parse(localStorage.getItem("cik_capability_annotations")) || {};
const unannotatedTasks = Object.values(dictionary).filter(task => task.annotations.length === 0);

if (unannotatedTasks.length > 0) {
// Select a random unannotated task
const randomIndex = Math.floor(Math.random() * unannotatedTasks.length);
const randomTask = unannotatedTasks[randomIndex];

// Load the random task
fetchAndRenderPage(randomTask.url);
} else {
alert("All tasks are annotated!");
}
}

// Helper function to shuffle an array
function shuffleArray(array) {
for (let i = array.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
[array[i], array[j]] = [array[j], array[i]];
}
return array;
}

// Fetch tasks from the source URL
async function fetchTasks() {
const pageUrl = document.getElementById("page-url").value;
const existingData = localStorage.getItem("cik_capability_annotations");

if (existingData) {
const overwrite = confirm("Found existing annotations in memory. Do you want to overwrite them?");
if (!overwrite) {
updateProgressBar();
return openRandomUnannotatedTask();
}
}

try {
// Fetch the HTML content of the page
const response = await fetch(pageUrl);
if (!response.ok) throw new Error(`HTTP error! Status: ${response.status}`);

const text = await response.text();

// Parse the HTML content
const parser = new DOMParser();
const doc = parser.parseFromString(text, "text/html");

// Find all <a> elements within <li>
const anchors = doc.querySelectorAll("li a");

// Create an array of task entries
const tasks = [];
anchors.forEach(anchor => {
const href = anchor.getAttribute("href")?.trim(); // Get the href attribute
const text = anchor.textContent.trim(); // Get the text content of the list item
if (href && text) {
const fullUrl = new URL(href, pageUrl).href; // Resolve relative URL
tasks.push([text, { url: fullUrl, annotations: [] }]); // Add as a key-value pair
}
});

// Shuffle the tasks
const randomizedTasks = shuffleArray(tasks);

// Convert the shuffled array back to a dictionary
const dictionary = Object.fromEntries(randomizedTasks);

// Store the dictionary in session storage
localStorage.setItem("cik_capability_annotations", JSON.stringify(dictionary));

// Update progress and open the first unannotated task
updateProgressBar();
openRandomUnannotatedTask();
} catch (error) {
alert(`Error: ${error.message}`);
}
}

// Download annotations as a JSON file
function downloadJSON() {
const dictionary = JSON.parse(localStorage.getItem("cik_capability_annotations"));
if (!dictionary) {
alert("No data to download!");
return;
}

const totalTasks = Object.keys(dictionary).length;
const annotatedTasks = Object.values(dictionary).filter(task => task.annotations.length > 0).length;

// Check if all tasks are annotated
if (annotatedTasks < totalTasks) {
const proceed = confirm(`Only ${annotatedTasks}/${totalTasks} tasks are annotated. Are you sure you want to download partial results?`);
if (!proceed) return; // Exit if the user cancels
}

const blob = new Blob([JSON.stringify(dictionary, null, 2)], { type: "application/json" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = "cik_capability_annotations.json";
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
}

// Disable the Next button initially
function toggleNextButtonState() {
const form = document.getElementById("capabilities-form");
const checkboxes = Array.from(form.elements).filter(input => input.type === "checkbox");
const nextButton = form.querySelector(".next-button");

// Enable the Next button if at least one checkbox is checked
const isAnyChecked = checkboxes.some(checkbox => checkbox.checked);
nextButton.disabled = !isAnyChecked; // Disable if none are checked
}

// Attach event listeners to checkboxes to update the button state
function initializeCheckboxListeners() {
const form = document.getElementById("capabilities-form");
const checkboxes = Array.from(form.elements).filter(input => input.type === "checkbox");

checkboxes.forEach(checkbox => {
checkbox.addEventListener("change", toggleNextButtonState);
});

// Ensure the button state is updated on page load
toggleNextButtonState();
}

// Call initializeCheckboxListeners on page load
window.onload = function () {
const defaultUrl = new URLSearchParams(window.location.search).get("url") || "https://anon-forecast.github.io/benchmark_report_dev/";
document.getElementById("page-url").value = defaultUrl;

fetchTasks();
initializeCheckboxListeners();
showDefinitions();
};
Loading

0 comments on commit 84336d1

Please sign in to comment.