diff --git a/docs/1-architecture/1-core-concepts.md b/docs/1-architecture/1-core-concepts.md index 70a95a1..0985954 100644 --- a/docs/1-architecture/1-core-concepts.md +++ b/docs/1-architecture/1-core-concepts.md @@ -39,8 +39,7 @@ export const state = { lastModified: null, // Last-Modified header from keywords file // Filter Settings - targetKeywordCount: 100, // Target number of keywords (default: minimal) - filterLevel: 0, // Current filter level (0-3) + filterLevel: 0, // Current filter level (0=Minimal to 3=Complete) lastBulkAction: null // Track when enable/disable all is used } ``` @@ -67,7 +66,6 @@ const saveData = { manuallyUnchecked: Array.from(state.manuallyUnchecked), mode: state.mode, lastModified: state.lastModified, - targetKeywordCount: state.targetKeywordCount, filterLevel: state.filterLevel, lastBulkAction: state.lastBulkAction } @@ -103,7 +101,7 @@ try { - Mode (returns to 'simple') - Selections (contexts, exceptions, categories) - UI state (search, filter, menu) - - Filter settings (level, target count) + - Filter level (returns to 0) ### Cache Management @@ -145,10 +143,14 @@ const debouncedUpdate = (() => { ## Mode System ### Simple Mode -- Context-based filtering with filter levels +- Context-based filtering with filter levels (0-3) - Keywords derived from selected contexts - Exceptions for granular control -- Filter levels determine target keyword count +- Filter levels determine keyword thresholds: + * Level 0 (Minimal) = Most restrictive + * Level 1 (Moderate) = Balanced filtering + * Level 2 (Extensive) = Broader inclusion + * Level 3 (Complete) = Most inclusive ### Advanced Mode - Direct keyword management @@ -179,7 +181,7 @@ const debouncedUpdate = (() => { 3. Performance - Use cache for expensive calculations - Throttle rapid updates (50ms threshold) - - Clear cache when target count changes + - Clear cache on filter level changes - Batch process large operations 4. Mode Management diff --git a/docs/1-architecture/11-simple-mode.md b/docs/1-architecture/11-simple-mode.md index d9224e3..8fe4c01 100644 --- a/docs/1-architecture/11-simple-mode.md +++ b/docs/1-architecture/11-simple-mode.md @@ -10,25 +10,33 @@ Simple mode provides an intuitive interface for content filtering through contex ```javascript class SimpleMode extends HTMLElement { constructor() { - this.currentLevel = 0; // Default level - this.levelTargets = { - 0: 100, // Minimal - 1: 300, // Moderate - 2: 500, // Extensive - 3: 2000 // Complete - }; + this.currentLevel = 0; // Default level (Minimal/most restrictive) } updateLevel(level) { if (level === this.currentLevel) return; this.currentLevel = level; + state.filterLevel = level; this.updateFilterUI(); - setTargetKeywordCount(this.levelTargets[level]); } } ``` -### 2. Context Management System +### 2. Weight Threshold System +```javascript +function getWeightThreshold(filterLevel) { + // Map levels to thresholds (0-3) + switch(filterLevel) { + case 0: return 3; // Minimal (most restrictive) + case 1: return 2; // Moderate + case 2: return 1; // Extensive + case 3: return 0; // Complete (most inclusive) + default: return 3; // Default to most restrictive + } +} +``` + +### 3. Context Management System #### Context Selection Handler ```javascript @@ -94,7 +102,7 @@ export function handleContextToggle(contextId) { } ``` -### 3. Exception System +### 4. Exception System #### Exception Toggle Handler ```javascript diff --git a/docs/1-architecture/4-mode-system.md b/docs/1-architecture/4-mode-system.md index 860fd82..008012c 100644 --- a/docs/1-architecture/4-mode-system.md +++ b/docs/1-architecture/4-mode-system.md @@ -3,11 +3,56 @@ ## Overview MuteSky operates in two distinct modes: -- Simple Mode: Context-based filtering with filter levels +- Simple Mode: Context-based filtering with filter levels (0-3) - Advanced Mode: Direct keyword management The system maintains consistency between these modes while preserving user preferences. +## Weight System Implementation + +### 1. Filter Level System +```javascript +// Map filter levels to thresholds +function getWeightThreshold(filterLevel) { + switch(filterLevel) { + case 0: return 3; // Minimal (most restrictive) + case 1: return 2; // Moderate + case 2: return 1; // Extensive + case 3: return 0; // Complete (most inclusive) + default: return 3; + } +} +``` + +### 2. Filter Level Handler +```javascript +export function handleFilterLevelChange(event) { + const level = event.detail.level; + state.filterLevel = level; + + // Store current exceptions + const currentExceptions = new Set(state.selectedExceptions); + + // Clear and rebuild active keywords + state.activeKeywords.clear(); + state.selectedContexts.forEach(contextId => { + const context = state.contextGroups[contextId]; + if (context?.categories) { + context.categories.forEach(category => { + if (!currentExceptions.has(category)) { + const keywords = getAllKeywordsForCategory(category, true); + keywords.forEach(keyword => state.activeKeywords.add(keyword)); + } + }); + } + }); + + // Restore exceptions and update UI + state.selectedExceptions = currentExceptions; + renderInterface(); +} +``` + ## Context System Implementation ### 1. Context Toggle Handler diff --git a/docs/migration.md b/docs/migration.md index be94300..d46fb09 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -1,172 +1,31 @@ -# Keyword Weighting System Migration - -## Current System - -The current system uses weights 1-10 for both categories and keywords, with higher numbers being more significant. - -### Category Weights -- 9-10: Most significant categories (e.g., Economic Policy, Education) -- 7-8: Important categories (e.g., Climate and Environment, Healthcare) -- 5-6: Extended coverage categories -- 1-4: Basic coverage categories - -### Keyword Weights -- 7-8: Highly frequent/significant terms -- 5-6: Common/regular terms -- 3-4: Occasional/moderate terms -- 1-2: Rare/basic terms - -### Distribution Levels -The system currently has four distribution levels with actual keyword counts: -- Minimal: 190 highest weighted keywords -- Moderate: 413 keywords -- Extensive: 815 keywords -- Complete: All remaining keywords (~2000+ total keywords) - -## Weight Threshold Algorithm - -The current algorithm in weightManager.js determines which keywords to include based on both category and keyword weights: - -```javascript -case 190: // Minimal - return categoryWeight >= 9 ? 8 : // For highest categories (9), include keywords weighted 8+ - categoryWeight >= 8 ? 9 : // For high categories (8), only include keywords weighted 9 - 11; // For others, exclude all - -case 413: // Moderate - return categoryWeight >= 9 ? 7 : // For highest categories, include keywords weighted 7+ - categoryWeight >= 8 ? 8 : // For high categories, include keywords weighted 8+ - 9; // For others, only highest weighted keywords - -case 815: // Extensive - return categoryWeight >= 9 ? 6 : // For highest categories, include keywords weighted 6+ - categoryWeight >= 8 ? 7 : // For high categories, include keywords weighted 7+ - 8; // For others, include keywords weighted 8+ -``` - -## New Scale (0-3) - -The new scale inverts the power relationship, with 0 being least significant and 3 being most significant. This aligns with common programming practices where array indices and enums typically start at 0. - -### Category Weights -- 3: Most significant categories (previously 9-10) -- 2: Important categories (previously 7-8) -- 1: Extended coverage categories (previously 5-6) -- 0: Basic coverage categories (previously 1-4) - -### Keyword Weights -- 3: Highly frequent/significant terms (previously 7-8) -- 2: Common/regular terms (previously 5-6) -- 1: Occasional/moderate terms (previously 3-4) -- 0: Rare/basic terms (previously 1-2) - -### Distribution Levels -The distribution levels remain the same but with inverted significance: -- Level 0 (Complete): All keywords (~2000+) -- Level 1 (Extensive): 815 keywords -- Level 2 (Moderate): 413 keywords -- Level 3 (Minimal): 190 keywords - -### Keyword Distribution - -The actual distribution of keywords across levels: -- Level 3 (Minimal): Top 190 keywords from highest weighted categories - * Category weight 3: Keywords weighted 3 - * Category weight 2: Keywords weighted 3 - * Others: None included -- Level 2 (Moderate): 413 keywords - * Category weight 3: Keywords weighted 2-3 - * Category weight 2: Keywords weighted 3 - * Others: Keywords weighted 3 only -- Level 1 (Extensive): 815 keywords - * Category weight 3: Keywords weighted 1-3 - * Category weight 2: Keywords weighted 2-3 - * Others: Keywords weighted 3 -- Level 0 (Complete): All 2000+ keywords included - -### Examples - -#### Economic Policy (Category Weight 3, previously 9) -- "recession" (Weight 3, previously 9): Highly frequent economic term -- "debt ceiling" (Weight 3, previously 8): Highly frequent policy crisis -- "banking crisis" (Weight 2, previously 7): Frequent financial term -- "tax cut" (Weight 1, previously 6): Common policy term -- "capital gains" (Weight 0, previously 4): Technical tax term - -#### Climate and Environment (Category Weight 2, previously 8) -- "climate change" (Weight 3, previously 9): Highly frequent environmental term -- "extreme heat" (Weight 3, previously 8): Frequent weather crisis term -- "drought" (Weight 2, previously 7): Frequent weather crisis term -- "carbon footprint" (Weight 1, previously 5): Regular environmental impact term -- "desertification" (Weight 0, previously 4): Occasional environmental term - -## Migration Benefits - -1. **Intuitive Scaling**: 0-3 provides a clearer, more concise range compared to 1-10 -2. **Programming Alignment**: Starts at 0, matching common programming patterns -3. **Simplified Logic**: Four distinct levels make the weighting system more straightforward -4. **Maintained Relationships**: Preserves the existing keyword distribution and category importance while using a cleaner scale - -## Implementation Steps - -1. **Update Category Files** - - Convert category weights: - * 9-10 → 3 - * 7-8 → 2 - * 5-6 → 1 - * 1-4 → 0 - - Convert keyword weights: - * 8-10 → 3 - * 7-6 → 2 - * 3-4 → 1 - * 1-2 → 0 - -2. **Update weightManager.js** - ```javascript - case 190: // Level 3 (Minimal) - return categoryWeight === 3 ? 3 : // For highest categories, include keywords weighted 3 - categoryWeight === 2 ? 3 : // For high categories, include keywords weighted 3 - 4; // For others, exclude all - - case 413: // Level 2 (Moderate) - return categoryWeight === 3 ? 2 : // For highest categories, include keywords weighted 2+ - categoryWeight === 2 ? 3 : // For high categories, include keywords weighted 3 - 3; // For others, only highest weighted keywords - - case 815: // Level 1 (Extensive) - return categoryWeight === 3 ? 1 : // For highest categories, include keywords weighted 1+ - categoryWeight === 2 ? 2 : // For high categories, include keywords weighted 2+ - 3; // For others, include keywords weighted 3 - ``` - -3. **Update UI Components** - - Modify any UI elements that display weight values - - Update any sorting logic that depends on weights - - Ensure filtering mechanisms reflect the new scale - -4. **Update Tests** - - Modify test cases to use new weight values - - Update expected results in keyword filtering tests - - Add migration-specific tests to verify correct weight conversion - -5. **Documentation Updates** - - Update API documentation - - Update user guides - - Add migration notes for developers - -## Migration Safety - -### Validation Steps -1. **Pre-migration Validation** - - Count total keywords at each level - - Generate distribution report for each category - - Verify current keyword inclusion patterns - -2. **Post-migration Validation** - - Verify total keyword counts match pre-migration - - Confirm keyword inclusion patterns are preserved - - Check category distribution matches expected patterns - - Validate that Level 3 (Minimal) still contains the same 190 most significant keywords - -### Rollback Procedure -1. Don't worry about it, we use source control. \ No newline at end of file +# Migration Notes + +## Weight System Simplification (January 2024) + +### Changes Made +1. Simplified weight system from complex thresholds to 0-3 scale: + - Level 0 (Minimal) = threshold 3 (most restrictive) + - Level 1 (Moderate) = threshold 2 + - Level 2 (Extensive) = threshold 1 + - Level 3 (Complete) = threshold 0 (most inclusive) + +2. Removed targetKeywordCount: + - Removed from state + - Removed setTargetKeywordCount function + - Updated state persistence + - Simplified filterLevel handling + +3. Removed category weights: + - Weight thresholds now based only on keyword weights + - Simplified filtering logic + - Maintained case sensitivity handling + +### Future Considerations +1. Categories will be removed in a future update +2. Current category-related code is maintained for backwards compatibility +3. New features should use filterLevel and keyword weights only + +### Migration Path +- Old state using targetKeywordCount will default to filterLevel 0 +- Existing keyword weights (0-3) work directly with new thresholds +- Category weights are ignored but preserved in data structure for now diff --git a/js/api.js b/js/api.js index e2aff02..be429a4 100644 --- a/js/api.js +++ b/js/api.js @@ -126,9 +126,6 @@ export async function fetchKeywordGroups(forceFresh = false) { const categoryFiles = await listCategoryFiles(); console.debug('Found category files:', categoryFiles); - // Get the target count from state or default to 2000 - const targetCount = state.targetKeywordCount || 2000; - // Fetch and process each category file const keywordGroups = {}; const results = await Promise.allSettled(categoryFiles.map(async (fileName) => { @@ -143,7 +140,7 @@ export async function fetchKeywordGroups(forceFresh = false) { // Store the entire category data structure keywordGroups[categoryName] = categoryData; - console.debug(`Loaded ${categoryName} with weight ${categoryData[categoryName].weight} and ${Object.keys(categoryData[categoryName].keywords).length} keywords`); + console.debug(`Loaded ${categoryName} with ${Object.keys(categoryData[categoryName].keywords).length} keywords`); } catch (error) { console.error(`Failed to load category ${fileName}:`, error); } @@ -208,7 +205,7 @@ export async function refreshAllData() { const selectedCategories = new Set(state.selectedCategories); const currentMode = state.mode; const menuOpen = state.menuOpen; - const targetCount = state.targetKeywordCount; + const filterLevel = state.filterLevel; // Preserve auth state const did = state.did; const authenticated = state.authenticated; @@ -230,7 +227,7 @@ export async function refreshAllData() { state.selectedCategories = selectedCategories; state.mode = currentMode; state.menuOpen = menuOpen; - state.targetKeywordCount = targetCount; + state.filterLevel = filterLevel; // Restore auth state state.did = did; state.authenticated = authenticated; diff --git a/js/categoryManager.js b/js/categoryManager.js index e6ff0dd..4ab2ee4 100644 --- a/js/categoryManager.js +++ b/js/categoryManager.js @@ -11,7 +11,7 @@ function calculateKeywordsToMute() { if (context && context.categories) { context.categories.forEach(category => { if (!state.selectedExceptions.has(category)) { - // Get keywords sorted by weight and limited by target count + // Get keywords sorted and filtered by weight based on current filter level const keywords = getAllKeywordsForCategory(category, true); console.debug(`Adding ${keywords.length} keywords from ${category} to mute list`); keywords.forEach(keyword => keywordsToMute.add(keyword)); diff --git a/js/handlers/context/contextCache.js b/js/handlers/context/contextCache.js index 2241f4a..d29d1b9 100644 --- a/js/handlers/context/contextCache.js +++ b/js/handlers/context/contextCache.js @@ -13,7 +13,7 @@ export const cache = { updateThreshold: 16, getKeywords(category, sortByWeight = false) { - const key = `${category}-${sortByWeight}-${state.targetKeywordCount}`; + const key = `${category}-${sortByWeight}-${state.filterLevel}`; if (!this.keywords.has(key)) { this.manageCache(this.keywords); const keywords = getAllKeywordsForCategory(category, sortByWeight); @@ -23,7 +23,7 @@ export const cache = { }, getActiveKeywordsForCategory(category) { - const key = `active-${category}-${state.targetKeywordCount}`; + const key = `active-${category}-${state.filterLevel}`; if (!this.activeKeywordsByCategory.has(key)) { this.manageCache(this.activeKeywordsByCategory); const keywords = this.getKeywords(category, true); @@ -46,7 +46,7 @@ export const cache = { }, getContextKeywords(contextId) { - const key = `${contextId}-${state.targetKeywordCount}`; + const key = `${contextId}-${state.filterLevel}`; if (!this.contextKeywords.has(key)) { this.manageCache(this.contextKeywords); const context = state.contextGroups[contextId]; diff --git a/js/keywordState.js b/js/keywordState.js index 5593186..a490566 100644 --- a/js/keywordState.js +++ b/js/keywordState.js @@ -84,13 +84,3 @@ export function getMuteUnmuteCounts() { return { toMute, toUnmute }; } - -// Helper to set target keyword count and trigger refresh -export function setTargetKeywordCount(count) { - const validCounts = new Set([100, 300, 500, 2000]); - if (!validCounts.has(count)) { - throw new Error('Invalid target keyword count. Must be one of: 100, 300, 500, 2000'); - } - state.targetKeywordCount = count; - keywordCache.clear(); // Clear cache when count changes -} diff --git a/js/main.js b/js/main.js index c4a0e7a..92cc22c 100644 --- a/js/main.js +++ b/js/main.js @@ -1,5 +1,5 @@ import { elements } from './dom.js'; -import { state, loadState, setTargetKeywordCount } from './state.js'; +import { state, loadState } from './state.js'; import { fetchKeywordGroups, fetchContextGroups, fetchDisplayConfig } from './api.js'; import { renderInterface } from './renderer.js'; import { debounce } from './utils.js'; @@ -187,23 +187,12 @@ function setupEventListeners() { document.addEventListener('filterLevelChange', (event) => { const level = event.detail.level; - // Map intensity levels to keyword counts based on performance thresholds - const levelToCount = { - 0: 100, // Minimal: ~100 highest weighted keywords - 1: 300, // Moderate: ~300 keywords - 2: 500, // Extensive: ~500 keywords - 3: 2000 // Complete: All keywords - }; - - // Update filter level in state to match event + // Update filter level in state state.filterLevel = level; // Store current exceptions const currentExceptions = new Set(state.selectedExceptions); - // Update target keyword count based on intensity level - setTargetKeywordCount(levelToCount[level]); - // Clear and rebuild active keywords while preserving exceptions state.activeKeywords.clear(); state.selectedContexts.forEach(contextId => { @@ -211,7 +200,7 @@ function setupEventListeners() { if (context && context.categories) { context.categories.forEach(category => { if (!currentExceptions.has(category)) { - // Get keywords sorted by weight and limited by new target count + // Get keywords sorted by weight const keywords = getAllKeywordsForCategory(category, true); keywords.forEach(keyword => state.activeKeywords.add(keyword)); } diff --git a/js/state.js b/js/state.js index bee5eb5..dcc3664 100644 --- a/js/state.js +++ b/js/state.js @@ -1,6 +1,6 @@ import { loadState, saveState, resetState, forceRefresh, getStorageKey } from './statePersistence.js'; import { setUser } from './userState.js'; -import { canUnmuteKeyword, getMuteUnmuteCounts, setTargetKeywordCount } from './keywordState.js'; +import { canUnmuteKeyword, getMuteUnmuteCounts } from './keywordState.js'; // Core state object export const state = { @@ -21,8 +21,7 @@ export const state = { filterMode: 'all', menuOpen: false, lastModified: null, // Last-Modified header from keywords file - targetKeywordCount: 100, // Default to minimal keywords since default mode is simple - filterLevel: 0, // Track current filter level + filterLevel: 0, // Track current filter level (0=Minimal to 3=Complete) lastBulkAction: null // Track when enable/disable all is used }; @@ -35,6 +34,5 @@ export { setUser, canUnmuteKeyword, getMuteUnmuteCounts, - setTargetKeywordCount, getStorageKey }; diff --git a/js/statePersistence.js b/js/statePersistence.js index 8b2ce0a..aaad525 100644 --- a/js/statePersistence.js +++ b/js/statePersistence.js @@ -25,7 +25,6 @@ const debouncedSave = (() => { manuallyUnchecked: Array.from(state.manuallyUnchecked), mode: state.mode, lastModified: state.lastModified, - targetKeywordCount: state.targetKeywordCount, filterLevel: state.filterLevel, lastBulkAction: state.lastBulkAction }; @@ -78,15 +77,11 @@ export function loadState() { // Load other state properties state.mode = data.mode || 'simple'; state.lastModified = data.lastModified || null; - state.targetKeywordCount = data.targetKeywordCount || (state.mode === 'simple' ? 100 : 2000); state.filterLevel = typeof data.filterLevel === 'number' ? data.filterLevel : 0; state.lastBulkAction = data.lastBulkAction || null; // Force cache refresh keywordCache.clear(); - } else { - // If no saved state, ensure targetKeywordCount matches mode - state.targetKeywordCount = state.mode === 'simple' ? 100 : 2000; } } catch (error) { console.error('Error loading saved state:', error); @@ -118,7 +113,6 @@ export function resetState() { state.filterMode = 'all'; state.menuOpen = false; state.lastModified = null; - state.targetKeywordCount = 100; state.filterLevel = 0; state.lastBulkAction = null; keywordCache.clear(); diff --git a/js/utils/categoryUtils.js b/js/utils/categoryUtils.js index 32cf354..87d7e52 100644 --- a/js/utils/categoryUtils.js +++ b/js/utils/categoryUtils.js @@ -30,7 +30,6 @@ export function extractKeywordsFromCategory(category, categoryData) { return Object.entries(categoryInfo.keywords).map(([keyword, data]) => ({ keyword, weight: data.weight || 0, - categoryWeight: categoryInfo.weight || 0, category })); } @@ -60,7 +59,7 @@ export function getAllKeywordsForCategory(category, sortByWeight = false) { if (sortByWeight) { keywords.sort((a, b) => b.weight - a.weight); - if (state.targetKeywordCount) { + if (state.filterLevel !== undefined) { const before = keywords.length; keywords = filterByWeight(keywords, category); logFilterResults(category, keywords, before); @@ -73,19 +72,19 @@ export function getAllKeywordsForCategory(category, sortByWeight = false) { function filterByWeight(keywords, category) { return keywords.filter(k => { - const threshold = getWeightThreshold(k.categoryWeight, state.targetKeywordCount); + const threshold = getWeightThreshold(state.filterLevel); const passes = k.weight >= threshold; if (passes) { - console.debug(`Including ${k.keyword} (weight: ${k.weight}) from ${k.category} (weight: ${k.categoryWeight})`); + console.debug(`Including ${k.keyword} (weight: ${k.weight}) from ${k.category}`); } return passes; }); } function logFilterResults(category, keywords, beforeCount) { - console.debug(`Category ${category} (weight ${keywords[0]?.categoryWeight || 'unknown'}): - - Target count: ${state.targetKeywordCount} - - Threshold: ${getWeightThreshold(keywords[0]?.categoryWeight, state.targetKeywordCount)} + console.debug(`Category ${category}: + - Filter level: ${state.filterLevel} + - Threshold: ${getWeightThreshold(state.filterLevel)} - Filtered from ${beforeCount} to ${keywords.length} keywords - Remaining keywords: ${keywords.map(k => `${k.keyword} (${k.weight})`).join(', ')}`); } diff --git a/js/utils/weightManager.js b/js/utils/weightManager.js index a8943c4..850fb11 100644 --- a/js/utils/weightManager.js +++ b/js/utils/weightManager.js @@ -1,22 +1,21 @@ -function getWeightThreshold(categoryWeight, targetCount) { - switch(targetCount) { - case 100: - return categoryWeight === 10 ? 8 : - categoryWeight === 9 ? 8 : - categoryWeight === 8 ? 8 : - categoryWeight === 7 ? 10 : 11; - case 300: - return categoryWeight === 10 ? 7 : - categoryWeight === 9 ? 7 : - categoryWeight === 8 ? 8 : - categoryWeight === 7 ? 9 : 11; - case 500: - return categoryWeight === 10 ? 4 : - categoryWeight === 9 ? 5 : - categoryWeight === 8 ? 6 : - categoryWeight === 7 ? 7 : 11; - default: +import { state } from '../state.js'; + +function getWeightThreshold(filterLevel) { + // Get filter level from state if not provided + const level = filterLevel ?? state?.filterLevel ?? 0; + + // Map levels to thresholds based on keyword weight of 3 + switch(level) { + case 0: // Minimal (most restrictive) + return 3; + case 1: // Moderate + return 2; + case 2: // Extensive + return 1; + case 3: // Complete (most inclusive) return 0; + default: + return 3; // Default to most restrictive } }