update nocapd && publisher to nip66@draft7

apps/nocapd/package.json

@@ -12,7 +12,7 @@
     "@nostrwatch/nocap": "0.5.2",
     "@nostrwatch/nocap-every-adapter-default": "1.4.0",
     "@nostrwatch/nwcache": "0.1.2",
-    "@nostrwatch/publisher": "0.4.3",
+    "@nostrwatch/publisher": "0.5.0",
     "@nostrwatch/seed": "0.0.2",
     "@nostrwatch/utils": "0.1.3",
     "bluebird": "3.7.2",

apps/nocapd/src/classes/Worker.js

@@ -133,14 +133,8 @@ export class NWWorker {
   async on_success(result){
     if(this.hard_stop) return
     this.log.debug(`on_success(): ${result.url}`)
-    if(this.config?.publisher?.kinds?.includes(30066) ){
-      const publish30066 = new Publish.Kind30066()
-      await publish30066.one( result ).catch(this.log.error)  
-    }
-    if(this.config?.publisher?.kinds?.includes(30166) ){
-      const publish30166 = new Publish.Kind30166()
-      await publish30166.one( result ).catch(this.log.error)  
-    }
+    const publish30166 = new Publish.Kind30166()
+    await publish30166.one( result ).catch(this.log.error)  
   }
 
   async on_fail(result){

apps/trawler/src/publish.js

@@ -3,7 +3,7 @@ import rcache from "./relaydb.js"
 import config from "./config.js"
 import { lastPublishedId } from "./utils.js"
 
-const p30066 = new Publish.Kind30066()
+const p30166 = new Publish.Kind30166()
 
 const filterRelayProperties = (relay) => {
   const relay_ = {}
@@ -37,14 +37,14 @@ const updatePublishTimes = async (relays=[]) => {
 export const publishOne = async (relay) => {
   relay = filterRelayProperties(relay)
   if(!relay) throw new Error('publishOne(): relay must be defined')
-  await p30066.one(relay)
+  await p30166.one(relay)
 }
 
 export const publishMany = async (relays = []) => {
   relays = filterRelaysProperties(relays)
   const filteredRelays = relays.filter(relayIsExpired);
   if (!filteredRelays.length) return;
-  await p30066.many(filteredRelays);
+  await p30166.many(filteredRelays);
   await updatePublishTimes(filteredRelays);
 }
 

apps/trawler/src/trawler.js

@@ -17,6 +17,8 @@ import { SyncQueue, TrawlQueue } from "@nostrwatch/controlflow"
 
 const logger = new Logger('trawler')
 
+const ignore = ["wss://nostr.searx.is/"]
+
 const { $Queue:$SyncQueue, $QueueEvents:$SyncEvents } = SyncQueue()
 const { $Queue:$TrawlQueue, $QueueEvents:$TrawlEvents } = TrawlQueue()
 
@@ -29,10 +31,10 @@ let promises,
 
 const addRelaysToCache = async (relayList) => {
   const ids = []
-  relayList.forEach(async (relayObj) => {
+  for (const relayObj of relayList) {
     ids.push(await rcache.relay.insertIfNotExists(relayObj))
-  })
-  return ids
+  }
+  return ids.filter(id => id !== undefined)
 }
 
 const noteInCache = async (ev, relay, lastEvent) => {
@@ -86,38 +88,54 @@ export const trawl = async function($job){
   listCount = 0
   $currentJob = $job
 
-  const relays = $job.data.relays
+  const relays = $job.data.relays.filter( relay => !ignore.includes(relay.url) )
   const pool = new SimplePool();
   const fetcher = NostrFetcher.withCustomPool(simplePoolAdapter(pool));
+  const kinds = [ 2, 10002 ]
 
   relays.forEach( async (relay) => {
 
     promises.push(new Promise( async (resolve) => {
-
       let lastEvent = 0
       let since = await determineSince(relay)
       $job.updateProgress({ type: 'resuming', source: relay, since })
       try {      
 
         const it = await fetcher.allEventsIterator(
           [ relay ],
-          { kinds: [ 2, 10002 ] },
+          { kinds },
           { since },
           { sort: true }
         )
         for await (const ev of it) {
+          if(!kinds.includes(ev.kind)) {
+            continue
+          }
           lastEvent = setLastEvent(ev, since, lastEvent)
           if( await noteInCache(ev, relay, lastEvent) ) {
-            console.log('noteInCache() true')
             continue 
           }
           const relayList = await relaysFromRelayList(ev)
 
-          const cacheIds = addRelaysToCache(relayList)
           if(relayList === false) continue
           listCount++
 
+          const cacheIds = await addRelaysToCache(relayList)
+
+          for( const id of cacheIds ) {
+            relaysPersisted.add(rcache.relay.get.one(id)?.url)
+          }
+
           deferPersist[ev.id] = async () => await rcache.note.set.one(ev)
+
+          if(cacheIds.length === 0) {
+            await deferPersist[ev.id]()
+            delete deferPersist[ev.id]
+            continue
+          }
+
+          console.log(`found ${cacheIds.length} new relays`)
+
           const roundtrip = { 
             requestedBy: process.env.DAEMON_PUBKEY,
             source: relay, 
@@ -126,6 +144,7 @@ export const trawl = async function($job){
           }
           if(config?.trawler?.sync?.out?.events)
             roundtrip.syncEventsCallback = syncEventsCallback
+
           const data = jobData(relayList, roundtrip)
           await sync.relays.out(data)
         }
@@ -139,8 +158,10 @@ export const trawl = async function($job){
     }))
   })
   await Promise.allSettled(Object.values(promises))
-  console.log(`relays persisted: ${[...relaysPersisted]}`)
-  return [...relaysPersisted]
+  if(relaysPersisted.size > 0 ){
+    console.log(`${relaysPersisted.size} relays persisted: ${[...relaysPersisted]}`)
+  }
+  return [...Array.from(relaysPersisted)]
 }
 
 const watchQueue = () => {
@@ -172,5 +193,4 @@ const finish = async (result, roundtrip) => {
   } 
   if(deferPersist?.[eventId])
     delete deferPersist[eventId]
-}
-
+}

internal/nwcache/mixins/cachetime.js

@@ -40,7 +40,7 @@ export default class CacheTimeMixin {
         return [...this.db.$.select(IDS).from( this.schema ).where( where )] || []
       },
       one: (key, select=null)  => {
-        select = this.parseSelect(select)
+        // select = this.parseSelect(select)
         if(!key.includes(this.id()))
           key = this.id(key)
         return this.db.$.get( key )?.v

internal/nwcache/mixins/relay.js

@@ -51,7 +51,7 @@ export default class RelayMixin {
 
   async patch(RelayFieldsObj) {
     this.validate(RelayFieldsObj)
-    const current = await this.db.$.get(relayId(RelayFieldsObj.url))
+    const current = this.db.$.get(relayId(RelayFieldsObj.url))
     if(!current)
       throw new Error(`Cannot patch because ${RelayFieldsObj.url} does not exist`)
     RelayFieldsObj.url = new URL(RelayFieldsObj.url).toString()

internal/nwcache/mixins/retry.js

@@ -1,8 +1,5 @@
-import { operators } from "lmdb-oql";
 import { Retry } from "../schemas.js"
 
-const { $eq, $gte } = operators
-
 export default class RetryMixin {
   constructor(db) {
     this.db = db;
@@ -29,13 +26,13 @@ export default class RetryMixin {
 
   async increment(key, amt=1){
     key = this.inferKey(key)
-    const current = await this.get(key)
+    const current = this.get(key)
     return this.set(key, current? current + amt: amt)
   }
 
   async decrement(key, amt=1){
     key = this.inferKey(key)
-    const current = await this.get(key)
+    const current = this.get(key)
     return this.set(key, current? current - amt: -amt)
   }
 }

internal/nwcache/mixins/service.js

@@ -21,7 +21,7 @@ class ServiceChecker {
   }
 
   async is(service, status) {
-    const result = await this.db.$.get(serviceId(service))
+    const result = this.db.$.get(serviceId(service))
     if(result?.online === status)
       return true
   }

internal/publisher/index.js

@@ -4,20 +4,17 @@ export { Publisher } from './src/Publisher.js'
 export { Kind0 } from './src/kinds/Kind0.js'
 export { Kind10002 } from './src/kinds/Kind10002.js'
 export { Kind10166 } from './src/kinds/Kind10166.js'
-export { Kind30066 } from './src/kinds/Kind30066.js'
 export { Kind30166 } from './src/kinds/Kind30166.js'
 
 //import for default export
 import { Kind0 } from './src/kinds/Kind0.js'
 import { Kind10002 } from './src/kinds/Kind10002.js'
 import { Kind10166 } from './src/kinds/Kind10166.js'
-import { Kind30066 } from './src/kinds/Kind30066.js'
 import { Kind30166 } from './src/kinds/Kind30166.js'
 
 export default {
   Kind0,
   Kind10002,
   Kind10166,
-  Kind30066,
   Kind30166
 }

internal/publisher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nostrwatch/publisher",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "type": "module",
   "description": "Library for publishing nostr.watch relay status and publisher registration events",
   "main": "index.js",

internal/publisher/src/Publisher.js

@@ -24,7 +24,7 @@ async function writeObjectToFile(obj) {
 
 export class Publisher { 
 
-  constructor(key="generic"){
+  constructor(key){
     this.logger = new Logger(`publisher[${key}]`)
   }
 
@@ -40,29 +40,9 @@ export class Publisher {
     }
   }
 
-  // generateEvent(data){
-  //   this.logger.warn('generateEvent(): has not been implemented by subclass, using generic functions')
-  //   const staticClass = eval(`kind${this.kind}`)
-  //   let tags = [], 
-  //       content = ""
-  //   if(staticClass?.generateTags)
-  //     tags = Kind30066.generateTags(data)
-
-  //   if(staticClass?.generateContent)
-  //     content = Kind30066.generateContent(data)
-
-  //   const event = {
-  //     ...this.tpl(),
-  //     content,
-  //     tags
-  //   }
-
-  //   return event
-  // }
-
-
-  generateEvent(){
-    return this.tpl(30066)
+  generateEvent(data){
+    data
+    return this.tpl(30166)
   }
 
   generateEvents(relays){
@@ -82,6 +62,7 @@ export class Publisher {
         throw new Error('generateEvent(): event does not validate')  
       return event
     } catch(e) {
+      console.log(event)
       this.logger.err(`signEvent(): Error: ${e}`)
       // console.log(event)
     }
@@ -116,8 +97,8 @@ export class Publisher {
 
 export class PublisherNocap extends Publisher {
 
-  constructor(){
-    super()
+  constructor(key="generic"){
+    super(key)
     this.logger = new Logger('publisher[nocap]')
   }
 

internal/publisher/src/kinds/Kind1066.js

@@ -34,7 +34,7 @@ export class Kind1066 extends Publisher {
 
     tags.push(['r', data.url])
 
-    const aTag = ['a', `30066:${data.pubkey}:${data.url}`]
+    const aTag = ['a', `30166:${data.pubkey}:${data.url}`]
     if(data?.relayHint)
       aTag.push(data.relayHint)
     tags.push(aTag)