{ "id": "5830b39b3dccdd1b5384afad96f21ef1b2d1a4971edc5b3f2c5019b1d75f9cd8", "pubkey": "91bea5cd9361504c409aaf459516988f68a2fcd482762fd969a7cdc71df4451c", "created_at": 1789351819, "kind": 1050, "tags": [ [ "filename", "nostr_bot_crawler.go" ], [ "client", "nosbin" ] ], "content": "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\tjson \"github.com/bytedance/sonic\"\n\n\t_ \"github.com/jackc/pgx/v5\"\n\t\"github.com/jackc/pgx/v5/pgxpool\"\n\t\"github.com/nbd-wtf/go-nostr\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// --- 1. Modelos e DTOs ---\n\ntype NIP11Info struct {\n\tName string `json:\"name\"`\n\tDescription string `json:\"description\"`\n\tPubkey string `json:\"pubkey\"`\n\tContact string `json:\"contact\"`\n\tSupportedNIPs []int `json:\"supported_nips\"`\n\tSoftware string `json:\"software\"`\n\tVersion string `json:\"version\"`\n}\n\ntype RelayRecord struct {\n\tURL string `json:\"url\"`\n\tName string `json:\"name\"`\n\tContact string `json:\"contact\"`\n\tNIPs []int `json:\"nips\"`\n\tNIP11 json.NoCopyRawMessage `json:\"nip11\"`\n\tLastAccessed time.Time `json:\"last_accessed\"`\n\tIsActive bool `json:\"is_active\"`\n\tResponseTime int64 `json:\"response_time_ms\"`\n}\n\n// --- 2. Interfaces (SOLID - DIP) ---\n\ntype QueueService interface {\n\tPush(ctx context.Context, url string) error\n\tPop(ctx context.Context) (string, error)\n\tIsVisited(ctx context.Context, url string) (bool, error)\n}\n\ntype LockService interface {\n\tAcquire(ctx context.Context, key string, ttl time.Duration) (bool, error)\n\tRelease(ctx context.Context, key string) error\n}\n\ntype Store interface {\n\tSave(ctx context.Context, record RelayRecord) error\n\tMigrate(ctx context.Context) error\n}\n\n// Nova interface para lidar com os servidores Blossom\ntype BlossomStore interface {\n\tAddServer(ctx context.Context, url string) error\n\tGetAllServers(ctx context.Context) ([]string, error)\n}\n\n// --- 3. Implementação Postgres (pgx) ---\n\ntype PostgresStore struct {\n\tpool *pgxpool.Pool\n}\n\nfunc NewPostgresStore(connStr string) (*PostgresStore, error) {\n\tconfig, err := pgxpool.ParseConfig(connStr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tpool, err := pgxpool.NewWithConfig(context.Background(), config)\n\treturn \u0026PostgresStore{pool: pool}, err\n}\n\nfunc (p *PostgresStore) Migrate(ctx context.Context) error {\n\tquery := `\n\tCREATE TABLE IF NOT EXISTS relays (\n\t\turl TEXT PRIMARY KEY,\n\t\tname TEXT,\n\t\tcontact TEXT,\n\t\tnip11 JSONB,\n\t\tnips INT[],\n\t\tlast_accessed TIMESTAMP WITH TIME ZONE,\n\t\tis_active BOOLEAN,\n\t\tresponse_time_ms BIGINT\n\t);\n\tCREATE INDEX IF NOT EXISTS idx_relays_nips ON relays USING GIN (nips);\n\tCREATE INDEX IF NOT EXISTS idx_relays_name ON relays (name);\n\tCREATE INDEX IF NOT EXISTS idx_relays_contact ON relays (contact);\n\tCREATE INDEX IF NOT EXISTS idx_relays_last_accessed ON relays (last_accessed);\n\tCREATE INDEX IF NOT EXISTS idx_relays_active ON relays (is_active);\n\t`\n\t_, err := p.pool.Exec(ctx, query)\n\treturn err\n}\n\nfunc (p *PostgresStore) Save(ctx context.Context, r RelayRecord) error {\n\tif len(r.NIP11) == 0 || string(r.NIP11) == \"null\" {\n\t\tr.NIP11 = json.NoCopyRawMessage(\"{}\")\n\t}\n\n\tquery := `\n\t\tINSERT INTO relays (url, name, contact, nip11, nips, last_accessed, is_active, response_time_ms)\n\t\tVALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n\t\tON CONFLICT (url) DO UPDATE SET\n\t\t\tname = EXCLUDED.name,\n\t\t\tcontact = EXCLUDED.contact,\n\t\t\tnip11 = EXCLUDED.nip11,\n\t\t\tnips = EXCLUDED.nips,\n\t\t\tlast_accessed = EXCLUDED.last_accessed,\n\t\t\tis_active = EXCLUDED.is_active,\n\t\t\tresponse_time_ms = EXCLUDED.response_time_ms;\n\t`\n\t_, err := p.pool.Exec(ctx, query,\n\t\tr.URL, r.Name, r.Contact, r.NIP11, r.NIPs, r.LastAccessed, r.IsActive, r.ResponseTime,\n\t)\n\treturn err\n}\n\n// --- 4. Implementação Redis (Queue, Lock \u0026 BlossomStore) ---\n\ntype RedisService struct {\n\tclient *redis.Client\n}\n\nfunc (r *RedisService) Push(ctx context.Context, url string) error {\n\treturn r.client.LPush(ctx, \"crawler:queue\", url).Err()\n}\n\nfunc (r *RedisService) Pop(ctx context.Context) (string, error) {\n\tres, err := r.client.BRPop(ctx, 0, \"crawler:queue\").Result()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn res[1], nil\n}\n\nfunc (r *RedisService) IsVisited(ctx context.Context, url string) (bool, error) {\n\tadded, err := r.client.SAdd(ctx, \"crawler:visited\", url).Result()\n\treturn added == 0, err\n}\n\nfunc (r *RedisService) Acquire(ctx context.Context, key string, ttl time.Duration) (bool, error) {\n\treturn r.client.SetNX(ctx, \"lock:\"+key, \"1\", ttl).Result()\n}\n\nfunc (r *RedisService) Release(ctx context.Context, key string) error {\n\treturn r.client.Del(ctx, \"lock:\"+key).Err()\n}\n\n// Métodos do BlossomStore\nfunc (r *RedisService) AddServer(ctx context.Context, url string) error {\n\treturn r.client.SAdd(ctx, \"blossom:servers\", url).Err()\n}\n\nfunc (r *RedisService) GetAllServers(ctx context.Context) ([]string, error) {\n\treturn r.client.SMembers(ctx, \"blossom:servers\").Result()\n}\n\n// --- 5. Crawler Core ---\n\ntype Crawler struct {\n\tqueue QueueService\n\tlock LockService\n\tstore Store\n\tblossomStore BlossomStore // Novo storage para servidores Blossom\n\thttpClient *http.Client\n\tmaxWorkers int\n}\n\nfunc (c *Crawler) fetchNIP11(ctx context.Context, url string) (json.NoCopyRawMessage, *NIP11Info) {\n\thttpURL := strings.Replace(strings.Replace(url, \"wss://\", \"https://\", 1), \"ws://\", \"http://\", 1)\n\n\treq, err := http.NewRequestWithContext(ctx, http.MethodGet, httpURL, nil)\n\tif err != nil {\n\t\treturn nil, nil\n\t}\n\treq.Header.Add(\"Accept\", \"application/nostr+json\")\n\n\tresp, err := c.httpClient.Do(req)\n\tif err != nil {\n\t\treturn nil, nil\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn nil, nil\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil || len(body) == 0 {\n\t\treturn nil, nil\n\t}\n\n\tif !json.Valid(body) {\n\t\treturn nil, nil\n\t}\n\n\tvar info NIP11Info\n\tif err := json.Unmarshal(body, \u0026info); err != nil {\n\t\treturn body, nil\n\t}\n\n\treturn body, \u0026info\n}\n\nfunc (c *Crawler) processRelay(ctx context.Context, url string) {\n\tlocked, _ := c.lock.Acquire(ctx, url, 1*time.Minute)\n\tif !locked {\n\t\treturn\n\t}\n\tdefer c.lock.Release(ctx, url)\n\n\tfmt.Printf(\"🔍 Processando: %s\\n\", url)\n\trecord := RelayRecord{\n\t\tURL: url,\n\t\tLastAccessed: time.Now(),\n\t\tIsActive: false,\n\t}\n\n\tstart := time.Now()\n\trelay, err := nostr.RelayConnect(ctx, url)\n\tif err == nil {\n\t\trecord.IsActive = true\n\t\trecord.ResponseTime = time.Since(start).Milliseconds()\n\n\t\t// Refatorado: Filtro para Relay List (10002) E Blossom Servers (10063)\n\t\tsub, _ := relay.Subscribe(ctx, nostr.Filters{{Kinds: []int{10002, 10063}, Limit: 50}})\n\t\tif sub != nil {\n\t\t\tgo func() {\n\t\t\t\tfor evt := range sub.Events {\n\t\t\t\t\tswitch evt.Kind {\n\t\t\t\t\tcase 10002: // Descoberta de novos relays Nostr\n\t\t\t\t\t\tfor _, tag := range evt.Tags {\n\t\t\t\t\t\t\tif tag[0] == \"r\" \u0026\u0026 len(tag) \u003e 1 {\n\t\t\t\t\t\t\t\tc.Enqueue(ctx, tag[1])\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\tcase 10063: // Servidores Blossom\n\t\t\t\t\t\tfor _, tag := range evt.Tags {\n\t\t\t\t\t\t\t// Servidores Blossom normalmente usam a tag \"server\"\n\t\t\t\t\t\t\tif tag[0] == \"server\" \u0026\u0026 len(tag) \u003e 1 {\n\t\t\t\t\t\t\t\tfmt.Printf(\"🌸 Blossom Server encontrado: %s\\n\", tag[1])\n\t\t\t\t\t\t\t\tc.blossomStore.AddServer(ctx, tag[1])\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\t\t\ttime.Sleep(3 * time.Second)\n\t\t}\n\t\trelay.Close()\n\t}\n\n\traw, info := c.fetchNIP11(ctx, url)\n\trecord.NIP11 = raw\n\tif info != nil {\n\t\trecord.Name = info.Name\n\t\trecord.Contact = info.Contact\n\t\trecord.NIPs = info.SupportedNIPs\n\t}\n\n\tif err := c.store.Save(ctx, record); err != nil {\n\t\tfmt.Printf(\"❌ Erro DB [%s]: %v\\n\", url, err)\n\t}\n}\n\nfunc (c *Crawler) Enqueue(ctx context.Context, url string) {\n\tif !strings.HasPrefix(url, \"ws\") {\n\t\treturn\n\t}\n\tvisited, _ := c.queue.IsVisited(ctx, url)\n\tif !visited {\n\t\tc.queue.Push(ctx, url)\n\t}\n}\n\nfunc (c *Crawler) Start(ctx context.Context) {\n\tvar wg sync.WaitGroup\n\tfor i := 0; i \u003c c.maxWorkers; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase \u003c-ctx.Done():\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t\turl, err := c.queue.Pop(ctx)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tc.processRelay(ctx, url)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n}\n\n// --- 6. Main e Setup ---\n\nfunc main() {\n\t// Refatorado: Context com cancelamento para suportar Graceful Shutdown\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\tredisAddr := \"localhost:6379\"\n\tpgConnStr := \"postgres://postgres:Strong@P4ssword@localhost:5432/nostr_crawler\"\n\n\trdb := redis.NewClient(\u0026redis.Options{Addr: redisAddr})\n\tredisSvc := \u0026RedisService{client: rdb}\n\n\tpgStore, err := NewPostgresStore(pgConnStr)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\t\n\t// Descomente caso deseje rodar a migração\n\t// if err := pgStore.Migrate(ctx); err != nil {\n\t// \tpanic(err)\n\t// }\n\n\tcrawler := \u0026Crawler{\n\t\tqueue: redisSvc,\n\t\tlock: redisSvc,\n\t\tstore: pgStore,\n\t\tblossomStore: redisSvc, // Injetando o Redis como armazenamento Blossom\n\t\tmaxWorkers: 20,\n\t\thttpClient: \u0026http.Client{Timeout: 10 * time.Second},\n\t}\n\n\tseeds := []string{\n\t\t\"wss://relay.damus.io\",\n\t\t\"wss://nos.lol\",\n\t\t\"wss://relay.snort.social\",\n\t}\n\tfor _, s := range seeds {\n\t\tcrawler.Enqueue(ctx, s)\n\t}\n\n\t// Configuração do Graceful Shutdown (Captura Ctrl+C)\n\tsigs := make(chan os.Signal, 1)\n\tsignal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)\n\n\tgo func() {\n\t\t\u003c-sigs // Bloqueia até receber um sinal de interrupção\n\t\tfmt.Println(\"\\n🛑 Encerrando crawler. Iniciando exportação dos servidores Blossom...\")\n\t\t\n\t\tservers, err := redisSvc.GetAllServers(context.Background())\n\t\tif err == nil \u0026\u0026 len(servers) \u003e 0 {\n\t\t\tfile, err := os.Create(\"blossom_servers_export.txt\")\n\t\t\tif err == nil {\n\t\t\t\tdefer file.Close()\n\t\t\t\tfor _, server := range servers {\n\t\t\t\t\tfile.WriteString(server + \"\\n\")\n\t\t\t\t}\n\t\t\t\tfmt.Printf(\"✅ %d servidores Blossom exportados com sucesso para 'blossom_servers_export.txt'!\\n\", len(servers))\n\t\t\t} else {\n\t\t\t\tfmt.Printf(\"❌ Erro ao criar arquivo de exportação: %v\\n\", err)\n\t\t\t}\n\t\t} else {\n\t\t\tfmt.Println(\"⚠️ Nenhum servidor Blossom foi encontrado durante a execução.\")\n\t\t}\n\t\t\n\t\tcancel() // Cancela o context, fazendo os workers pararem\n\t}()\n\n\tfmt.Println(\"🚀 Crawler Multi-Instância Iniciado. Pressione Ctrl+C para encerrar e exportar a lista de Blossom servers.\")\n\tcrawler.Start(ctx)\n}", "sig": "8adf57aadb4edbb9d86eb77f73830258475a08b59346efb276f07960e209d1d0c260fd641a39b475d7b3e42537574c8858b12548e8602f5279a432c574f6c031" }