Refactor search agent for concurrency and simplify flow

Removed redundant fields and callbacks in the search agent while introducing concurrent processing for reading search results. Updated logic to enhance readability and modularity, including capped reads and streamlined interaction with search results. Adjusted dependencies and related usage to align with the refactored design.

cmd/agent/cmd.go

@@ -13,8 +13,6 @@ import (
 
 	"gitea.stevedudenhoeffer.com/steve/answer/pkg/agents/shared"
 
-	"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/duckduckgo"
-
 	"gitea.stevedudenhoeffer.com/steve/answer/pkg/agents/searcher"
 
 	"github.com/joho/godotenv"
@@ -106,24 +104,12 @@ func main() {
 
 			search := searcher.Agent{
 				Model: m,
-				OnGoingToNextPage: func(ctx context.Context) error {
-					slog.Info("going to next page")
-					return nil
-				},
-				OnReadingSearchResult: func(ctx context.Context, sr duckduckgo.Result) (any, error) {
-					slog.Info("reading search result", "url", sr.URL, "title", sr.Title, "description", sr.Description)
-					return nil, nil
-				},
-				OnFinishedReadingSearchResult: func(ctx context.Context, sr duckduckgo.Result, newKnowledge []string, err error, onReadingResult any) error {
-					slog.Info("finished reading search result", "err", err, "newKnowledge", newKnowledge)
-					return nil
-				},
+
 				OnDone: func(ctx context.Context, knowledge shared.Knowledge) error {
 					slog.Info("done", "knowledge", knowledge)
 					return nil
 				},
-				MaxReads:       20,
-				MaxNextResults: 10,
+				MaxReads: 20,
 			}
 
 			processor := knowledge2.KnowledgeProcessor{Model: m}
@@ -135,7 +121,7 @@ func main() {
 
 			slog.Info("knowledge", "knowledge", knowledge)
 
-			sum, err := processor.ProcessKnowledge(ctx, knowledge)
+			sum, err := processor.Process(ctx, knowledge)
 
 			fmt.Println(sum)
 			return nil