fixed sub-domain filter

kashifkhan0771 · kashifkhan0771 · commit aade78b9cc18 · 2026-04-09T12:31:23.000+05:00
diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
@@ -162,7 +162,7 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	// request validations
 	collector.OnRequest(func(r *colly.Request) {
 		host := r.URL.Hostname()
-		if host != parsedURL.Hostname() && !strings.HasSuffix(host, parsedURL.Hostname()) {
+		if host != parsedURL.Hostname() && !strings.HasSuffix(host, "."+parsedURL.Hostname()) {
 			ctx.Logger().V(5).Info("blocked by domain filter", "url", r.URL.String())
 			r.Abort()
 		}

Original file line number	Diff line number	Diff line change
`@@ -162,7 +162,7 @@ func (s Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan `
`162`	`162`	`// request validations`
`163`	`163`	`collector.OnRequest(func(r *colly.Request) {`
`164`	`164`	`host := r.URL.Hostname()`
`165`		`- if host != parsedURL.Hostname() && !strings.HasSuffix(host, parsedURL.Hostname()) {`
	`165`	`+ if host != parsedURL.Hostname() && !strings.HasSuffix(host, "."+parsedURL.Hostname()) {`
`166`	`166`	`ctx.Logger().V(5).Info("blocked by domain filter", "url", r.URL.String())`
`167`	`167`	`r.Abort()`
`168`	`168`	`}`