Versions in this module Expand all Collapse all v0 v0.0.11 Jul 31, 2025 Changes in this version + type Callback func(ctx context.Context, result *Result) + type Crawler struct + func New(opts Options) (*Crawler, error) + func (c *Crawler) AddFetcherRules(rules ...*FetcherRule) error + func (c *Crawler) AddParserRules(rule ...*ParserRule) error + func (c *Crawler) Crawl(ctx context.Context, urls []string, callback Callback) error + func (c *Crawler) GetStats() *CrawlerStats + func (c *Crawler) Stop() + type CrawlerStats struct + func (s *CrawlerStats) GetFailed() int64 + func (s *CrawlerStats) GetProcessed() int64 + func (s *CrawlerStats) GetSucceeded() int64 + func (s *CrawlerStats) IncrementFailed() + func (s *CrawlerStats) IncrementProcessed() + func (s *CrawlerStats) IncrementSucceeded() + type FetcherRule struct + Fetcher fetch.Fetcher + func NewFetcherRule(pattern string, fetcher fetch.Fetcher, opts ...FetcherRuleOption) *FetcherRule + type FetcherRuleOption func(*FetcherRule) + func WithFetcherMatchType(matchType MatchType) FetcherRuleOption + func WithFetcherPriority(priority int) FetcherRuleOption + type FollowBehavior string + const FollowAny + const FollowNone + const FollowRelatedSubdomains + const FollowSameDomain + type MatchRule struct + Pattern string + Priority int + Type MatchType + func (r *MatchRule) Compile() error + func (r *MatchRule) Matches(value string) bool + type MatchType string + const MatchExact + const MatchGlob + const MatchPrefix + const MatchRegex + const MatchSuffix + type MockParser struct + func NewMockParser() *MockParser + func (m *MockParser) Parse(ctx context.Context, page *fetch.Response) (any, error) + func (m *MockParser) SetParseFunc(fn func(ctx context.Context, page *fetch.Response) (any, error)) + type Options struct + Cache cache.Cache + DefaultFetcher fetch.Fetcher + DefaultParser Parser + FetcherRules []*FetcherRule + FollowBehavior FollowBehavior + KnownURLs []string + Logger *slog.Logger + MaxURLs int + ParserRules []*ParserRule + QueueSize int + RequestDelay time.Duration + ShowProgress bool + ShowProgressInterval time.Duration + Workers int + type Parser interface + Parse func(ctx context.Context, page *fetch.Response) (any, error) + type ParserRule struct + Parser Parser + func NewParserRule(pattern string, parser Parser, opts ...ParserRuleOption) *ParserRule + type ParserRuleOption func(*ParserRule) + func WithParserMatchType(matchType MatchType) ParserRuleOption + func WithParserPriority(priority int) ParserRuleOption + type Result struct + Error error + Links []string + Parsed any + Response *fetch.Response + URL *url.URL
Go to main page