bluesnake

type AlreadyVisitedError struct {
	// Destination is the URL that was attempted to be visited.
	// It might not match the URL passed to Visit if redirect
	// was followed.
	Destination *url.URL
}

type Collector struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// Custom headers for the request
	Headers *http.Header
	// MaxDepth limits the recursion depth of visited URLs.
	// Set it to 0 for infinite recursion (default).
	MaxDepth int
	// AllowedDomains is a domain whitelist.
	// Leave it blank to allow any domains to be visited
	AllowedDomains []string
	// DisallowedDomains is a domain blacklist.
	DisallowedDomains []string
	// DisallowedURLFilters is a list of regular expressions which restricts
	// visiting URLs. If any of the rules matches to a URL the
	// request will be stopped. DisallowedURLFilters will
	// be evaluated before URLFilters
	// Leave it blank to allow any URLs to be visited
	DisallowedURLFilters []*regexp.Regexp

	// Leave it blank to allow any URLs to be visited
	URLFilters []*regexp.Regexp

	// AllowURLRevisit allows multiple downloads of the same URL
	AllowURLRevisit bool
	// MaxBodySize is the limit of the retrieved response body in bytes.
	// 0 means unlimited.
	// The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
	MaxBodySize int
	// CacheDir specifies a location where GET requests are cached as files.
	// When it's not defined, caching is disabled.
	CacheDir string
	// IgnoreRobotsTxt allows the Collector to ignore any restrictions set by
	// the target host's robots.txt file.  See http://www.robotstxt.org/ for more
	// information.
	IgnoreRobotsTxt bool
	// Async turns on asynchronous network communication. Use Collector.Wait() to
	// be sure all requests have been finished.
	Async bool
	// ParseHTTPErrorResponse allows parsing HTTP responses with non 2xx status codes.
	// By default, BlueSnake parses only successful HTTP responses. Set ParseHTTPErrorResponse
	// to true to enable it.
	ParseHTTPErrorResponse bool
	// ID is the unique identifier of a collector
	ID uint32
	// DetectCharset can enable character encoding detection for non-utf8 response bodies
	// without explicit charset declaration. This feature uses https://github.com/saintfish/chardet
	DetectCharset bool

	// CheckHead performs a HEAD request before every GET to pre-validate the response
	CheckHead bool
	// TraceHTTP enables capturing and reporting request performance for crawler tuning.
	// When set to true, the Response.Trace will be filled in with an HTTPTrace object.
	TraceHTTP bool
	// Context is the context that will be used for HTTP requests. You can set this
	// to support clean cancellation of scraping.
	Context context.Context
	// MaxRequests limit the number of requests done by the instance.
	// Set it to 0 for infinite requests (default).
	MaxRequests uint32
	// EnableRendering enables JavaScript rendering using headless Chrome.
	// When set to true, pages will be rendered with chromedp before parsing.
	EnableRendering bool
	// EnableContentHash enables content-based duplicate detection
	EnableContentHash bool
	// ContentHashAlgorithm specifies the hash algorithm to use ("xxhash", "md5", "sha256")
	ContentHashAlgorithm string
	// ContentHashConfig contains detailed configuration for content hashing
	ContentHashConfig *ContentHashConfig

	// CacheExpiration sets the maximum age for cache files.
	// If a cached file is older than this duration, it will be ignored and refreshed.
	CacheExpiration time.Duration
	// contains filtered or unexported fields
}

type CollectorConfig struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// Headers contains custom headers for HTTP requests
	Headers map[string]string
	// MaxDepth limits the recursion depth of visited URLs.
	// Set it to 0 for infinite recursion (default).
	MaxDepth int
	// AllowedDomains is a domain whitelist.
	// Leave it blank to allow any domains to be visited
	AllowedDomains []string
	// DisallowedDomains is a domain blacklist.
	DisallowedDomains []string
	// DisallowedURLFilters is a list of regular expressions which restricts
	// visiting URLs. If any of the rules matches to a URL the
	// request will be stopped. DisallowedURLFilters will
	// be evaluated before URLFilters
	DisallowedURLFilters []*regexp.Regexp
	// URLFilters is a list of regular expressions which restricts
	// visiting URLs. If any of the rules matches to a URL the
	// request won't be stopped. DisallowedURLFilters will
	// be evaluated before URLFilters
	URLFilters []*regexp.Regexp
	// AllowURLRevisit allows multiple downloads of the same URL
	AllowURLRevisit bool
	// MaxBodySize is the limit of the retrieved response body in bytes.
	// 0 means unlimited.
	// The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
	MaxBodySize int
	// CacheDir specifies a location where GET requests are cached as files.
	// When it's not defined, caching is disabled.
	CacheDir string
	// IgnoreRobotsTxt allows the Collector to ignore any restrictions set by
	// the target host's robots.txt file.
	IgnoreRobotsTxt bool
	// Async turns on asynchronous network communication.
	Async bool
	// ParseHTTPErrorResponse allows parsing HTTP responses with non 2xx status codes.
	ParseHTTPErrorResponse bool
	// ID is the unique identifier of a collector (auto-assigned if 0)
	ID uint32
	// DetectCharset can enable character encoding detection for non-utf8 response bodies
	DetectCharset bool
	// CheckHead performs a HEAD request before every GET to pre-validate the response
	CheckHead bool
	// TraceHTTP enables capturing and reporting request performance.
	TraceHTTP bool
	// Context is the context that will be used for HTTP requests.
	Context context.Context
	// MaxRequests limit the number of requests done by the instance.
	// Set it to 0 for infinite requests (default).
	MaxRequests uint32
	// EnableRendering enables JavaScript rendering using headless Chrome.
	EnableRendering bool
	// CacheExpiration sets the maximum age for cache files.
	CacheExpiration time.Duration
	// Debugger is the debugger instance to use
	Debugger debug.Debugger
	// DiscoveryMechanisms specifies which mechanisms to use for URL discovery.
	// Can be any combination: ["spider"], ["sitemap"], or ["spider", "sitemap"].
	// Default is ["spider"].
	DiscoveryMechanisms []DiscoveryMechanism
	// SitemapURLs specifies custom sitemap URLs to fetch (optional).
	// If nil/empty when sitemap discovery is enabled, tries default locations
	// (/sitemap.xml, /sitemap_index.xml).
	SitemapURLs []string
	// EnableContentHash enables content-based duplicate detection
	// When true, pages with identical content will be detected even if URLs differ
	EnableContentHash bool
	// ContentHashAlgorithm specifies the hash algorithm to use
	// Options: "xxhash" (fastest, default), "md5", "sha256"
	ContentHashAlgorithm string
	// ContentHashConfig contains detailed configuration for content hashing
	ContentHashConfig *ContentHashConfig
}

type ContentHashConfig struct {
	// ExcludeTags specifies HTML tags to exclude from content hashing
	// Default: ["script", "style", "nav", "footer"]
	ExcludeTags []string
	// IncludeOnlyTags specifies to only include specific tags in content hashing
	// If empty, all content (minus ExcludeTags) is included
	// Example: ["article", "main"] to focus only on main content
	IncludeOnlyTags []string
	// StripTimestamps removes timestamp patterns from content before hashing
	StripTimestamps bool
	// StripAnalytics removes analytics and tracking code from content
	StripAnalytics bool
	// StripComments removes HTML comments from content
	StripComments bool
	// CollapseWhitespace normalizes whitespace (multiple spaces/newlines to single)
	CollapseWhitespace bool
}

type Context struct {
	// contains filtered or unexported fields
}

type Crawler struct {
	// Collector is the underlying low-level collector (exported for advanced configuration)
	Collector *Collector
	// contains filtered or unexported fields
}

type DiscoveryMechanism string

type ErrorCallback func(*Response, error)

type HTMLCallback func(*HTMLElement)

type HTMLElement struct {
	// Name is the name of the tag
	Name string
	Text string

	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
	// Index stores the position of the current element within all the elements matched by an OnHTML callback
	Index int
	// contains filtered or unexported fields
}

type HTTPTrace struct {
	ConnectDuration   time.Duration
	FirstByteDuration time.Duration
	// contains filtered or unexported fields
}

type Index struct {
	XMLName xml.Name `xml:"sitemapindex"`
	Sitemap []parts  `xml:"sitemap"`
}

type LimitRule struct {
	// DomainRegexp is a regular expression to match against domains
	DomainRegexp string
	// DomainGlob is a glob pattern to match against domains
	DomainGlob string
	// Delay is the duration to wait before creating a new request to the matching domains
	Delay time.Duration
	// RandomDelay is the extra randomized duration to wait added to Delay before creating a new request
	RandomDelay time.Duration
	// Parallelism is the number of the maximum allowed concurrent requests of the matching domains
	Parallelism int
	// contains filtered or unexported fields
}

type Link struct {
	// URL is the target URL
	URL string `json:"url"`
	// Type is the link type: "anchor", "image", "script", "stylesheet", "iframe", "canonical", "video", "audio"
	Type string `json:"type"`
	// Text is the anchor text, alt text, or empty for other link types
	Text string `json:"text"`
	// Context is the surrounding text context where the link appears
	Context string `json:"context,omitempty"`
	// IsInternal indicates if this link points to the same domain/subdomain
	IsInternal bool `json:"isInternal"`
	// Status is the HTTP status code if this URL has been crawled (200, 404, 301, etc.)
	Status *int `json:"status,omitempty"`
	// Title is the page title if this URL has been crawled
	Title string `json:"title,omitempty"`
	// ContentType is the MIME type if this URL has been crawled
	ContentType string `json:"contentType,omitempty"`
	// Position indicates the semantic location of the link on the page
	// Values: "content", "navigation", "header", "footer", "sidebar", "breadcrumbs", "pagination", "unknown"
	Position string `json:"position,omitempty"`
	// DOMPath is a simplified DOM path showing the link's location in the HTML structure
	// Example: "body > main > article > p > a"
	DOMPath string `json:"domPath,omitempty"`
}

type Links struct {
	// Internal links point to same domain/subdomain
	Internal []Link `json:"internal"`
	// External links point to different domains
	External []Link `json:"external"`
}

type MockResponse struct {
	// StatusCode is the HTTP status code to return (default: 200)
	StatusCode int
	// Body is the response body content (used if BodyFunc is nil)
	Body string
	// BodyFunc is a function that generates the body dynamically based on the request
	// If set, this takes precedence over Body
	BodyFunc func(*http.Request) string
	// Headers are the HTTP headers to include in the response
	Headers http.Header
	// Delay simulates network latency before returning the response
	Delay time.Duration
	// Error simulates a network error
	Error error
}

type MockTransport struct {
	// contains filtered or unexported fields
}

type OnCrawlCompleteFunc func(wasStopped bool, totalPages int, totalDiscovered int)

type OnPageCrawledFunc func(*PageResult)

type PageMetadata struct {
	Status      int
	Title       string
	ContentType string
}

type PageResult struct {
	// URL is the URL that was crawled
	URL string
	// Status is the HTTP status code (e.g., 200, 404, 500)
	Status int
	// Title is the page title extracted from the <title> tag (for HTML pages)
	Title string
	// MetaDescription is the content of the <meta name="description"> tag
	MetaDescription string
	// Indexable indicates if search engines can index this page
	// Values: "Yes", "No", or "-" for non-HTML resources
	Indexable string
	// ContentType is the Content-Type header value (e.g., "text/html", "application/json")
	ContentType string
	// Error contains any error message if the crawl failed, empty otherwise
	Error string
	// Links contains all outbound links from this page (internal and external)
	Links *Links
	// ContentHash is the hash of the normalized page content (empty if content hashing is disabled)
	ContentHash string
	// IsDuplicateContent indicates if this content hash has been seen before on a different URL
	IsDuplicateContent bool
	// contains filtered or unexported fields
}

type ProxyFunc func(*http.Request) (*url.URL, error)

type Request struct {
	// URL is the parsed URL of the HTTP request
	URL *url.URL
	// Headers contains the Request's HTTP headers
	Headers *http.Header
	// the Host header
	Host string
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Depth is the number of the parents of the request
	Depth int
	// Method is the HTTP method of the request
	Method string
	// Body is the request body which is used on POST/PUT requests
	Body io.Reader
	// ResponseCharacterencoding is the character encoding of the response body.
	// Leave it blank to allow automatic character encoding of the response body.
	// It is empty by default and it can be set in OnRequest callback.
	ResponseCharacterEncoding string
	// ID is the Unique identifier of the request
	ID uint32

	// ProxyURL is the proxy address that handles the request
	ProxyURL string
	// contains filtered or unexported fields
}

type RequestCallback func(*Request)

type Response struct {
	// StatusCode is the status code of the Response
	StatusCode int
	// Body is the content of the Response
	Body []byte
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Request is the Request object of the response
	Request *Request
	// Headers contains the Response's HTTP headers
	Headers *http.Header
	// Trace contains the HTTPTrace for the request. Will only be set by the
	// collector if Collector.TraceHTTP is set to true.
	Trace *HTTPTrace
}

type ResponseCallback func(*Response)

type ResponseHeadersCallback func(*Response)

type ScrapedCallback func(*Response)

type Sitemap struct {
	XMLName xml.Name `xml:"urlset"`
	URL     []URL    `xml:"url"`
}

type URL struct {
	Loc        string  `xml:"loc"`
	LastMod    string  `xml:"lastmod"`
	ChangeFreq string  `xml:"changefreq"`
	Priority   float32 `xml:"priority"`
}

type XMLCallback func(*XMLElement)

type XMLElement struct {
	// Name is the name of the tag
	Name string
	Text string

	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the DOM object of the page. DOM is relative
	// to the current XMLElement and is either a html.Node or xmlquery.Node
	// based on how the XMLElement was created.
	DOM interface{}

	// Index stores the position of the current element within all the elements matched by an OnXML callback
	Index int
	// contains filtered or unexported fields
}