scrapemateapp

type Config struct {
	// Concurrency is the number of concurrent scrapers to run.
	// If not set, it defaults to 1.
	Concurrency int `validate:"required,gte=1"`

	// Cache is the cache to use for storing scraped data.
	// If left empty then no caching will be used.
	// Otherwise the CacheType must be one of file or leveldb.
	CacheType string `validate:"omitempty,oneof=file leveldb"`
	// CachePath is the path to the cache file or directory.
	// It is required to be a valid path if CacheType is set.
	CachePath string `validate:"required_with=CacheType"`

	// UseJS is whether to use JavaScript to render the page.
	UseJS bool `validate:"omitempty"`
	// UseStealth is whether to use stealth mode to scrape the page.
	// uses a special http client to scrape the page.
	UseStealth bool `validate:"omitempty"`
	// StealthBrowser is the browser to use for stealth mode.
	StealthBrowser string `validate:"omitempty"`
	// JSOpts are the options for the JavaScript renderer.
	JSOpts jsOptions

	// ProviderType is the type of provider to use.
	// It is required to be a valid type if Provider is set.
	// If not set the memory provider will be used.
	Provider scrapemate.JobProvider

	// Writers are the writers to use for writing the results.
	// At least one writer must be provided.
	Writers []scrapemate.ResultWriter `validate:"required,gt=0"`
	// InitJob is the job to initialize the app with.
	InitJob scrapemate.IJob
	// ExitOnInactivityDuration is whether to exit the app when there are no more jobs to run.
	ExitOnInactivityDuration time.Duration
	// Proxies are the proxies to use for the app.
	Proxies []string
	// BrowserReuseLimit is the limit of browser reuse.
	// Only applicable when using JavaScript renderer.
	// By default it is 0, which means the browser will be reused indefinitely.
	BrowserReuseLimit int
	// PageReuseLimit is the limit of page reuse.
	// Only applicable when using JavaScript renderer.
	// By default it is 0, which means the page will not be reused.
	PageReuseLimit int
}

type RetryConfig struct {
	MaxAttempts     int
	InitialInterval time.Duration
	MaxInterval     time.Duration
	Multiplier      float64
}

type ScrapemateApp struct {
	// contains filtered or unexported fields
}