crawl

type Crawler struct {
	Semaphore    chan bool
	URLTopoCh    chan *URLTopological
	Logger       *log.Logger
	Server       *headless.Server
	DownloadHTML bool
	// contains filtered or unexported fields
}

type URLTopological struct {
	URL   *url.URL
	Depth int
}