Documentation
¶
Index ¶
- type CloseSpiderRequest
- type CloseSpiderResponse
- type DistributeReqponse
- type DistributeRequest
- type HttpServer
- type HttpServerAnts
- type HttpServerCluster
- type HttpServerCrawler
- type LeftMeInRequest
- type LeftMeInResponse
- type MulticastClient
- type MulticastServer
- type ReportRequest
- type ReportResponse
- type RpcBase
- type RpcClient
- type RpcClientAnts
- type RpcClientCluster
- type RpcClientCrawl
- type RpcServer
- type RpcServerAnts
- type RpcServerCluster
- type RpcServerCrawl
- type StopRequest
- type StopResponse
- type Watcher
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CloseSpiderRequest ¶
type CloseSpiderResponse ¶
type DistributeReqponse ¶
type DistributeReqponse struct {
RpcBase
}
type DistributeRequest ¶
type HttpServer ¶
type HttpServer interface {
ServeHTTP(w http.ResponseWriter, r *http.Request)
Welcome(w http.ResponseWriter, r *http.Request)
}
http server
type HttpServerAnts ¶
type HttpServerAnts interface {
HttpServer
HttpServerCluster
HttpServerCrawler
}
ants http server
type HttpServerCluster ¶
type HttpServerCluster interface {
Cluster(w http.ResponseWriter, r *http.Request)
}
http server for cluster
type HttpServerCrawler ¶
type HttpServerCrawler interface {
Spiders(w http.ResponseWriter, r *http.Request)
Crawl(w http.ResponseWriter, r *http.Request)
CrawlCluster(w http.ResponseWriter, r *http.Request)
CrawlNode(w http.ResponseWriter, r *http.Request)
}
http server for crawler
type LeftMeInRequest ¶
type LeftMeInRequest struct {
RpcBase
}
type LeftMeInResponse ¶
type LeftMeInResponse struct {
RpcBase
}
if result is true, it is master ;if false,close the client and connect to nodeinfo
type MulticastClient ¶
type MulticastClient interface {
// contains filtered or unexported methods
}
type MulticastServer ¶
type MulticastServer interface {
// contains filtered or unexported methods
}
type ReportRequest ¶
type ReportRequest struct {
RpcBase
ScrapeResult *crawler.ScrapeResult
}
report to master the result of crawl request
type ReportResponse ¶
type ReportResponse struct {
RpcBase
}
type RpcClient ¶
type RpcClient interface {
Dial(ip string, port int) (*rpc.Client, error)
Detect() // if client connection is down
Start() // start dead loop for detect
}
* RpcClient * *
type RpcClientAnts ¶
type RpcClientAnts interface {
RpcClient
RpcClientCrawl
RpcClientCluster
}
* RpcClientAnts *
type RpcClientCluster ¶
type RpcClientCluster interface {
LetMeIn(ip string, port int) error
Connect(ip string, port int) error
StartSpider(nodeName, spiderName string) error
}
* RpcClientCluster *
type RpcClientCrawl ¶
type RpcClientCrawl interface {
StopNode(nodeName string) error
Distribute(nodeName string, request *http.Request) error
ReportResult(nodeName string, result *crawler.ScrapeResult) error
CloseSpider(nodeName, spiderName string) error
}
* RpcClientCrawl *
type RpcServerAnts ¶
type RpcServerAnts interface {
RpcServer
RpcServerCrawl
RpcServerCluster
}
* RpcServerAnts *
type RpcServerCluster ¶
type RpcServerCluster interface {
LetMeIn(request *LeftMeInRequest, response *LeftMeInResponse) error
Connect(request *RpcBase, response *RpcBase) error
StopNode(request *StopRequest, response *StopResponse) error
StartSpider(request *DistributeRequest, response *DistributeReqponse) error
}
* RpcServerCluster *
type RpcServerCrawl ¶
type RpcServerCrawl interface {
AcceptRequest(request *DistributeRequest, response *DistributeReqponse) error
AcceptResult(request *ReportRequest, response *ReportResponse) error
CloseSpider(request *CloseSpiderRequest, response *CloseSpiderResponse) error
}
* RpcServerCrawl *
type StopResponse ¶
type StopResponse struct {
RpcBase
}
Click to show internal directories.
Click to hide internal directories.