dev

package module
v1.2.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 13, 2022 License: MIT Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// UserAgent will be used in requests.
	UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
)

Variables

View Source 
var (
	// ErrNotFound is returned for 404s.
	ErrNotFound = fmt.Errorf("server responded with a 404")

	// ErrNotOK is returned for non 200s.
	ErrNotOK = fmt.Errorf("server responded with a non-200")
)
View Source 
var (
	// Timeout is the default HTTP timeout.
	Timeout = time.Second * 15
)

Functions

func Fetch 

func Fetch(url string) ([]byte, error)

Fetch performs a simple GET request to grab html data.

func FetchAndParse 

func FetchAndParse(url string) (*html.Node, error)

FetchAndParse takes care of the page fetch and root parsing.

func GetAllTags 

func GetAllTags(cb func(t []Tag)) error

GetAllTags will fetch all tags returned from the Dev API. An artifical cap of 10,000 pages has been set.

func GetUserHTML 

func GetUserHTML(iurl string) ([]byte, error)

GetUserHTML will fetch a page of tags from the Dev API.

func GetUsersFromID 

func GetUsersFromID(id int, cb func(u *User)) error

GetUsersFromID will iterate through all users starting from the ID. An artifical cap has been set to 1m. Dev has roughly 800K users at the moment.

func QuerySelector 

func QuerySelector(root *html.Node, selector string) (*html.Node, error)

QuerySelector will parse a selector and perform a query on the DOM.

func QuerySelectorAll 

func QuerySelectorAll(root *html.Node, selector string) ([]*html.Node, error)

QuerySelectorAll will parse a selector and perform a query on the DOM.

Types

type Article 

type Article struct {
	ID                     int      `json:"id"`
	Type                   string   `json:"type_of"`
	Title                  string   `json:"title"`
	Description            string   `json:"description"`
	ReadablePublishDate    string   `json:"readable_publish_date"`
	Slug                   string   `json:"slug"`
	Path                   string   `json:"path"`
	URL                    string   `json:"url"`
	CommentsCount          int      `json:"comments_count"`
	PublicReactionsCount   int      `json:"public_reactions_count"`
	CollectionID           int      `json:"collection_id"`
	PublishedTimestamp     string   `json:"published_timestamp"`
	PositiveReactionsCount int      `json:"positive_reactions_count"`
	CoverImage             string   `json:"cover_image"`
	SocialImage            string   `json:"social_image"`
	CanonicalURL           string   `json:"canonical_url"`
	CreatedAt              string   `json:"created_at"`
	EditedAt               string   `json:"edited_at"`
	CrosspostedAt          string   `json:"crossposted_at"`
	PublishedAt            string   `json:"published_at"`
	LastCommentAt          string   `json:"last_comment_at"`
	TagList                []string `json:"tag_list"`
	Tags                   string   `json:"tags"`
	User                   User     `json:"user"`
}

Article represents a single user article (post).

func GetAllArticles 

func GetAllArticles(username string) ([]Article, error)

GetAllArticles will fetch a all pages of articles for a user from the Dev API. We set an artifical cap at 5 pages (5,000 articles).

func GetArticles 

func GetArticles(username string, page int, limit int) ([]Article, error)

GetArticles will fetch a page of articles for a user from the Dev API.

func ParseArticle 

func ParseArticle(b []byte) (*Article, error)

ParseArticle will parse the JSON of a single article.

func ParseArticles 

func ParseArticles(b []byte) ([]Article, error)

ParseArticles will parse the JSON of a multiple articles.

func (*Article) CreatedTime 

func (a *Article) CreatedTime() (time.Time, error)

CreatedTime will return a time object.

type Articles 

type Articles []Article

Articles is a collection of Article.

func (Articles) JSON 

func (a Articles) JSON() ([]byte, error)

JSON converts the structs to JSON.

type Comment 

type Comment struct {
	Title     string `json:"title"`
	URL       string `json:"url"`
	Comment   string `json:"comment"`
	CreatedAt string `json:"created_at"`
}

Comment is a user comment.

func (*Comment) CreatedTime 

func (c *Comment) CreatedTime() (time.Time, error)

CreatedTime will parse a comment's time.

type Snapshot 

type Snapshot struct {
	User     User     `json:"user"`
	Articles Articles `json:"articles"`
	HTMLPage []byte   `json:"html_page"`
	UserPage UserPage `json:"user_details"`
}

Snapshot is an aggregate of all user data. This includes API data as well as data collected over via scraping.

func GetSnapshot 

func GetSnapshot(id string) (ss *Snapshot, err error)

GetSnapshot will collect all info for a user in one shot.

type Tag 

type Tag struct {
	ID           int    `json:"id"`
	Name         string `json:"name"`
	BGColorHex   string `json:"bg_color_hex"`
	TextColorHex string `json:"text_color_hex"`
}

Tag is an article tag.

func GetTags 

func GetTags(page int) ([]Tag, error)

GetTags will fetch a page of tags from the Dev API.

func ParseTag 

func ParseTag(b []byte) (*Tag, error)

ParseTag will parse the JSON of a single tag.

func ParseTags 

func ParseTags(b []byte) ([]Tag, error)

ParseTags will parse the JSON of a multiple tags.

type User 

type User struct {
	ID              int    `json:"id"`
	Type            string `json:"type_of"`
	Name            string `json:"name"`
	Username        string `json:"username"`
	Summary         string `json:"summary"`
	TwitterUsername string `json:"twitter_username"`
	GithubUsername  string `json:"github_username"`
	WebsiteURL      string `json:"website_url"`
	Location        string `json:"location"`
	JoinedAt        string `json:"joined_at"`
	ProfileImage    string `json:"profile_image"`
}

User represents a dev.to user record.

func GetUserByID 

func GetUserByID(id string) (*User, error)

GetUserByID will fetch a single user from Dev's API.

func GetUserByUsername 

func GetUserByUsername(username string) (*User, error)

GetUserByUsername will fetch a single user from Dev's API.

func ParseUser 

func ParseUser(b []byte) (*User, error)

ParseUser will parse the JSON of a single user.

func (*User) JSON 

func (u *User) JSON() ([]byte, error)

JSON converts the struct to JSON.

func (*User) JoinedAtTime 

func (u *User) JoinedAtTime() (time.Time, error)

JoinedAtTime will return a time struct for the JoinedAt time.

func (*User) URL 

func (u *User) URL() string

URL returns the fully qualified URL to the user's dev.to profile.

type UserPage 

type UserPage struct {
	Education    string    `json:"education"`
	Title        string    `json:"title"`
	Company      string    `json:"company"`
	CompanyURL   string    `json:"company_url"`
	Email        string    `json:"email"`
	Links        []string  `json:"links"`
	Badges       []string  `json:"badges"`
	Comments     []Comment `json:"comments"`
	Skills       []string  `json:"skills"`
	Interests    []string  `json:"interests"`
	AvailableFor string    `json:"available_for"`

	// TODO: Not Yet Implemented
	PostCount    int `json:"post_count"`
	CommentCount int `json:"comment_count"`
	TagsFollowed int `json:"tags_followed"`
}

UserPage is a user's web page collected via scraping.

func ParseUserPage 

func ParseUserPage(b []byte) (*UserPage, error)

ParseUserPage will parse the HTML of a user's wwww page.

func (UserPage) JSON 

func (u UserPage) JSON() ([]byte, error)

JSON converts the UserPage to JSON.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.