scraper/news/news.go

package news

import (
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"

	"github.com/mmcdole/gofeed"

	"../requests"
	"../utils"
)

type Sport struct {
	Id 			int
	Name 		string
	CleanName	string
}

type Source struct {
	Id      	int
	Name    	string
	CleanName 	string
	Urls    	map[int]string

	Error 		*string
	Trace 		*string
}

type News struct {
	Id 		 	int
	Source  	*Source
	Sport	  	*Sport
	LeagueId 	int
	TeamId   	int

	Title       string
	CleanTitle  string
	PubDate     *time.Time
	Description string
	Link        string
	Image       string

	Teaser   	string
	Author   	string
	Content  	[]string
	Redirect 	string

	Haystack 	string
	Tags     	[]string
	CleanTags 	[]string

	Error 		string
	Trace 		string
}

func (n *News) Feed() error {
	doc, err := requests.GetDocumentFromURL(n.Link)
	if err != nil {
		return err
	}

	switch n.Source.Name {
	case "Eurosport":
		n.Teaser = strings.TrimSpace(doc.Find("h2").Text())
		doc.Find(".article-body .article-s4-rs p").Each(func(i int, s *goquery.Selection) {
			n.Content = append(n.Content, s.Text())
		})
		n.Author = strings.TrimSpace(doc.Find(".flex a.caption-s5-fx div.font-bold").Text())
		doc.Find(".related-topics .atom-tag").Each(func(i int, s *goquery.Selection) {
			tag := strings.TrimSpace(s.Text())
			cleanTag := utils.Sanitize(tag)
			if !utils.ArrayContains(n.CleanTags, cleanTag) {
				n.Tags = append(n.Tags, tag)
				n.CleanTags = append(n.CleanTags, cleanTag)
			}
		})
	case "L'équipe":
		n.Teaser = strings.TrimSpace(doc.Find("h2.Article__chapo").Text())
		doc.Find(".Paragraph__content").Each(func(i int, s *goquery.Selection) {
			n.Content = append(n.Content, s.Text())
		})
		n.Author = strings.TrimSpace(doc.Find(".Author__name").Text())
		doc.Find(".RelatedLinks a.RelatedLinks__link").Each(func(i int, s *goquery.Selection) {
			tag := strings.TrimSpace(s.Text())
			cleanTag := utils.Sanitize(tag)
			if !utils.ArrayContains(n.CleanTags, cleanTag) {
				n.Tags = append(n.Tags, tag)
				n.CleanTags = append(n.CleanTags, cleanTag)
			}
		})
	case "FFTT":
		n.Teaser = strings.TrimSpace(doc.Find(".news-description p").First().Text())
		doc.Find(".news-description p").Each(func(i int, s *goquery.Selection) {
			if i > 0 {
				n.Content = append(n.Content, s.Text())
			}
		})
		doc.Find(".social-shares-large-wrapper a.link").Each(func(i int, s *goquery.Selection) {
			tag := strings.TrimSpace(s.Text())
			cleanTag := utils.Sanitize(tag)
			if !utils.ArrayContains(n.CleanTags, cleanTag) {
				n.Tags = append(n.Tags, tag)
				n.CleanTags = append(n.CleanTags, cleanTag)
			}
		})
	case "Foot Mercato":
		n.Teaser = strings.TrimSpace(doc.Find("h2.article__lead").Text())
		doc.Find(".article__content p").Each(func(i int, s *goquery.Selection) {
			n.Content = append(n.Content, s.Text())
		})
		n.Author = strings.TrimSpace(doc.Find(".article__author a").Text())
	default:
		return fmt.Errorf("unknown source %s", n.Source.Name)
	}
	if len(n.Content) == 0 {
		n.Redirect = n.Link
	}
	if len(n.CleanTags) == 0 {
		n.Tags = append(n.Tags, n.Sport.Name)
		n.CleanTags = append(n.CleanTags, n.Sport.CleanName)
	}
	n.Haystack = fmt.Sprintf("%s-%s", n.CleanTitle, strings.Join(n.CleanTags, "-"))
	return nil
}

func (s *Source) ListNews(sport *Sport, url string) ([]*News, error) {
	var newsList []*News

	fp := gofeed.NewParser()
	feed, err := fp.ParseURL(url)
	if err != nil {
		return nil, err
	}

	for _, item := range feed.Items {
		n := &News{
			Source:      s,
			Sport:       sport,
			Title:       item.Title,
			Description: regexp.MustCompile(`<[^>]*>`).ReplaceAllLiteralString(item.Description, ""),
			CleanTitle:    utils.Sanitize(item.Title),
			PubDate:     item.PublishedParsed,
			Link:        item.Link,
		}
		for _, tags := range item.Categories {
			for _, tag := range strings.Split(tags, ",") {
				n.Tags = append(n.Tags, strings.TrimSpace(tag))
				n.CleanTags = append(n.CleanTags, utils.Sanitize(strings.TrimSpace(tag)))
			}
		}
		if item.Image != nil {
			n.Image = item.Image.URL
		} else if len(item.Enclosures) > 0 {
			n.Image = item.Enclosures[0].URL
		} else if s.Name == "Eurosport" {
			doc, err := goquery.NewDocumentFromReader(strings.NewReader(item.Description))
			if err == nil {
				if src, ok := doc.Find("img").Attr("src"); ok {
					n.Image = src
				}
			}
		}
		if item.Author != nil {
			n.Author = item.Author.Name
		}
		newsList = append(newsList, n)
	}
	return newsList, nil
}