137 lines
2.8 KiB
Go
137 lines
2.8 KiB
Go
package main
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"sync"
|
|
|
|
"github.com/lib/pq"
|
|
|
|
"1bet.fr/scraper/match"
|
|
"1bet.fr/scraper/news"
|
|
"1bet.fr/scraper/postgres"
|
|
)
|
|
|
|
const (
|
|
nbProcesses = 50
|
|
maxNewsPerSource = 50
|
|
)
|
|
|
|
func updateNews() {
|
|
defer postgres.Close()
|
|
|
|
sources, err := postgres.ListSources()
|
|
if err != nil {
|
|
log.Fatalf("error while getting list of sources : %s", err)
|
|
}
|
|
|
|
waitGroup := sync.WaitGroup{}
|
|
newsChannel := make(chan *news.News)
|
|
for i := 0; i < nbProcesses; i++ {
|
|
waitGroup.Add(1)
|
|
go func(nc chan *news.News, wg *sync.WaitGroup) {
|
|
defer wg.Done()
|
|
for n := range nc {
|
|
if err := n.Feed(); err != nil {
|
|
log.Fatalf("error while feeding news : %s", err)
|
|
}
|
|
if _, err := postgres.UpdateNews(n); err != nil {
|
|
log.Fatalf("error while update news in postgres : %s", err)
|
|
}
|
|
}
|
|
}(newsChannel, &waitGroup)
|
|
}
|
|
|
|
for _, source := range sources {
|
|
log.Printf("[+] Starting parse of source : %s", source.FeedUrl)
|
|
newsList, err := source.ListNews()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
for i, n := range newsList {
|
|
if i >= maxNewsPerSource {
|
|
log.Printf("Stopping parse of source with %d news added", i)
|
|
break
|
|
}
|
|
if err := postgres.InsertNews(n); err != nil {
|
|
if err, ok := err.(*pq.Error); ok {
|
|
if err.Code.Name() == "unique_violation" {
|
|
log.Printf("Stopping parse of source (unique violation) with %d news added", i)
|
|
break
|
|
} else {
|
|
log.Fatalf("error while inserting news in postgres : %s", err.Message)
|
|
}
|
|
} else {
|
|
log.Fatalf("error while getting pq.Error object")
|
|
}
|
|
}
|
|
log.Printf("Adding news %s", n.Link)
|
|
newsChannel <- n
|
|
}
|
|
}
|
|
close(newsChannel)
|
|
waitGroup.Wait()
|
|
}
|
|
|
|
func updateSchedule() {
|
|
defer postgres.Close()
|
|
|
|
leagues, err := postgres.ListLeagues()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
waitGroup := sync.WaitGroup{}
|
|
sourceChannel := make(chan *match.Source)
|
|
for i := 0; i < nbProcesses; i++ {
|
|
waitGroup.Add(1)
|
|
go func(sc chan *match.Source, wg *sync.WaitGroup) {
|
|
defer wg.Done()
|
|
for s := range sc {
|
|
matches, err := s.GetMatches()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
for _, m := range matches {
|
|
fmt.Println(m)
|
|
//if err = postgres.InsertMatch(m); err != nil {
|
|
// log.Fatal(err)
|
|
//}
|
|
}
|
|
continue
|
|
}
|
|
}(sourceChannel, &waitGroup)
|
|
}
|
|
|
|
for _, league := range leagues {
|
|
sources, err := league.ListSources()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
for _, s := range sources {
|
|
log.Printf("Adding source %s", s.Url.String())
|
|
sourceChannel <- s
|
|
}
|
|
}
|
|
}
|
|
|
|
func main() {
|
|
flag.Parse()
|
|
args := flag.Args()
|
|
if len(args) != 1 {
|
|
log.Fatalf("unexpected number of args : len(%s) != 1", args)
|
|
}
|
|
|
|
switch args[0] {
|
|
case "news":
|
|
updateNews()
|
|
case "schedule":
|
|
updateSchedule()
|
|
default:
|
|
log.Fatalf("unexpected arg : %s", args[0])
|
|
}
|
|
}
|