scraper/postgres/postgres.go

package postgres

import (
	"crypto/sha256"
	"database/sql"
	"encoding/json"
	"fmt"
	"log"
	"math"
	"os"
	"strings"

	"github.com/lib/pq"

	"1bet.fr/scraper/match"
	"1bet.fr/scraper/news"
	"1bet.fr/scraper/utils"
)

type Postgres struct {
	host     	string
	port     	int
	username    string
	password 	string
	database   	string

	psqlInfo    string
	psqlConn    *sql.DB
	isConnected bool
}

func sValue(s *string) interface{} {
	if s == nil {
		return nil
	}
	return *s
}

func iValue(i *int) interface{} {
	if i == nil {
		return nil
	}
	return *i
}

func aValue(a *[]string) interface{} {
	if a == nil {
		return nil
	}
	return pq.Array(*a)
}

func foreignId(i int) interface{} {
	if i == 0 {
		return nil
	}
	return i
}

func concatWS(a []interface{}, s string) string {
	var b []string
	for _, x := range a {
		if x == nil {
			b = append(b, "")
		} else {
			b = append(b, fmt.Sprint(x))
		}
	}
	return strings.Join(b, s)
}

var pg *Postgres

func init() {
	var err error

	pg = &Postgres{
		host:        os.Getenv("POSTGRES_HOST"),
		port:        utils.AtoI(os.Getenv("POSTGRES_PORT")),
		username:    os.Getenv("POSTGRES_USERNAME"),
		password:    os.Getenv("POSTGRES_PASSWORD"),
		database:    os.Getenv("POSTGRES_DATABASE"),
	}

	pg.psqlInfo = fmt.Sprintf(
		"host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
		pg.host, pg.port, pg.username, pg.password, pg.database,
	)

	pg.psqlConn, err = sql.Open("postgres", pg.psqlInfo)
	if err != nil {
		log.Fatalf("error while opening pg connection : %s", err)
	}

	if err = pg.psqlConn.Ping(); err != nil {
		log.Fatalf("error while pinging pg server : %s", err)
	}
	pg.isConnected = true
}

func Close() {
	if !pg.isConnected {
		return
	}
	if err := pg.psqlConn.Close(); err != nil {
		log.Fatalf("error while closing pg connection : %s", err)
	}
	pg.isConnected = false
}

func ListSources() ([]*news.Source, error) {
	var sources []*news.Source

	rows, err := pg.psqlConn.Query(`
		SELECT 
			mainapp_source.id, sport_id, mainapp_source.name, mainapp_source.clean_name, feed_url,
			mainapp_sport.name, mainapp_sport.clean_name 
		FROM 
			mainapp_source, mainapp_sport
		WHERE
			mainapp_sport.id = mainapp_source.sport_id
	`)
	if err != nil {
		return nil, fmt.Errorf("error while querying postgres : %s", err)
	}

	for rows.Next() {
		source := &news.Source{Sport: &news.Sport{}}
		if err = rows.Scan(&source.Id, &source.Sport.Id, &source.Name, &source.CleanName, &source.FeedUrl,
			&source.Sport.Name, &source.Sport.CleanName); err != nil {
			return nil, fmt.Errorf("error while scanning row from postgres : %s", err)
		}
		sources = append(sources, source)
	}
	return sources, nil
}

func ListLeagues() ([]*match.League, error) {
	var leagues []*match.League

	rows, err := pg.psqlConn.Query(`
		SELECT 
			mainapp_league.id, sport_id, country_id, mainapp_league.name, mainapp_league.clean_name, gender, 
			schedule_url, ranking_url, channel_url, 
			mdays, matches_by_mday, rounds, groups,
			mainapp_sport.name, mainapp_sport.clean_name
		FROM 
			mainapp_league, mainapp_sport
		WHERE
			mainapp_sport.id = mainapp_league.sport_id
	`)
	if err != nil {
		return nil, fmt.Errorf("error while querying postgres : %s", err)
	}

	for rows.Next() {
		league := &match.League{Sport: &match.Sport{}, Country: &match.Country{}}
		if err = rows.Scan(
			&league.Id, &league.Sport.Id, &league.Country.Id, &league.Name, &league.CleanName, &league.Gender,
			&league.ScheduleUrl, &league.RankingUrl, &league.ChannelUrl,
			&league.MatchDays, &league.MatchesByMatchDay, pq.Array(&league.Rounds), pq.Array(&league.Groups),
			&league.Sport.Name, &league.Sport.CleanName,
		); err != nil {
			return nil, fmt.Errorf("error while scanning row from postgres : %s", err)
		}
		leagues = append(leagues, league)
	}
	return leagues, nil
}

func UpdateLeague(l *match.League) (int64, error) {
	res, err := pg.psqlConn.Exec(`
			UPDATE mainapp_league
			SET error = $1, trace = $2
			WHERE id = $3
		`, sValue(l.Error), sValue(l.Trace), l.Id,
	)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}

func InsertNews(n *news.News) error {
	return pg.psqlConn.QueryRow(`
			INSERT INTO mainapp_news 
				(title, clean_title, link, pub_date, description, image, teaser, author, 
				content, redirect, haystack, tags, clean_tags, error, trace, 
				league_id, source_id, team_id)
			VALUES 
				($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)
			RETURNING 
				id
		`, n.Title, n.CleanTitle, n.Link, n.PubDate, sValue(n.Description),
		sValue(n.Image), sValue(n.Teaser), sValue(n.Author),
		aValue(n.Content), sValue(n.Redirect), sValue(n.Haystack),
		aValue(n.Tags), aValue(n.CleanTags), sValue(n.Error), sValue(n.Trace),
		iValue(n.LeagueId), n.Source.Id, iValue(n.TeamId),
	).Scan(&n.Id)
}

func UpdateNews(n *news.News) (int64, error) {
	res, err := pg.psqlConn.Exec(`
			UPDATE mainapp_news
			SET title = $1, clean_title = $2, pub_date = $3, link = $4, description = $5,
				image = $6, teaser = $7, author = $8, content = $9, redirect = $10,
				haystack = $11, tags = $12, clean_tags = $13, error = $14, trace = $15,
				league_id = get_matching_league($11, $18), source_id = $16, team_id = $17
			WHERE id = $19
		`, n.Title, n.CleanTitle, n.PubDate, n.Link,sValue(n.Description),
		sValue(n.Image), sValue(n.Teaser), sValue(n.Author),
		aValue(n.Content), sValue(n.Redirect), sValue(n.Haystack),
		aValue(n.Tags), aValue(n.CleanTags), sValue(n.Error), sValue(n.Trace),
		n.Source.Id, iValue(n.TeamId), n.Source.Sport.Id, n.Id,
	)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}

func DeleteNews(n *news.News) (int64, error) {
	res, err := pg.psqlConn.Exec(`
		DELETE FROM mainapp_news
		WHERE id = $1
	`, n.Id)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}

func InsertTeamBySourceName(t *match.Team) error {
	var host, name string

	if t.Names == nil {
		return fmt.Errorf("no source name given")
	}
	shortName := ""
	for host, name = range *t.Names {
		runeName := []rune(strings.ToUpper(name))
		shortName = string(runeName[:int(math.Min(3, float64(len(runeName))))])
		break
	}

	// First try to get existing team if any from database
	err := pg.psqlConn.QueryRow(`
		SELECT id, name 
		FROM mainapp_team 
		WHERE names->>$1 = $2 AND sport_id = $3 AND gender = $4
		ORDER BY id ASC 
		LIMIT 1
	`, utils.Sanitize(host), name, t.Sport.Id, t.Gender).Scan(&t.Id, &t.Name)

	// Else create team in database
	if err != nil {
		cleanName := utils.Sanitize(name)
		jsonHost := fmt.Sprintf("{\"%s\"}", utils.Sanitize(host))
		jsonImages, err := json.Marshal(t.Images)
		if err != nil {
			return err
		}
		return pg.psqlConn.QueryRow(`
			INSERT INTO mainapp_team 
				(sport_id, country_id, name, clean_name, short_name, long_name, gender, images,
				names, tags, clean_tags, news_count)
			VALUES 
				($1, $2, $3, $4, $5, $6, $7, $8,
				jsonb_set('{}', $9, to_jsonb($10::text), true), $11, $12, 0)
			ON CONFLICT ON CONSTRAINT custom_unique_team DO UPDATE SET
				names = jsonb_set(mainapp_team.names, $13, to_jsonb($14::text), true)
			RETURNING id, name
		`, t.Sport.Id, t.Country.Id, name, cleanName, shortName, name, iValue(t.Gender), jsonImages,
		jsonHost, name, pq.Array([]string{name}), pq.Array([]string{cleanName}),
		jsonHost, name).Scan(&t.Id, &t.Name)
	}
	return nil
}

func DeleteTeam(t *match.Team) (int64, error) {
	res, err := pg.psqlConn.Exec(`
		DELETE FROM mainapp_team
		WHERE id = $1
	`, t.Id)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}

func InsertMatch(m *match.Match) error {
	var arr []interface{}
	arr = append(arr, m.League.Id)
	arr = append(arr, foreignId(m.TeamHome.Id))
	arr = append(arr, foreignId(m.TeamAway.Id))
	arr = append(arr, foreignId(m.PlayerHome.Id))
	arr = append(arr, foreignId(m.PlayerAway.Id))
	arr = append(arr, sValue(m.Round))
	hash := sha256.New()
	hash.Write([]byte(concatWS(arr, "|")))
	sign := fmt.Sprintf("%x", hash.Sum(nil))

	return pg.psqlConn.QueryRow(`
		INSERT INTO mainapp_match
			(league_id, team_home_id, team_away_id, player_home_id, 
			player_away_id, mday, round, leg, sign, mday_id, 
			status, base_url, start_date, error, trace)
		VALUES 
			($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
		ON CONFLICT ON CONSTRAINT mainapp_match_sign_key DO UPDATE SET
			base_url = $12, start_date = $13, error = $14, trace = $15 
		RETURNING id
	`, m.League.Id, foreignId(m.TeamHome.Id), foreignId(m.TeamAway.Id), foreignId(m.PlayerHome.Id),
	foreignId(m.PlayerAway.Id), iValue(m.MatchDay), sValue(m.Round), iValue(m.Leg), sign, iValue(m.MatchDayId),
	match.StatusComing, sValue(m.BaseUrl), m.StartDate, sValue(m.Error), sValue(m.Trace)).Scan(&m.Id)
}

func DeleteMatch(m *match.Match) (int64, error) {
	res, err := pg.psqlConn.Exec(`
		DELETE FROM mainapp_match
		WHERE id = $1
	`, m.Id)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}