package postgres import ( "crypto/sha256" "database/sql" "fmt" "log" "math" "os" "strings" "github.com/lib/pq" "1bet.fr/scraper/match" "1bet.fr/scraper/news" "1bet.fr/scraper/utils" ) type Postgres struct { host string port int username string password string database string psqlInfo string psqlConn *sql.DB isConnected bool } func sValue(s *string) interface{} { if s == nil { return nil } return *s } func iValue(i *int) interface{} { if i == nil { return nil } return *i } func aValue(a *[]string) interface{} { if a == nil { return nil } return pq.Array(*a) } func foreignId(i int) interface{} { if i == 0 { return nil } return i } func concatWS(a []interface{}, s string) string { var b []string for _, x := range a { if x == nil { b = append(b, "") } else { b = append(b, fmt.Sprint(x)) } } return strings.Join(b, s) } var pg *Postgres func init() { var err error pg = &Postgres{ host: os.Getenv("POSTGRES_HOST"), port: utils.AtoI(os.Getenv("POSTGRES_PORT")), username: os.Getenv("POSTGRES_USERNAME"), password: os.Getenv("POSTGRES_PASSWORD"), database: os.Getenv("POSTGRES_DATABASE"), } pg.psqlInfo = fmt.Sprintf( "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", pg.host, pg.port, pg.username, pg.password, pg.database, ) pg.psqlConn, err = sql.Open("postgres", pg.psqlInfo) if err != nil { log.Fatalf("error while opening pg connection : %s", err) } if err = pg.psqlConn.Ping(); err != nil { log.Fatalf("error while pinging pg server : %s", err) } pg.isConnected = true } func Close() { if !pg.isConnected { return } if err := pg.psqlConn.Close(); err != nil { log.Fatalf("error while closing pg connection : %s", err) } pg.isConnected = false } func ListSources() ([]*news.Source, error) { var sources []*news.Source rows, err := pg.psqlConn.Query(` SELECT mainapp_source.id, sport_id, mainapp_source.name, mainapp_source.clean_name, feed_url, mainapp_sport.name, mainapp_sport.clean_name FROM mainapp_source, mainapp_sport WHERE mainapp_sport.id = mainapp_source.sport_id `) if err != nil { return nil, fmt.Errorf("error while querying postgres : %s", err) } for rows.Next() { source := &news.Source{Sport: &news.Sport{}} if err = rows.Scan(&source.Id, &source.Sport.Id, &source.Name, &source.CleanName, &source.FeedUrl, &source.Sport.Name, &source.Sport.CleanName); err != nil { return nil, fmt.Errorf("error while scanning row from postgres : %s", err) } sources = append(sources, source) } return sources, nil } func ListLeagues() ([]*match.League, error) { var leagues []*match.League rows, err := pg.psqlConn.Query(` SELECT mainapp_league.id, sport_id, country_id, mainapp_league.name, mainapp_league.clean_name, gender, schedule_url, ranking_url, channel_url, mdays, matches_by_mday, rounds, groups, mainapp_sport.name, mainapp_sport.clean_name FROM mainapp_league, mainapp_sport WHERE mainapp_sport.id = mainapp_league.sport_id `) if err != nil { return nil, fmt.Errorf("error while querying postgres : %s", err) } for rows.Next() { league := &match.League{Sport: &match.Sport{}, Country: &match.Country{}} if err = rows.Scan( &league.Id, &league.Sport.Id, &league.Country.Id, &league.Name, &league.CleanName, &league.Gender, &league.ScheduleUrl, &league.RankingUrl, &league.ChannelUrl, &league.MatchDays, &league.MatchesByMatchDay, pq.Array(&league.Rounds), pq.Array(&league.Groups), &league.Sport.Name, &league.Sport.CleanName, ); err != nil { return nil, fmt.Errorf("error while scanning row from postgres : %s", err) } leagues = append(leagues, league) } return leagues, nil } func UpdateLeague(l *match.League) (int64, error) { res, err := pg.psqlConn.Exec(` UPDATE mainapp_league SET error = $1, trace = $2 WHERE id = $3 `, sValue(l.Error), sValue(l.Trace), l.Id, ) if err != nil { return 0, err } return res.RowsAffected() } func InsertNews(n *news.News) error { return pg.psqlConn.QueryRow(` INSERT INTO mainapp_news (title, clean_title, link, pub_date, description, image, teaser, author, content, redirect, haystack, tags, clean_tags, error, trace, league_id, source_id, team_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) RETURNING id `, n.Title, n.CleanTitle, n.Link, n.PubDate, sValue(n.Description), sValue(n.Image), sValue(n.Teaser), sValue(n.Author), aValue(n.Content), sValue(n.Redirect), sValue(n.Haystack), aValue(n.Tags), aValue(n.CleanTags), sValue(n.Error), sValue(n.Trace), iValue(n.LeagueId), n.Source.Id, iValue(n.TeamId), ).Scan(&n.Id) } func UpdateNews(n *news.News) (int64, error) { res, err := pg.psqlConn.Exec(` UPDATE mainapp_news SET title = $1, clean_title = $2, pub_date = $3, link = $4, description = $5, image = $6, teaser = $7, author = $8, content = $9, redirect = $10, haystack = $11, tags = $12, clean_tags = $13, error = $14, trace = $15, league_id = get_matching_league($11, $18), source_id = $16, team_id = $17 WHERE id = $19 `, n.Title, n.CleanTitle, n.PubDate, n.Link,sValue(n.Description), sValue(n.Image), sValue(n.Teaser), sValue(n.Author), aValue(n.Content), sValue(n.Redirect), sValue(n.Haystack), aValue(n.Tags), aValue(n.CleanTags), sValue(n.Error), sValue(n.Trace), n.Source.Id, iValue(n.TeamId), n.Source.Sport.Id, n.Id, ) if err != nil { return 0, err } return res.RowsAffected() } func DeleteNews(n *news.News) (int64, error) { res, err := pg.psqlConn.Exec(` DELETE FROM mainapp_news WHERE id = $1 `, n.Id) if err != nil { return 0, err } return res.RowsAffected() } func InsertTeamBySourceName(t *match.Team, l *match.League) error { var host, name string if t.Names == nil { return fmt.Errorf("no source name given") } shortName := "" for host, name = range *t.Names { runeName := []rune(strings.ToUpper(name)) shortName = string(runeName[:int(math.Min(3, float64(len(runeName))))]) break } err := pg.psqlConn.QueryRow("SELECT id, name FROM mainapp_team WHERE names->>$1 = $2", host, name).Scan(&t.Id, &t.Name) if err != nil { cleanName := utils.Sanitize(name) jsonHost := fmt.Sprintf("{\"%s\"}", utils.Sanitize(host)) return pg.psqlConn.QueryRow(` INSERT INTO mainapp_team (sport_id, country_id, name, clean_name, short_name, long_name, gender, names, tags, clean_tags, news_count) VALUES ($1, $2, $3, $4, $5, $6, $7, jsonb_set('{}', $8, to_jsonb($9::text), true), $10, $11, 0) ON CONFLICT ON CONSTRAINT custom_unique_team DO UPDATE SET names = jsonb_set(mainapp_team.names, $12, to_jsonb($13::text), true) RETURNING id, name `, l.Sport.Id, l.Country.Id, name, cleanName, shortName, name, iValue(l.Gender), jsonHost, name, pq.Array([]string{name}), pq.Array([]string{cleanName}), jsonHost, name).Scan(&t.Id, &t.Name) } return nil } func DeleteTeam(t *match.Team) (int64, error) { res, err := pg.psqlConn.Exec(` DELETE FROM mainapp_team WHERE id = $1 `, t.Id) if err != nil { return 0, err } return res.RowsAffected() } func InsertMatch(m *match.Match) error { var arr []interface{} arr = append(arr, m.League.Id) arr = append(arr, foreignId(m.TeamHome.Id)) arr = append(arr, foreignId(m.TeamAway.Id)) arr = append(arr, foreignId(m.PlayerHome.Id)) arr = append(arr, foreignId(m.PlayerAway.Id)) arr = append(arr, sValue(m.Round)) hash := sha256.New() hash.Write([]byte(concatWS(arr, "/"))) sign := fmt.Sprintf("%x", hash.Sum(nil)) return pg.psqlConn.QueryRow(` INSERT INTO mainapp_match (league_id, team_home_id, team_away_id, player_home_id, player_away_id, mday, round, leg, sign, mday_id, base_url, start_date, error, trace) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT ON CONSTRAINT mainapp_match_sign_key DO UPDATE SET base_url = $11, start_date = $12, error = $13, trace = $14 RETURNING id `, m.League.Id, foreignId(m.TeamHome.Id), foreignId(m.TeamAway.Id), foreignId(m.PlayerHome.Id), foreignId(m.PlayerAway.Id), iValue(m.MatchDay), sValue(m.Round), iValue(m.Leg), sign, iValue(m.MatchDayId), sValue(m.BaseUrl), m.StartDate, sValue(m.Error), sValue(m.Trace)).Scan(&m.Id) } func DeleteMatch(m *match.Match) (int64, error) { res, err := pg.psqlConn.Exec(` DELETE FROM mainapp_match WHERE id = $1 `, m.Id) if err != nil { return 0, err } return res.RowsAffected() }