from time import sleep from urllib.parse import urljoin from core.mysqldb import MysqlDB from lib.browser import Browser from lib.league import League paths = { # 'resultats': 'fs-results', 'calendrier': 'fs-fixtures' } db = None browser = None try: mysqldb = MysqlDB() browser = Browser(user_agent=db.get_random_ua(), headless=False) for league in League.get_leagues(mysqldb): if 'flashresultats' in league.urls: print('~~~ League {} - {} ~~~'.format(league.id, league.name)) matches = list(league.get_matches(db)) for path, dom_id in paths.items(): url = '{}/{}'.format(league.urls['flashresultats'].rstrip('/'), path) print('~~~ URL {} ~~~'.format(url)) browser.get(url) for _ in range(5): browser.execute_script('loadMoreGames();') sleep(5) mday = None for tr in browser.find_elements_by_xpath('//div[@id="{}"]//tr'.format(dom_id)): tds = tr.find_elements_by_tag_name('td') if len(tds) == 6: home = tds[2].find_element_by_tag_name('span').text.strip() away = tds[3].find_element_by_tag_name('span').text.strip() id_ = tr.get_attribute('id').split('_')[2] url = urljoin(league.urls['flashresultats'], '/match/{}'.format(id_)) for match in matches: if match.mday == mday: if match.home.names['flashresultats'] == home: if match.away.names['flashresultats'] == away: match.urls['flashresultats'] = url match.store_urls(db) print('[+] match {} - {}'.format(match.id, match.urls['flashresultats'])) break else: print('[-] match {} not found ({} - {})'.format(id_, home, away)) elif len(tds) == 1: mday = int(tds[0].text.split(' ')[1]) print('~~~ MDAY {} ~~~'.format(mday)) except BaseException as err: print('ERROR {}: {}'.format(type(err), err)) finally: if browser is not None: browser.quit() if db is not None: db.commit() db.close()