78 lines
2.8 KiB
Python
78 lines
2.8 KiB
Python
|
from traceback import format_exc
|
||
|
|
|
||
|
|
from providers.controller import ProviderController
|
||
|
|
from lib.news import NewsSource
|
||
|
|
from core.webutils import WebUtils
|
||
|
|
import setting
|
||
|
|
|
||
|
|
|
||
|
|
class UpdateNews(WebUtils):
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
super().__init__(module_='update_news')
|
||
|
|
news_sources = list(NewsSource.get_sources(db=self.mysqldb))
|
||
|
|
self.nb_tasks = len(news_sources)
|
||
|
|
self.start()
|
||
|
|
self.run(news_sources, self.update_news_from_source)
|
||
|
|
self.end()
|
||
|
|
|
||
|
|
def update_news_from_source(self, news_source, data):
|
||
|
|
try:
|
||
|
|
provider = ProviderController.get_provider(news_source.url)
|
||
|
|
newss = list()
|
||
|
|
for news in provider.get_newss_from_source(news_source, data):
|
||
|
|
news.store(db=self.mysqldb)
|
||
|
|
if news.id > 0: # News is not already existing
|
||
|
|
newss.append(news)
|
||
|
|
if len(newss) > 2 * setting.SEMAPHORE:
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
break # News already added so stop to prevent duplicate entries
|
||
|
|
|
||
|
|
self.run(newss, self.update_news_content)
|
||
|
|
self.run([news.image for news in newss if news.image.url is not None], self.update_news_image)
|
||
|
|
|
||
|
|
except BaseException as e:
|
||
|
|
news_source.error = format_exc()
|
||
|
|
self.logger.error('[-] news_source {}: {} - {}\n{}'.format(news_source.id, type(e), e, news_source.error))
|
||
|
|
else:
|
||
|
|
news_source.error = None
|
||
|
|
self.logger.info('[+] news source {}: OK'.format(news_source.id))
|
||
|
|
self.tasks_done += 1
|
||
|
|
finally:
|
||
|
|
news_source.store_error(db=self.mysqldb)
|
||
|
|
|
||
|
|
def update_news_content(self, news, data):
|
||
|
|
try:
|
||
|
|
provider = ProviderController.get_provider(news.url)
|
||
|
|
provider.get_news_content(news, data)
|
||
|
|
news.store_content(db=self.mysqldb)
|
||
|
|
|
||
|
|
except BaseException as e:
|
||
|
|
news.error = format_exc()
|
||
|
|
self.logger.error('[-] news content {}: {} - {}\n{}'.format(news.id, type(e), e, news.error))
|
||
|
|
self.errors += 1
|
||
|
|
else:
|
||
|
|
news.error = None
|
||
|
|
self.logger.info('[+] news content {}: OK'.format(news.id))
|
||
|
|
self.quantity += 1
|
||
|
|
finally:
|
||
|
|
news.store_error(db=self.mysqldb)
|
||
|
|
|
||
|
|
def update_news_image(self, news_image, data):
|
||
|
|
try:
|
||
|
|
with open(news_image.abspath, 'wb') as file:
|
||
|
|
file.write(data)
|
||
|
|
news_image.store(db=self.mysqldb)
|
||
|
|
|
||
|
|
except BaseException as e:
|
||
|
|
self.logger.error('[-] news image {}: {} - {}\n{}'.format(news_image.id_news, type(e), e, format_exc()))
|
||
|
|
self.errors += 1
|
||
|
|
else:
|
||
|
|
self.logger.info('[+] news image {}: OK'.format(news_image.id_news))
|
||
|
|
self.quantity += 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
UpdateNews()
|