Serverless Galore - 3 Open Source Libraries Compared

from goose3 import Goose
import time
import pkgutil
from typing import Tuple
TIMEOUT = 50 # [seconds]
ERRORS = (
'Access Denied',
'Site Not Available',
'Page not found',
'Unavailable for legal reasons',
'404 Error'
)
def extract(g: Goose, url: str, extracted: int, missed: int, elapsed: int) -> Tuple[int, int]:
    try:
        article = g.extract(url=url)
        if (len(article.title) > 20):
            extracted += 1
        # it looks so strange but goose3 return exception message as article.title rather than rising an exception
        if (article.title in ERRORS):
            missed += 1
        print(f'{extracted}/{missed}', ' ', int(elapsed), ' ', article.title)
    except Exception as exc:
        print(f'{url}: {extracted}/{missed}', ' ', int(elapsed), 'exc: ', exc)
        missed += 1
   return extracted, missed
class Goose3Benchmark:
'''Goose3 Python library test service'''
def _run_single_thread(self) -> Tuple[int, int]:
    '''run Goose3 benchmark in one thread'''
    extracted = 0
    missed = 0
    urls = pkgutil.get_data('data', 'urls.txt').decode('utf-8').split('\n')
    start = time.time()
 
    with Goose() as g:
        for url in urls:
            elapsed = time.time() - start
            if elapsed > TIMEOUT:
                break
            extracted, missed = extract(g, url, extracted, missed, elapsed)
return extracted, missed

caios test_remote run