Corona: rewrite for new data source

Corona/README.md

@@ -2,12 +2,8 @@ Return the latest Coronavirus (COVID-19) statistics globally or by country/state
 
 [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=T8E56M6SP9JH2)
 
-`config plugins.corona.template` - Configure the template for replies
+Plugin has been rewritten to use https://www.worldometers.info/coronavirus/ as data source.
-
-Template default: `\x02$location: \x0307$confirmed\x03 infected, \x0304$dead\x03 dead ($ratio), \x0309$recovered\x03 recovered. (Last update: $updated)`
 
 `config plugins.corona.countryFirst` - Country name abbreviations take precedence over USA state name abbreviations when `True`
 
 countryFirst default: `False`
-
-`config plugins.corona.cacheLifetime` - Time in seconds to cache API results. Default: `600`

Corona/config.py

@@ -50,11 +50,5 @@ def configure(advanced):
 
 Corona = conf.registerPlugin('Corona')
 
-conf.registerChannelValue(Corona, 'template',
-    registry.String("\x02$location: \x0307$confirmed\x03 infected, \x0304$dead\x03 dead ($ratio), \x0309$recovered\x03 recovered. (Last update: $updated)", _("""Template for replies""")))
-
 conf.registerChannelValue(Corona, 'countryFirst',
     registry.Boolean(False, _("Give preference to country name abbreviations over USA state name abbreviations")))
-
-conf.registerGlobalValue(Corona, 'cacheLifetime',
-    registry.Integer(600, _("Amount of time in seconds to cache API results")))

Corona/plugin.py

@@ -28,12 +28,13 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import json
 import requests
-import csv
+import pendulum
-import datetime
+import re
+from bs4 import BeautifulSoup
 from supybot import utils, plugins, ircutils, callbacks, log
 from supybot.commands import *
+
 try:
     from supybot.i18n import PluginInternationalization
     _ = PluginInternationalization('Corona')
@@ -59,7 +60,7 @@ countries = {
     "AU": "AUSTRALIA",
     "AT": "AUSTRIA",
     "AZ": "AZERBAIJAN",
-    "BS": "BAHAMAS, THE",
+    "BS": "BAHAMAS",
     "BH": "BAHRAIN",
     "BD": "BANGLADESH",
     "BB": "BARBADOS",
@@ -83,9 +84,9 @@ countries = {
     "KH": "CAMBODIA",
     "CM": "CAMEROON",
     "CA": "CANADA",
-    "CV": "CAPE VERDE",
+    "CV": "CABO VERDE",
     "KY": "CAYMAN ISLANDS",
-    "CF": "CENTRAL AFRICAN REPUBLIC",
+    "CF": "CAR",
     "TD": "CHAD",
     "CL": "CHILE",
     "CN": "CHINA",
@@ -93,8 +94,8 @@ countries = {
     "CC": "COCOS ISLANDS",
     "CO": "COLOMBIA",
     "KM": "COMOROS",
-    "CG": "CONGO (BRAZZAVILLE)",
+    "CG": "CONGO",
-    "CD": "CONGO (KINSHASA)",
+    "CD": "CONGO",
     "CK": "COOK ISLANDS",
     "CR": "COSTA RICA",
     "CI": "CÔTE D'IVOIRE",
@@ -119,11 +120,11 @@ countries = {
     "FJ": "FIJI",
     "FI": "FINLAND",
     "FR": "FRANCE",
-    "GF": "GUIANA",
+    "GF": "FRENCH GUIANA",
-    "PF": "POLYNESIA",
+    "PF": "FRENCH POLYNESIA",
     "TF": "FRENCH SOUTHERN TERRITORIES",
     "GA": "GABON",
-    "GM": "GAMBIA, THE",
+    "GM": "GAMBIA",
     "GE": "GEORGIA",
     "DE": "GERMANY",
     "GH": "GHANA",
@@ -136,11 +137,11 @@ countries = {
     "GT": "GUATEMALA",
     "GG": "GUERNSEY",
     "GN": "GUINEA",
-    "GW": "GUINEA-BISSAU",
+    "GW": "GUINEA",
     "GY": "GUYANA",
     "HT": "HAITI",
     "HM": "HEARD ISLAND AND MCDONALD ISLANDS",
-    "VA": "HOLY SEE",
+    "VA": "VATICAN CITY",
     "HN": "HONDURAS",
     "HK": "HONG KONG",
     "HU": "HUNGARY",
@@ -160,8 +161,8 @@ countries = {
     "KZ": "KAZAKHSTAN",
     "KE": "KENYA",
     "KI": "KIRIBATI",
-    "KP": "KOREA, NORTH",
+    "KP": "N. KOREA",
-    "KR": "KOREA, SOUTH",
+    "KR": "S. KOREA",
     "KW": "KUWAIT",
     "KG": "KYRGYZSTAN",
     "LA": "LAOS",
@@ -212,7 +213,7 @@ countries = {
     "OM": "OMAN",
     "PK": "PAKISTAN",
     "PW": "PALAU",
-    "PS": "PALESTINE, STATE OF",
+    "PS": "PALESTINE",
     "PA": "PANAMA",
     "PG": "PAPUA NEW GUINEA",
     "PY": "PARAGUAY",
@@ -227,13 +228,13 @@ countries = {
     "RO": "ROMANIA",
     "RU": "RUSSIA",
     "RW": "RWANDA",
-    "BL": "SAINT BARTHÉLEMY",
+    "BL": "ST. BARTH",
     "SH": "SAINT HELENA",
     "KN": "SAINT KITTS AND NEVIS",
     "LC": "SAINT LUCIA",
     "MF": "SAINT MARTIN",
     "PM": "SAINT PIERRE AND MIQUELON",
-    "VC": "SAINT VINCENT AND THE GRENADINES",
+    "VC": "ST. VINCENT GRENADINES",
     "WS": "SAMOA",
     "SM": "SAN MARINO",
     "ST": "SAO TOME AND PRINCIPE",
@@ -250,7 +251,7 @@ countries = {
     "SO": "SOMALIA",
     "ZA": "SOUTH AFRICA",
     "GS": "GEORGIA",
-    "SS": "SOUTH SUDAN",
+    "SS": "SUDAN",
     "ES": "SPAIN",
     "LK": "SRI LANKA",
     "SD": "SUDAN",
@@ -272,22 +273,22 @@ countries = {
     "TN": "TUNISIA",
     "TR": "TURKEY",
     "TM": "TURKMENISTAN",
-    "TC": "TURKS AND CAICOS ISLANDS",
+    "TC": "TURKS AND CAICOS",
     "TV": "TUVALU",
     "UG": "UGANDA",
     "UA": "UKRAINE",
     "AE": "UNITED ARAB EMIRATES",
-    "GB": "UNITED KINGDOM",
+    "GB": "UK",
-    "UK": "UNITED KINGDOM",
+    "UK": "UK",
-    "US": "US",
+    "US": "USA",
     "UM": "UNITED STATES MINOR OUTLYING ISLANDS",
     "UY": "URUGUAY",
     "UZ": "UZBEKISTAN",
     "VU": "VANUATU",
     "VE": "VENEZUELA",
     "VN": "VIETNAM",
-    "VG": "VIRGIN ISLANDS",
+    "VG": "U.S. VIRGIN ISLANDS",
-    "VI": "VIRGIN ISLANDS",
+    "VI": "U.S. VIRGIN ISLANDS",
     "WF": "WALLIS AND FUTUNA",
     "EH": "WESTERN SAHARA",
     "YE": "YEMEN",
@@ -362,58 +363,20 @@ class Corona(callbacks.Plugin):
     def __init__(self, irc):
         self.__parent = super(Corona, self)
         self.__parent.__init__(irc)
-        self.cache = {}
+        self.data = requests.structures.CaseInsensitiveDict()
+        self.updated = pendulum.yesterday()
 
-    def getCSV(self):
+    def time_created(self, time):
-        data = None
+        """
-        try:
+        Return relative time delta between now and s (dt string).
-            day = datetime.date.today().strftime('%m-%d-%Y')
+        """
-            url = "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/{0}.csv".format(day)
+        d = pendulum.now() - time
-            r = requests.get(url, timeout=10)
-            r.raise_for_status()
-        except (requests.exceptions.RequestException, requests.exceptions.HTTPError) as e:
-            log.debug('Corona: error retrieving data for today: {0}'.format(e))
-            try:
-                day = datetime.date.today() - datetime.timedelta(days=1)
-                day = day.strftime('%m-%d-%Y')
-                url = "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/{0}.csv".format(day)
-                r = requests.get(url, timeout=10)
-                r.raise_for_status()
-            except (requests.exceptions.RequestException, requests.exceptions.HTTPError) as e:
-                log.debug('Corona: error retrieving data for yesterday: {0}'.format(e))
-            else:
-                data = csv.DictReader(r.iter_lines(decode_unicode = True))
-        else:
-            data = csv.DictReader(r.iter_lines(decode_unicode = True))
-        return data
-
-    def getAPI(self):
-        data = None
-        url = "https://services1.arcgis.com/0MSEUqKaxRlEPj5g/arcgis/rest/services/ncov_cases/FeatureServer/1/query?f=json&where=Confirmed>0&outFields=*"
-        try:
-            r = requests.get(url, timeout=10)
-            r.raise_for_status()
-        except (requests.exceptions.RequestException, requests.exceptions.HTTPError) as e:
-            log.debug('Corona: error retrieving data from API: {0}'.format(e))
-        else:
-            try:
-                r = json.loads(r.content.decode())
-                data = r.get('features')
-            except:
-                data = None
-            if not data:
-                log.debug("Corona: Error retrieving features data from API.")
-        return data
-
-    def timeCreated(self, time):
-        time = datetime.datetime.fromtimestamp(time/1000.0)
-        d = datetime.datetime.now() - time
         if d.days:
-            rel_time = "{:1d} days ago".format(abs(d.days))
+            rel_time = "{:1d}d ago".format(abs(d.days))
         elif d.seconds > 3600:
-            rel_time = "{:.1f} hours ago".format(round((abs(d.seconds) / 3600),1))
+            rel_time = "{:.1f}h ago".format(round((abs(d.seconds) / 3600),1))
         elif 60 <= d.seconds < 3600:
-            rel_time = "{:.1f} minutes ago".format(round((abs(d.seconds) / 60),1))
+            rel_time = "{:.1f}m ago".format(round((abs(d.seconds) / 60),1))
         else:
             rel_time = "%ss ago" % (abs(d.seconds))
         return rel_time
@@ -426,65 +389,84 @@ class Corona(callbacks.Plugin):
         character) country abbreviations and US Postal (two character) state abbreviations.
         Invalid region names or search terms without data return global results.
         """
-        git = api = False
+        OK = False
-        data = None
+        try:
-        if len(self.cache) > 0:
+            r = requests.get('https://www.worldometers.info/coronavirus/', timeout=10)
-            cacheLifetime = self.registryValue("cacheLifetime")
+            r.raise_for_status()
-            now = datetime.datetime.now()
+            OK = True
-            seconds = (now - self.cache['timestamp']).total_seconds()
+        except (requests.exceptions.RequestException, requests.exceptions.HTTPError) as e:
-            if seconds < cacheLifetime:
+            log.debug('Corona: error retrieving World data from API: {0}'.format(e))
-                data = self.cache['data']
+            OK = False
-                api = True
+        soup = BeautifulSoup(r.content)
-                log.debug("Corona: returning cached API data")
+        updated = soup.find("div", text = re.compile('Last updated:'))
+        updated = updated.text.split(':', 1)[1].replace('GMT', '').strip()
+        updated = pendulum.from_format(updated, "MMMM DD, YYYY, HH:mm")
+        if OK and updated > self.updated:
+            self.updated = updated
+            table = soup.find("table", { "id" : "main_table_countries_today" })
+            n = 0
+            for row in table.findAll("tr"):
+                cells = row.findAll("td")
+                if len(cells) == 9:
+                    n += 1
+                    country = cells[0].text.strip()
+                    self.data[country] = {}
+                    self.data[country]['name'] = country
+                    self.data[country]['country'] = True
+                    self.data[country]['total_cases'] = cells[1].text.strip()
+                    if cells[2].text.strip():
+                        self.data[country]['new_cases'] = cells[2].text.strip()
                     else:
-                data = self.getAPI()
+                        self.data[country]['new_cases'] = '+0'
-                if data:
+                    self.data[country]['total_deaths'] = cells[3].text.strip()
-                    api = True
+                    if cells[4].text.strip():
-                    self.cache['timestamp'] = datetime.datetime.now()
+                        self.data[country]['new_deaths'] = cells[4].text.strip()
-                    self.cache['data'] = data
-                    log.debug("Corona: caching API data")
                     else:
-                    now = datetime.datetime.now()
+                        self.data[country]['new_deaths'] = '+0'
-                    midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
+                    self.data[country]['total_recovered'] = cells[5].text.strip()
-                    seconds = (now - midnight).seconds
+                    self.data[country]['active'] = cells[6].text.strip()
-                    if seconds > (now - self.cache['timestamp']).total_seconds():
+                    self.data[country]['serious'] = cells[7].text.strip()
-                        data = self.cache['data']
+                    self.data[country]['per_million'] = cells[8].text.strip()
-                        api = True
+                    self.data[country]['rank'] = "#{}".format(n)
-                        log.debug("Corona: error accessing API, returning cached API data")
+            try:
+                r = requests.get('https://www.worldometers.info/coronavirus/country/us/', timeout=10)
+                r.raise_for_status()
+                OK = True
+            except (requests.exceptions.RequestException, requests.exceptions.HTTPError) as e:
+                log.debug('Corona: error retrieving USA data from API: {0}'.format(e))
+                OK = False
+            if OK:
+                soup = BeautifulSoup(r.content)
+                table = soup.find("table", { "id" : "usa_table_countries_today" })
+                n = 0
+                for row in table.findAll("tr")[:-1]:
+                    cells = row.findAll("td")
+                    if len(cells) == 7:
+                        n += 1
+                        state = cells[0].text.strip()
+                        self.data[state] = {}
+                        self.data[state]['country'] = False
+                        self.data[state]['name'] = state
+                        self.data[state]['total_cases'] = cells[1].text.strip()
+                        if cells[2].text.strip():
+                            self.data[state]['new_cases'] = cells[2].text.strip()
                         else:
-                        data = self.getCSV()
+                            self.data[state]['new_cases'] = '+0'
-                        if data:
+                        self.data[state]['total_deaths'] = cells[3].text.strip()
-                            git = True
+                        if cells[4].text.strip():
+                            self.data[state]['new_deaths'] = cells[4].text.strip()
                         else:
-            data = self.getAPI()
+                            self.data[state]['new_deaths'] = '+0'
-            if data:
+                        self.data[state]['active'] = cells[5].text.strip()
-                api = True
+                        self.data[state]['rank'] = "#{}".format(n)
-                self.cache['timestamp'] = datetime.datetime.now()
-                self.cache['data'] = data
-                log.debug("Corona: caching API data")
             else:
-                data = self.getCSV()
+                log.debug("Corona: unable to retrieve latest USA data")
-                if data:
+        elif len(self.data) > 0:
-                    git = True
+            log.debug("Corona: data not yet updated, using cache")
-        if not data:
+        else:
-            irc.reply("Error. Unable to access database.")
+            log.debug("Corona: Error. Unable to retrieve data.")
             return
-        total_confirmed = total_deaths = total_recovered = last_update = 0
+        if search and len(search) == 2:
-        confirmed = deaths = recovered = updated = 0
-        location = 'Global'
-        for region in data:
-            if api:
-                r = region.get('attributes')
-            else:
-                r = region
-            if search:
-                if api:
-                    region = r.get('Country_Region')
-                    state = r.get('Province_State')
-                else:
-                    region = r.get('Country/Region')
-                    state = r.get('Province/State')
-                if len(search) == 2:
             if self.registryValue("countryFirst", msg.channel):
                 try:
                     search = countries[search.upper()]
@@ -501,64 +483,49 @@ class Corona(callbacks.Plugin):
                         search = countries[search.upper()]
                     except KeyError:
                         pass
-                if search.lower() == 'usa' or 'united states' in search.lower():
+        if search and self.data.get(search):
-                    search = 'us'
+            if self.data[search]['country']:
-                if 'korea' in search.lower():
+                ratio_dead = "{0:.1%}".format(int(self.data[search]['total_deaths'].replace(',', ''))/int(self.data[search]['total_cases'].replace(',', '')))
-                    search = 'korea, south'
+                mild = int(self.data[search]['active'].replace(',', '')) - int(self.data[search]['serious'].replace(',', ''))
-                if region and search.lower() == region.lower():
+                irc.reply("\x02\x1F{0}\x1F: World Rank: {1} | Cases: \x0307{2}\x03 (\x0307{3}\x03) | Deaths: \x0304{4}\x03 (\x0304{5}\x03) (\x0304{6}\x03) | Recovered: \x0309{7}\x03 | Active: \x0307{8}\x03 (\x0310{9}\x03 Mild) (\x0313{10}\x03 Serious) | Updated: {11}".format(
-                    location = region
+                    self.data[search]['name'],
-                    confirmed += int(r.get('Confirmed'))
+                    self.data[search]['rank'],
-                    deaths += int(r.get('Deaths'))
+                    self.data[search]['total_cases'],
-                    recovered += int(r.get('Recovered'))
+                    self.data[search]['new_cases'],
-                    if api:
+                    self.data[search]['total_deaths'],
-                        time = int(r.get('Last_Update'))
+                    self.data[search]['new_deaths'],
-                    if git:
+                    ratio_dead,
-                        time = datetime.datetime.strptime(r.get('Last Update'), "%Y-%m-%dT%H:%M:%S")
+                    self.data[search]['total_recovered'],
-                        time = int(time.timestamp()*1000)
+                    self.data[search]['active'],
-                    if time > updated:
+                    '{:,}'.format(mild),
-                        updated = time
+                    self.data[search]['serious'],
-                    local_ratio_dead = "{0:.1%}".format(deaths/confirmed)
+                    self.time_created(updated)))
-                elif state and search.lower() == state.lower():
-                    location = state
-                    confirmed += int(r.get('Confirmed'))
-                    deaths += int(r.get('Deaths'))
-                    recovered += int(r.get('Recovered'))
-                    if api:
-                        time = int(r.get('Last_Update'))
-                    if git:
-                        time = datetime.datetime.strptime(r.get('Last Update'), "%Y-%m-%dT%H:%M:%S")
-                        time = int(time.timestamp()*1000)
-                    if time > updated:
-                        updated = time
-                    local_ratio_dead = "{0:.1%}".format(deaths/confirmed)
-            total_confirmed += int(r.get('Confirmed'))
-            total_deaths += int(r.get('Deaths'))
-            total_recovered += int(r.get('Recovered'))
-            if api:
-                time = int(r.get('Last_Update'))
-            if git:
-                time = datetime.datetime.strptime(r.get('Last Update'), "%Y-%m-%dT%H:%M:%S")
-                time = int(time.timestamp()*1000)
-            if time > last_update:
-                last_update = time
-        ratio_dead = "{0:.1%}".format(total_deaths/total_confirmed)
-        template = self.registryValue("template", msg.channel)
-        if location == 'Global':
-            last_update = self.timeCreated(last_update)
-            template = template.replace("$location", location)
-            template = template.replace("$confirmed", str(total_confirmed))
-            template = template.replace("$dead", str(total_deaths))
-            template = template.replace("$recovered", str(total_recovered))
-            template = template.replace("$ratio", ratio_dead)
-            template = template.replace("$updated", last_update)
             else:
-            updated = self.timeCreated(updated)
+                ratio_dead = "{0:.1%}".format(int(self.data[search]['total_deaths'].replace(',', ''))/int(self.data[search]['total_cases'].replace(',', '')))
-            template = template.replace("$location", location)
+                irc.reply("\x02\x1F{0}\x1F: USA Rank: {1} | Cases: \x0307{2}\x03 (\x0307{3}\x03) | Deaths: \x0304{4}\x03 (\x0304{5}\x03) (\x0304{6}\x03) | Active: \x0307{7}\x03 | Updated: {8}".format(
-            template = template.replace("$confirmed", str(confirmed))
+                    self.data[search]['name'],
-            template = template.replace("$dead", str(deaths))
+                    self.data[search]['rank'],
-            template = template.replace("$recovered", str(recovered))
+                    self.data[search]['total_cases'],
-            template = template.replace("$ratio", local_ratio_dead)
+                    self.data[search]['new_cases'],
-            template = template.replace("$updated", updated)
+                    self.data[search]['total_deaths'],
-        irc.reply(template)
+                    self.data[search]['new_deaths'],
+                    ratio_dead,
+                    self.data[search]['active'],
+                    self.time_created(updated)))
+        else:
+            mild = int(self.data['total:']['active'].replace(',', '')) - int(self.data['total:']['serious'].replace(',', ''))
+            ratio_dead = "{0:.1%}".format(int(self.data['total:']['total_deaths'].replace(',', ''))/int(self.data['total:']['total_cases'].replace(',', '')))
+            irc.reply("\x02\x1F{0}\x1F: Cases: \x0307{1}\x03 (\x0307+{2}\x03) | Deaths: \x0304{3}\x03 (\x0304+{4}\x03) (\x0304{5}\x03) | Recovered: \x0309{6}\x03 | Active: \x0307{7}\x03 (\x0310{8}\x03 Mild) (\x0313{9}\x03 Serious) | Updated: {10}".format(
+                'Global',
+                self.data['total:']['total_cases'],
+                self.data['total:']['new_cases'],
+                self.data['total:']['total_deaths'],
+                self.data['total:']['new_deaths'],
+                ratio_dead,
+                self.data['total:']['total_recovered'],
+                self.data['total:']['active'],
+                '{:,}'.format(mild),
+                self.data['total:']['serious'],
+                self.time_created(updated)))
 
 Class = Corona

Corona/requirements.txt

@@ -1 +1,3 @@
 requests
+pendulum
+beautifulsoup4