From f6e499d85ec99ff7ff0aba222a32505f4a078e5e Mon Sep 17 00:00:00 2001 From: James Lu Date: Fri, 23 Jun 2017 23:19:07 -0700 Subject: [PATCH 1/9] PkgInfo: abstract 'pkg' to support multiple backend functions --- PkgInfo/plugin.py | 171 ++++++++++++++++++++++++++++++---------------- 1 file changed, 113 insertions(+), 58 deletions(-) diff --git a/PkgInfo/plugin.py b/PkgInfo/plugin.py index 291d21e..de09db2 100644 --- a/PkgInfo/plugin.py +++ b/PkgInfo/plugin.py @@ -1,5 +1,5 @@ ### -# Copyright (c) 2014-2016, James Lu +# Copyright (c) 2014-2017, James Lu # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -43,6 +43,7 @@ except ImportError: # Python 2 from urllib import urlencode, quote import json import re +import sys try: from bs4 import BeautifulSoup except ImportError: @@ -50,6 +51,14 @@ except ImportError: " at http://www.crummy.com/software/BeautifulSoup/bs4/" "doc/#installing-beautiful-soup") +# supybot.commands overrides any by default which is horrible ... +# Also horrible is how accessing items from __builtins__ requires different +# syntax on Python 2 and 3. +if sys.version_info[0] >= 3: + any = __builtins__['any'] +else: + any = __builtins__.any + try: from supybot.i18n import PluginInternationalization _ = PluginInternationalization('PkgInfo') @@ -122,41 +131,44 @@ def _getDistro(release): elif release.startswith(debian_archive): return "debian-archive" +class UnknownDistributionError(ValueError): + pass + class PkgInfo(callbacks.Plugin): """Fetches package information from the repositories of Arch Linux, CentOS, Debian, Fedora, FreeBSD, Linux Mint, and Ubuntu.""" threaded = True - _deptypes = ['dep', 'rec', 'sug', 'enh', 'adep', 'idep'] - _dependencyColor = utils.str.MultipleReplacer({'rec:': '\x0312rec:\x03', - 'dep:': '\x0304dep:\x03', - 'sug:': '\x0309sug:\x03', - 'adep:': '\x0305adep:\x03', - 'idep:': '\x0302idep:\x03', - 'enh:': '\x0308enh:\x03'}) - def package(self, irc, msg, args, release, pkg, opts): - """ [--depends] [--source] + _get_dependency_color = utils.str.MultipleReplacer({ + # Debian/Ubuntu names + 'dep:': '\x0304dep:\x03', + 'rec:': '\x0312rec:\x03', + 'sug:': '\x0309sug:\x03', + 'adep:': '\x0305adep:\x03', + 'idep:': '\x0302idep:\x03', + 'enh:': '\x0308enh:\x03', + }) - Fetches information for from Debian or Ubuntu's repositories. - is the codename/release name (e.g. 'trusty', 'squeeze'). If - --depends is given, fetches dependency info for . If --source - is given, look up the source package instead of a binary.""" - pkg = pkg.lower() - distro = _getDistro(release) - opts = dict(opts) - source = 'source' in opts - try: - url = addrs[distro] - except KeyError: - irc.error(unknowndist, Raise=True) - if source: # Source package was requested + def get_distro_fetcher(self, dist): + dist = dist.lower() + + debian = ("oldoldstable", "oldstable", "wheezy", "stable", + "jessie", "testing", "sid", "unstable", "stretch", "buster", + "experimental", "bullseye") + ubuntu = ("precise", "trusty", "xenial", "yakkety", "zesty", "artful") + + if dist.startswith(debian): + return self.debian_fetcher + elif dist.startswith(ubuntu): + return self.ubuntu_fetcher + + def debian_fetcher(self, release, query, baseurl='https://packages.debian.org/', fetch_source=False, fetch_depends=False): + url = baseurl + if fetch_source: # Source package was requested url += 'source/' - url += "{}/{}".format(release, pkg) + url += "{}/{}".format(release, query) - try: - text = utils.web.getUrl(url).decode("utf-8") - except utils.web.Error as e: - irc.error(str(e), Raise=True) + text = utils.web.getUrl(url).decode("utf-8") # Workaround unescaped << in package versions (e.g. "python (<< 2.8)") not being parsed # correctly. @@ -167,27 +179,33 @@ class PkgInfo(callbacks.Plugin): if "Error" in soup.title.string: err = soup.find('div', attrs={"id": "content"}).find('p').string if "two or more packages specified" in err: - irc.error("Unknown distribution/release.", Raise=True) - irc.reply(err) - return + raise UnknownDistributionError("Unknown distribution/release.") # If we're using the --depends option, handle that separately. - if 'depends' in opts: + if fetch_depends: items = soup.find('div', {'id': 'pdeps'}).find_all('dt') # Store results by type and name, but in an ordered fashion: show dependencies first, # followed by recommends, suggests, and enhances. - res = OrderedDict((deptype+':', []) for deptype in self._deptypes) + # "adep" and "idep" are arch-dependent and arch-independent build-dependencies + # respectively. + res = OrderedDict((deptype, []) for deptype in ('dep:', 'rec:', 'sug:', 'enh:', 'adep:', 'idep:')) + for item in items: # Get package name and related versions and architectures: # (>= 1.0) [arch1, arch2] try: + # The dependency type is in a element in front of the package name, + # which is expressed as a link. deptype = item.span.text if item.find('span') else '' if deptype not in res: continue # Ignore unsupported fields + # Also include any parts directly after the package name (usually a version + # restriction). name = '%s %s' % (ircutils.bold(item.a.text), item.a.next_sibling.replace('\n', '').strip()) name = utils.str.normalizeWhitespace(name).strip() + if item.find('span'): res[deptype].append(name) else: @@ -196,44 +214,81 @@ class PkgInfo(callbacks.Plugin): except AttributeError: continue - if res: - s = format("Package \x02%s\x02 dependencies: ", pkg) - for deptype, packages in res.items(): - if packages: - deptype = self._dependencyColor(deptype) - s += format("%s %L; ", deptype, packages) - s += format("%u", url) - - irc.reply(s) - - else: - irc.error("%s doesn't seem to have any dependencies." % pkg) - return + return res + # Fetch package information from the packages page's tags. desc = soup.find('meta', attrs={"name": "Description"})["content"] + keywords = soup.find('meta', attrs={"name": "Keywords"})["content"] + keywords = keywords.replace(",", "").split() + try: + real_distribution = keywords[1] + except IndexError: + return # No such package + version = keywords[-1] - # Override description if we selected source lookup, since the meta - # tag Description should be empty for those. Replace this with a list + # Override the description if we selected source lookup, since the meta + # tag Description will be empty for those. Replace this with a list # of binary packages that the source package builds. - if source: + if fetch_source: binaries = soup.find('div', {'id': "pbinaries"}) binaries = [ircutils.bold(obj.a.text) for obj in binaries.find_all('dt')] desc = format('Built packages: %L', binaries) - # Get package information from the meta tags - keywords = soup.find('meta', attrs={"name": "Keywords"})["content"] - keywords = keywords.replace(",", "").split() - version = keywords[-1] - # Handle virtual packages by showing a list of packages that provide it if version == "virtual": providing = [ircutils.bold(obj.a.text) for obj in soup.find_all('dt')] desc = "Virtual package provided by: %s" % ', '.join(providing[:10]) - if len(providing) > 10: + if len(providing) > 10: # XXX: arbitrary limit desc += " and %s others" % (ircutils.bold(len(providing) - 10)) - s = format("Package: \x02%s (%s)\x02 in %s - %s, View more at: %u", - pkg, version, keywords[1], desc, url) - irc.reply(s) + + return (query, version, real_distribution, desc, url) + + def ubuntu_fetcher(self, *args, **kwargs): + kwargs['baseurl'] = 'https://packages.ubuntu.com/' + return self.debian_fetcher(*args, **kwargs) + + def package(self, irc, msg, args, dist, query, opts): + """ [--depends] [--source] + + Fetches information for from Debian or Ubuntu's repositories. + is the codename/release name (e.g. 'trusty', 'squeeze'). If + --depends is given, fetches dependency info for . If --source + is given, look up the source package instead of a binary.""" + + distro_fetcher = self.get_distro_fetcher(dist) + if distro_fetcher is None: + irc.error("Unknown distribution version %r" % dist, Raise=True) + + opts = dict(opts) + fetch_source = 'source' in opts + fetch_depends = 'depends' in opts + + result = distro_fetcher(dist, query, fetch_source=fetch_source, fetch_depends=fetch_depends) + if not result: + irc.error("Unknown package %r" % query, Raise=True) + + if fetch_depends: + # results is a dictionary mapping dependency type to a list + # of packages. + if any(result.values()): + deplists = [] + for deptype, packages in result.items(): + if packages: + deptype = self._get_dependency_color(deptype) + # Join together the dependency type and package list for each list + # that isn't empty. + deplists.append("%s %s" % (ircutils.bold(deptype), ', '.join(packages))) + + irc.reply(format("Package %s dependencies: %s", ircutils.bold(query), '; '.join(deplists))) + + else: + irc.error("%s doesn't seem to have any dependencies." % ircutils.bold(query)) + else: + # result is formatted in the order: packagename, version, real_distribution, desc, url + self.log.info('args: %s', str(result)) + s = format("Package: \x02%s (%s)\x02 in %s - %s, View more at: %u", *result) + irc.reply(s) + pkg = wrap(package, ['somethingWithoutSpaces', 'somethingWithoutSpaces', getopts({'depends': '', 'source': ''})]) From 8b6ab5965db7efca5d3598960eabc7fed1d4dabb Mon Sep 17 00:00:00 2001 From: James Lu Date: Fri, 23 Jun 2017 23:22:58 -0700 Subject: [PATCH 2/9] PkgInfo: reduce code duplication between guess_distro and get_distro_fetcher --- PkgInfo/plugin.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/PkgInfo/plugin.py b/PkgInfo/plugin.py index de09db2..09a4eb5 100644 --- a/PkgInfo/plugin.py +++ b/PkgInfo/plugin.py @@ -113,7 +113,7 @@ addrs = {'ubuntu': 'https://packages.ubuntu.com/', 'debian-archive': 'http://archive.debian.net/'} _normalize = lambda text: utils.str.normalizeWhitespace(text).strip() -def _getDistro(release): +def _guess_distro(release): """ Guesses the distribution from the release name.""" @@ -121,15 +121,12 @@ def _getDistro(release): debian = ("oldoldstable", "oldstable", "wheezy", "stable", "jessie", "testing", "sid", "unstable", "stretch", "buster", "experimental", "bullseye") - debian_archive = ("bo", "hamm", "slink", "potato", "woody", "sarge", - "etch", "lenny", "squeeze") ubuntu = ("precise", "trusty", "xenial", "yakkety", "zesty", "artful") + if release.startswith(debian): return "debian" elif release.startswith(ubuntu): return "ubuntu" - elif release.startswith(debian_archive): - return "debian-archive" class UnknownDistributionError(ValueError): pass @@ -150,16 +147,11 @@ class PkgInfo(callbacks.Plugin): }) def get_distro_fetcher(self, dist): - dist = dist.lower() + dist = _guess_distro(dist) - debian = ("oldoldstable", "oldstable", "wheezy", "stable", - "jessie", "testing", "sid", "unstable", "stretch", "buster", - "experimental", "bullseye") - ubuntu = ("precise", "trusty", "xenial", "yakkety", "zesty", "artful") - - if dist.startswith(debian): + if dist == 'debian': return self.debian_fetcher - elif dist.startswith(ubuntu): + elif dist == 'ubuntu': return self.ubuntu_fetcher def debian_fetcher(self, release, query, baseurl='https://packages.debian.org/', fetch_source=False, fetch_depends=False): @@ -302,7 +294,7 @@ class PkgInfo(callbacks.Plugin): pkg, distro = map(str.lower, (pkg, distro)) supported = ("debian", "ubuntu", "derivatives", "all") if distro not in supported: - distro = _getDistro(distro) + distro = _guess_distro(distro) if distro is None: irc.error(unknowndist, Raise=True) opts = dict(opts) @@ -411,7 +403,7 @@ class PkgInfo(callbacks.Plugin): 'debian', 'ubuntu', and 'debian-archive'.""" distro = distro.lower() if distro not in addrs.keys(): - distro = _getDistro(distro) + distro = _guess_distro(distro) try: url = '%ssearch?keywords=%s' % (addrs[distro], quote(query)) except KeyError: @@ -454,7 +446,7 @@ class PkgInfo(callbacks.Plugin): Searches what package in Debian or Ubuntu has which file. is the codename/release name (e.g. xenial or jessie).""" release = release.lower() - distro = _getDistro(release) + distro = _guess_distro(release) try: url = '%ssearch?keywords=%s&searchon=contents&suite=%s' % (addrs[distro], quote(query), quote(release)) From 9c7e088ee6a81674b12bdecfae2e9fe2501e3f16 Mon Sep 17 00:00:00 2001 From: James Lu Date: Fri, 23 Jun 2017 23:51:25 -0700 Subject: [PATCH 3/9] PkgInfo: add arch_fetcher to implement 'pkg arch ' --- PkgInfo/plugin.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/PkgInfo/plugin.py b/PkgInfo/plugin.py index 09a4eb5..defcea2 100644 --- a/PkgInfo/plugin.py +++ b/PkgInfo/plugin.py @@ -147,15 +147,19 @@ class PkgInfo(callbacks.Plugin): }) def get_distro_fetcher(self, dist): - dist = _guess_distro(dist) + dist = dist.lower() + guess_dist = _guess_distro(dist) - if dist == 'debian': + if dist in ('archlinux', 'arch'): + return self.arch_fetcher + elif guess_dist == 'debian': return self.debian_fetcher - elif dist == 'ubuntu': + elif guess_dist == 'ubuntu': return self.ubuntu_fetcher def debian_fetcher(self, release, query, baseurl='https://packages.debian.org/', fetch_source=False, fetch_depends=False): url = baseurl + query = query.lower() if fetch_source: # Source package was requested url += 'source/' url += "{}/{}".format(release, query) @@ -239,6 +243,24 @@ class PkgInfo(callbacks.Plugin): kwargs['baseurl'] = 'https://packages.ubuntu.com/' return self.debian_fetcher(*args, **kwargs) + def arch_fetcher(self, release, query, fetch_source=False, fetch_depends=False): + search_url = 'https://www.archlinux.org/packages/search/json/?%s&arch=x86_64&arch=any' % urlencode({'name': query}) + + self.log.debug("PkgInfo: using url %s for arch_fetcher", search_url) + + fd = utils.web.getUrl(search_url) + data = json.loads(fd.decode("utf-8")) + + if data['valid'] and data['results']: + pkgdata = data['results'][0] + name, version, repo, arch, desc = pkgdata['pkgname'], pkgdata['pkgver'], pkgdata['repo'], pkgdata['arch'], pkgdata['pkgdesc'] + + # Package site URLs use a form like https://www.archlinux.org/packages/extra/x86_64/python/ + friendly_url = 'https://www.archlinux.org/packages/%s/%s/%s' % (repo, arch, name) + return (name, version, repo, desc, friendly_url) + else: + return # No results found! + def package(self, irc, msg, args, dist, query, opts): """ [--depends] [--source] @@ -277,8 +299,8 @@ class PkgInfo(callbacks.Plugin): irc.error("%s doesn't seem to have any dependencies." % ircutils.bold(query)) else: # result is formatted in the order: packagename, version, real_distribution, desc, url - self.log.info('args: %s', str(result)) - s = format("Package: \x02%s (%s)\x02 in %s - %s, View more at: %u", *result) + self.log.debug('PkgInfo result args: %s', str(result)) + s = format("Package: \x02%s (%s)\x02 in %s - %s %u", *result) irc.reply(s) pkg = wrap(package, ['somethingWithoutSpaces', 'somethingWithoutSpaces', From 4a1f670795939d70b6aeca8b8df6e674150a76c3 Mon Sep 17 00:00:00 2001 From: James Lu Date: Sat, 24 Jun 2017 00:13:31 -0700 Subject: [PATCH 4/9] PkgInfo: implement --depends support for arch_fetcher --- PkgInfo/plugin.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/PkgInfo/plugin.py b/PkgInfo/plugin.py index defcea2..0cdd79a 100644 --- a/PkgInfo/plugin.py +++ b/PkgInfo/plugin.py @@ -144,6 +144,8 @@ class PkgInfo(callbacks.Plugin): 'adep:': '\x0305adep:\x03', 'idep:': '\x0302idep:\x03', 'enh:': '\x0308enh:\x03', + # Generic + 'depends': '\x0304depends\x03', }) def get_distro_fetcher(self, dist): @@ -255,6 +257,21 @@ class PkgInfo(callbacks.Plugin): pkgdata = data['results'][0] name, version, repo, arch, desc = pkgdata['pkgname'], pkgdata['pkgver'], pkgdata['repo'], pkgdata['arch'], pkgdata['pkgdesc'] + if fetch_depends: + deps = set() + for dep in pkgdata['depends']: + # XXX: Arch's API does not differentiate between required deps and optional ones w/o explanation... + + # Sort through the API info and better explain optional dependencies with reasons in them. + if ':' in dep: + name, explanation = dep.split(':', 1) + dep = '%s (optional; needed for %s)' % (ircutils.bold(name), explanation.strip()) + else: + dep = ircutils.bold(dep) + deps.add(dep) + + return {'depends:': deps} + # Package site URLs use a form like https://www.archlinux.org/packages/extra/x86_64/python/ friendly_url = 'https://www.archlinux.org/packages/%s/%s/%s' % (repo, arch, name) return (name, version, repo, desc, friendly_url) @@ -293,7 +310,7 @@ class PkgInfo(callbacks.Plugin): # that isn't empty. deplists.append("%s %s" % (ircutils.bold(deptype), ', '.join(packages))) - irc.reply(format("Package %s dependencies: %s", ircutils.bold(query), '; '.join(deplists))) + irc.reply(format("%s %s", ircutils.bold(query), '; '.join(deplists))) else: irc.error("%s doesn't seem to have any dependencies." % ircutils.bold(query)) From 743545c41dbe95c2bd0a9ca340710652cd72d495 Mon Sep 17 00:00:00 2001 From: James Lu Date: Sat, 24 Jun 2017 00:27:59 -0700 Subject: [PATCH 5/9] PkgInfo: flag outdated versions in arch_fetcher as red --- PkgInfo/plugin.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/PkgInfo/plugin.py b/PkgInfo/plugin.py index 0cdd79a..39e35d6 100644 --- a/PkgInfo/plugin.py +++ b/PkgInfo/plugin.py @@ -35,6 +35,7 @@ import supybot.plugins as plugins import supybot.ircutils as ircutils import supybot.callbacks as callbacks import supybot.log as log +import supybot.conf as conf from collections import OrderedDict, defaultdict try: # Python 3 @@ -44,6 +45,7 @@ except ImportError: # Python 2 import json import re import sys +import time try: from bs4 import BeautifulSoup except ImportError: @@ -257,6 +259,16 @@ class PkgInfo(callbacks.Plugin): pkgdata = data['results'][0] name, version, repo, arch, desc = pkgdata['pkgname'], pkgdata['pkgver'], pkgdata['repo'], pkgdata['arch'], pkgdata['pkgdesc'] + if pkgdata['flag_date']: + # Mark flagged-as-outdated versions in red. + version = '\x0304%s\x03' % version + + # Note the flagged date in the package description. + t = time.strptime(pkgdata['flag_date'], '%Y-%m-%dT%H:%M:%S.%fZ') # Why can't strptime be smarter and guess this?! + # Convert the time format to the globally configured one. + out_t = time.strftime(conf.supybot.reply.format.time(), t) + desc += ' [flagged as \x0304outdated\x03 on %s]' % out_t + if fetch_depends: deps = set() for dep in pkgdata['depends']: From ea2c6b0030d9ce67478dbddb1929f36ecb7c4cb9 Mon Sep 17 00:00:00 2001 From: James Lu Date: Sat, 24 Jun 2017 12:09:03 -0700 Subject: [PATCH 6/9] PkgInfo: consistently drop colons from _get_dependency_color --- PkgInfo/plugin.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/PkgInfo/plugin.py b/PkgInfo/plugin.py index 39e35d6..f607bf9 100644 --- a/PkgInfo/plugin.py +++ b/PkgInfo/plugin.py @@ -140,14 +140,15 @@ class PkgInfo(callbacks.Plugin): _get_dependency_color = utils.str.MultipleReplacer({ # Debian/Ubuntu names - 'dep:': '\x0304dep:\x03', - 'rec:': '\x0312rec:\x03', - 'sug:': '\x0309sug:\x03', - 'adep:': '\x0305adep:\x03', - 'idep:': '\x0302idep:\x03', - 'enh:': '\x0308enh:\x03', + 'dep': '\x0304dep\x03', + 'rec': '\x0312rec\x03', + 'sug': '\x0309sug\x03', + 'adep': '\x0305adep\x03', + 'idep': '\x0302idep\x03', + 'enh': '\x0308enh\x03', # Generic 'depends': '\x0304depends\x03', + 'optdepends': '\x0312optdepends\x03' }) def get_distro_fetcher(self, dist): @@ -282,7 +283,7 @@ class PkgInfo(callbacks.Plugin): dep = ircutils.bold(dep) deps.add(dep) - return {'depends:': deps} + return {'depends': deps} # Package site URLs use a form like https://www.archlinux.org/packages/extra/x86_64/python/ friendly_url = 'https://www.archlinux.org/packages/%s/%s/%s' % (repo, arch, name) From 3d3855c3427292acda0cf09c5594f748a6933074 Mon Sep 17 00:00:00 2001 From: James Lu Date: Sat, 24 Jun 2017 12:24:08 -0700 Subject: [PATCH 7/9] PkgInfo.debian_fetcher: fix parsing OR dependencies and unavailable packages --- PkgInfo/plugin.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/PkgInfo/plugin.py b/PkgInfo/plugin.py index f607bf9..020b160 100644 --- a/PkgInfo/plugin.py +++ b/PkgInfo/plugin.py @@ -184,36 +184,43 @@ class PkgInfo(callbacks.Plugin): # If we're using the --depends option, handle that separately. if fetch_depends: - items = soup.find('div', {'id': 'pdeps'}).find_all('dt') + items = soup.find('div', {'id': 'pdeps'}).find_all('dl') # Store results by type and name, but in an ordered fashion: show dependencies first, # followed by recommends, suggests, and enhances. # "adep" and "idep" are arch-dependent and arch-independent build-dependencies # respectively. res = OrderedDict((deptype, []) for deptype in ('dep:', 'rec:', 'sug:', 'enh:', 'adep:', 'idep:')) - for item in items: + for item_wrapper in items: # Get package name and related versions and architectures: # (>= 1.0) [arch1, arch2] - try: + last_deptype = '' + for count, item in enumerate(item_wrapper.find_all('dt')): # The dependency type is in a element in front of the package name, # which is expressed as a link. - deptype = item.span.text if item.find('span') else '' + deptype = item.span.text if item.find('span') else last_deptype + last_deptype = deptype if deptype not in res: continue # Ignore unsupported fields # Also include any parts directly after the package name (usually a version # restriction). - name = '%s %s' % (ircutils.bold(item.a.text), - item.a.next_sibling.replace('\n', '').strip()) + try: + name = '%s %s' % (ircutils.bold(item.a.text), + item.a.next_sibling.replace('\n', '').strip()) + except AttributeError: + # No package link usually means that the package isn't available + name = item.string + if name: + name = ircutils.bold(name.splitlines()[1].strip()) name = utils.str.normalizeWhitespace(name).strip() + self.log.debug('PkgInfo.debian_fetcher: got %s %s for package %s', deptype, name, query) - if item.find('span'): + if count == 0: res[deptype].append(name) else: # OR dependency; format accordingly res[deptype][-1] += " or %s" % name - except AttributeError: - continue return res From bc9b1e9e58a98e598a71eb17887ed459f9d2f318 Mon Sep 17 00:00:00 2001 From: James Lu Date: Sat, 24 Jun 2017 12:38:18 -0700 Subject: [PATCH 8/9] PkgInfo: add 'pkg aur ' fetcher --- PkgInfo/plugin.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/PkgInfo/plugin.py b/PkgInfo/plugin.py index 020b160..eb93835 100644 --- a/PkgInfo/plugin.py +++ b/PkgInfo/plugin.py @@ -157,6 +157,8 @@ class PkgInfo(callbacks.Plugin): if dist in ('archlinux', 'arch'): return self.arch_fetcher + elif dist in ('archaur', 'aur'): + return self.arch_aur_fetcher elif guess_dist == 'debian': return self.debian_fetcher elif guess_dist == 'ubuntu': @@ -298,6 +300,56 @@ class PkgInfo(callbacks.Plugin): else: return # No results found! + + def arch_aur_fetcher(self, release, query, fetch_source=False, fetch_depends=False): + search_url = 'https://aur.archlinux.org/rpc/?' + urlencode( + {'arg[]': query, 'v': 5,'type': 'info'} + ) + + self.log.debug("PkgInfo: using url %s for arch_aur_fetcher", search_url) + + fd = utils.web.getUrl(search_url) + data = json.loads(fd.decode("utf-8")) + + if data['results']: + pkgdata = data['results'][0] + name, version, votecount, popularity, desc = pkgdata['Name'], pkgdata['Version'], \ + pkgdata['NumVotes'], pkgdata['Popularity'], pkgdata['Description'] + + verbose_info = ' [Popularity: \x02%s\x02; Votes: \x02%s\x02' % (popularity, votecount) + + if pkgdata['OutOfDate']: + # Mark flagged-as-outdated versions in red. + version = '\x0304%s\x03' % version + + flag_time = time.strftime(conf.supybot.reply.format.time(), time.gmtime(pkgdata['OutOfDate'])) + verbose_info += '; flagged as \x0304outdated\x03 on %s' % flag_time + verbose_info += ']' + + if fetch_depends: + deplist = pkgdata['MakeDepends'] if fetch_source else pkgdata['Depends'] + deplist = [ircutils.bold(dep) for dep in deplist] + + # Fill in opt depends + optdepends = set() + for dep in pkgdata.get('OptDepends', []): + if ':' in dep: + name, explanation = dep.split(':', 1) + dep = '%s (optional; needed for %s)' % (ircutils.bold(name), explanation.strip()) + else: + dep = '%s (optional)' % ircutils.bold(dep) + optdepends.add(dep) + + # Note: this is an ordered dict so that depends always show before optdepends + return OrderedDict((('depends', deplist), ('optdepends', optdepends))) + + # Package site URLs use a form like https://www.archlinux.org/packages/extra/x86_64/python/ + friendly_url = 'https://aur.archlinux.org/packages/%s/' % name + desc += verbose_info + return (name, version, 'Arch Linux AUR', desc, friendly_url) + else: + return # No results found! + def package(self, irc, msg, args, dist, query, opts): """ [--depends] [--source] @@ -326,6 +378,8 @@ class PkgInfo(callbacks.Plugin): for deptype, packages in result.items(): if packages: deptype = self._get_dependency_color(deptype) + if ':' not in deptype: + deptype += ':' # Join together the dependency type and package list for each list # that isn't empty. deplists.append("%s %s" % (ircutils.bold(deptype), ', '.join(packages))) From 0c82f3f4aa35b0bace2f5311c0b9fd2ae55ae2ae Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 6 Jul 2017 13:58:37 -0700 Subject: [PATCH 9/9] PkgInfo: merge Fedora and Linux Mint support into 'package' --- PkgInfo/plugin.py | 95 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/PkgInfo/plugin.py b/PkgInfo/plugin.py index eb93835..e3d46e2 100644 --- a/PkgInfo/plugin.py +++ b/PkgInfo/plugin.py @@ -124,15 +124,24 @@ def _guess_distro(release): "jessie", "testing", "sid", "unstable", "stretch", "buster", "experimental", "bullseye") ubuntu = ("precise", "trusty", "xenial", "yakkety", "zesty", "artful") + mint = ("betsy", "qiana", "rebecca", "rafaela", "rosa", "sarah", "serena", "sonya") if release.startswith(debian): return "debian" elif release.startswith(ubuntu): return "ubuntu" + elif release.startswith(mint): + return "mint" class UnknownDistributionError(ValueError): pass +class AmbiguousDistributionError(UnknownDistributionError): + pass + +class UnsupportedOperationError(NotImplementedError): + pass + class PkgInfo(callbacks.Plugin): """Fetches package information from the repositories of Arch Linux, CentOS, Debian, Fedora, FreeBSD, Linux Mint, and Ubuntu.""" @@ -155,7 +164,18 @@ class PkgInfo(callbacks.Plugin): dist = dist.lower() guess_dist = _guess_distro(dist) - if dist in ('archlinux', 'arch'): + if dist == 'debian': + raise AmbiguousDistributionError("You must specify a distribution version (e.g. 'stretch' or 'unstable')") + elif dist == 'ubuntu': + raise AmbiguousDistributionError("You must specify a distribution version (e.g. 'trusty' or 'xenial')") + elif dist in ('mint', 'linuxmint'): + raise AmbiguousDistributionError("You must specify a distribution version (e.g. 'sonya' or 'betsy')") + elif dist == 'fedora': + raise AmbiguousDistributionError("You must specify a distribution version (e.g. 'f26', 'rawhide' or 'epel7')") + elif dist == 'master': + raise AmbiguousDistributionError("'master' is ambiguous: for Fedora rawhide, use the release 'rawhide'") + + elif dist in ('archlinux', 'arch'): return self.arch_fetcher elif dist in ('archaur', 'aur'): return self.arch_aur_fetcher @@ -163,6 +183,10 @@ class PkgInfo(callbacks.Plugin): return self.debian_fetcher elif guess_dist == 'ubuntu': return self.ubuntu_fetcher + elif guess_dist == 'mint': + return self.mint_fetcher + elif dist.startswith(('f', 'el', 'epel', 'olpc', 'rawhide')): + return self.fedora_fetcher def debian_fetcher(self, release, query, baseurl='https://packages.debian.org/', fetch_source=False, fetch_depends=False): url = baseurl @@ -350,13 +374,74 @@ class PkgInfo(callbacks.Plugin): else: return # No results found! + def fedora_fetcher(self, release, query, fetch_source=False, fetch_depends=False): + if fetch_source or fetch_depends: + raise UnsupportedOperationError("--depends and --source lookup are not supported for Fedora") + + if release == 'master': + release = 'rawhide' + + url = 'https://admin.fedoraproject.org/pkgdb/api/packages/%s?format=json&branches=%s' % (quote(query), quote(release)) + self.log.debug("PkgInfo: using url %s for fedora_fetcher", url) + fd = utils.web.getUrl(url).decode("utf-8") + data = json.loads(fd) + result = data["packages"][0] + friendly_url = 'https://apps.fedoraproject.org/packages/%s' % query + + # XXX: find some way to fetch the package version, as pkgdb's api doesn't provide that info + return (result['name'], 'some version, see URL for details', release, result['description'].replace('\n', ' '), friendly_url) + + def mint_fetcher(self, release, query, fetch_source=False, fetch_depends=False): + if fetch_source: + addr = 'http://packages.linuxmint.com/list-src.php?' + else: + addr = 'http://packages.linuxmint.com/list.php?' + addr += urlencode({'release': release}) + + fd = utils.web.getUrl(addr).decode("utf-8") + + soup = BeautifulSoup(fd) + + # Linux Mint puts their package lists in tables, so use HTML parsing + results = soup.find_all("td") + + versions = {} + query = query.lower() + + for result in results: + name = result.contents[0].string # Package name + + if query == name: + # This feels like really messy code, but we have to find tags + # relative to our results. + # Ascend to find the section name (in

): + section = result.parent.parent.parent.previous_sibling.\ + previous_sibling.string + + # Find the package version in the next ; for some reason we + # have to go two siblings further, as the first .next_sibling + # returns '\n'. This is mentioned briefly in Beautiful Soup 4's + # documentation... + version = result.next_sibling.next_sibling.string + + # Create a list of versions because a package can exist multiple + # times in different sections of the repository (e.g. one in Main, + # one in Backports, etc.) + versions[section] = version + + return (query, ', '.join('%s: %s' % (k, v) for k, v in versions.items()), + 'Linux Mint %s' % release.title(), 'no description available', addr) + def package(self, irc, msg, args, dist, query, opts): """ [--depends] [--source] - Fetches information for from Debian or Ubuntu's repositories. - is the codename/release name (e.g. 'trusty', 'squeeze'). If - --depends is given, fetches dependency info for . If --source - is given, look up the source package instead of a binary.""" + Fetches information for from Arch Linux, Debian, Fedora, Linux Mint, or Ubuntu's repositories. + is the codename/release name (e.g. 'xenial', 'unstable', 'rawhide', 'f26', 'arch', archaur'). + + If --depends is given, fetches dependency info for . If --source is given, look up the source package + instead of a binary. + + This command replaces the 'fedora', 'archlinux', and 'archaur' commands from earlier versions of PkgInfo.""" distro_fetcher = self.get_distro_fetcher(dist) if distro_fetcher is None: