PsychoticNinja/oddluck-limnoria-plugins
1
0
Fork 0
mirror of https://github.com/oddluck/limnoria-plugins.git synced 2025-04-29 06:51:12 -05:00
Code Issues Projects Releases Wiki Activity

fix some bad links

Browse Source
This commit is contained in:
cottongin 2022-04-21 12:53:12 -05:00
parent b4f11088ec
commit ed3d345185
No known key found for this signature in database
GPG Key ID: C0D8954A47DCB4ED
1 changed files with 79 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
SpiffyTitles/plugin.py
View File

@ -266,7 +266,9 @@ class SpiffyTitles(callbacks.Plugin):
""" """
if self.registryValue("cacheGlobal"): if self.registryValue("cacheGlobal"):
channel = "global" channel = "global"
cached_link = self.get_link_from_cache(url, channel) # cached_link = self.get_link_from_cache(url, channel)
log.info("Skipping cache check")
cached_link = None
if cached_link: if cached_link:
title = cached_link["title"] title = cached_link["title"]
else: else:
@ -420,6 +422,9 @@ class SpiffyTitles(callbacks.Plugin):
Get the HTML of a website based on a URL Get the HTML of a website based on a URL
""" """
max_retries = self.registryValue("maxRetries") max_retries = self.registryValue("maxRetries")
size = conf.supybot.protocols.http.peekSize()
generic_error = self.registryValue("badLinkText", channel=channel)
response_size = 0
if retries >= max_retries: if retries >= max_retries:
log.debug("SpiffyTitles: hit maximum retries for %s" % url) log.debug("SpiffyTitles: hit maximum retries for %s" % url)
return (None, False) return (None, False)
@ -436,6 +441,14 @@ class SpiffyTitles(callbacks.Plugin):
stream=True, stream=True,
proxies=self.proxies, proxies=self.proxies,
) as request: ) as request:
for chunk in request.iter_content(chunk_size=1024):
if response_size > size:
request.close()
return (generic_error, False)
if 'content-length' in request.headers:
if int(request.headers['content-length']) > size:
log.debug("SpiffyTitles: URL ignored due to exceeding content size")
return (generic_error, False)
request.raise_for_status() request.raise_for_status()
if request.history: if request.history:
# check the top two domain levels # check the top two domain levels
@ -458,7 +471,7 @@ class SpiffyTitles(callbacks.Plugin):
" match: %s" " match: %s"
% url % url
) )
return return (None, False)
whitelist_pattern = self.registryValue( whitelist_pattern = self.registryValue(
"whitelistDomainPattern", channel=channel "whitelistDomainPattern", channel=channel
) )
@ -471,7 +484,7 @@ class SpiffyTitles(callbacks.Plugin):
" mismatch: %s" " mismatch: %s"
% url % url
) )
return return (None, False)
text = self.get_title_by_url(request.url, channel) text = self.get_title_by_url(request.url, channel)
text = text.lstrip("\x02").lstrip("^").strip() text = text.lstrip("\x02").lstrip("^").strip()
return (text, is_redirect) return (text, is_redirect)
@ -480,9 +493,11 @@ class SpiffyTitles(callbacks.Plugin):
acceptable_types = self.registryValue("default.mimeTypes") acceptable_types = self.registryValue("default.mimeTypes")
log.debug("SpiffyTitles: content type %s" % (content_type)) log.debug("SpiffyTitles: content type %s" % (content_type))
if content_type in acceptable_types: if content_type in acceptable_types:
text = request.content if chunk:
if text: title = self.get_title_from_html(chunk, channel)
return (self.get_title_from_html(text, channel), is_redirect) if not title:
continue
return (title, is_redirect)
else: else:
log.debug("SpiffyTitles: empty content from %s" % (url)) log.debug("SpiffyTitles: empty content from %s" % (url))
else: else:
@ -500,12 +515,13 @@ class SpiffyTitles(callbacks.Plugin):
{"type": content_type, "size": size} {"type": content_type, "size": size}
) )
return (text, is_redirect) return (text, is_redirect)
response_size += len(chunk)
except requests.exceptions.MissingSchema as e: except requests.exceptions.MissingSchema as e:
url_wschema = "http://%s" % (url) url_wschema = "http://%s" % (url)
log.error("SpiffyTitles missing schema. Retrying with %s" % (url_wschema)) log.error("SpiffyTitles missing schema. Retrying with %s" % (url_wschema))
info = urlparse(url_wschema) info = urlparse(url_wschema)
if self.is_ignored_domain(info.netloc, channel): if self.is_ignored_domain(info.netloc, channel):
return return (None, False)
else: else:
return self.get_source_by_url(url_wschema, channel) return self.get_source_by_url(url_wschema, channel)
except requests.exceptions.Timeout as e: except requests.exceptions.Timeout as e: