diff options
| -rw-r--r-- | searx/engines/deviantart.py | 82 | ||||
| -rw-r--r-- | searx/settings.yml | 1 |
2 files changed, 45 insertions, 38 deletions
diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py index 8cbfb7ad9..776a3eef9 100644 --- a/searx/engines/deviantart.py +++ b/searx/engines/deviantart.py @@ -1,11 +1,18 @@ # SPDX-License-Identifier: AGPL-3.0-or-later """Deviantart (Images)""" +import typing as t + import urllib.parse from lxml import html +from searx.result_types import EngineResults from searx.utils import extract_text, eval_xpath, eval_xpath_list +if t.TYPE_CHECKING: + from searx.extended_types import SXNG_Response + from searx.search.processors import OnlineParams + # about about = { "website": 'https://www.deviantart.com/', @@ -23,63 +30,62 @@ paging = True # search-url base_url = 'https://www.deviantart.com' -results_xpath = '//div[@class="V_S0t_"]/div/div/a' -url_xpath = './@href' -thumbnail_src_xpath = './div/img/@src' -img_src_xpath = './div/img/@srcset' -title_xpath = './@aria-label' -premium_xpath = '../div/div/div/text()' -premium_keytext = 'Watch the artist to view this deviation' -cursor_xpath = '(//a[@class="vQ2brP"]/@href)[last()]' - +results_xpath = '//div[@data-testid="content_row"]//a[.//*[@data-testid="thumb"]]' +img_src_xpath = './/img/@srcset' +thumbnail_src_xpath = './/img/@src' +author_xpath = './/*[@property="schema:name"]/@content' +cursor_xpath = '//a[contains(@href, "cursor=") and contains(., "Next")]/@href' -def request(query, params): +def request(query: str, params: "OnlineParams"): # https://www.deviantart.com/search?q=foo - nextpage_url = params['engine_data'].get('nextpage') - # don't use nextpage when user selected to jump back to page 1 - if params['pageno'] > 1 and nextpage_url is not None: - params['url'] = nextpage_url - else: - params['url'] = f"{base_url}/search?{urllib.parse.urlencode({'q': query})}" + args = {'q': query} + if params['pageno'] > 1: + cursor = params['engine_data'].get('cursor') + if cursor: + args['cursor'] = cursor - return params + params['url'] = f"{base_url}/search?{urllib.parse.urlencode(args)}" -def response(resp): +def response(resp: "SXNG_Response") -> EngineResults: - results = [] + res = EngineResults() dom = html.fromstring(resp.text) for result in eval_xpath_list(dom, results_xpath): - # skip images that are blurred - _text = extract_text(eval_xpath(result, premium_xpath)) - if _text and premium_keytext in _text: - continue + thumbnail_src = extract_text(eval_xpath(result, thumbnail_src_xpath)) img_src = extract_text(eval_xpath(result, img_src_xpath)) + # mature locked thumbs have blur transform (blur_15, blur_30 etc..) + if ',blur_' in f'{thumbnail_src}{img_src}': + continue if img_src: img_src = img_src.split(' ')[0] parsed_url = urllib.parse.urlparse(img_src) img_src = parsed_url._replace(path=parsed_url.path.split('/v1')[0]).geturl() - results.append( - { - 'template': 'images.html', - 'url': extract_text(eval_xpath(result, url_xpath)), - 'img_src': img_src, - 'thumbnail_src': extract_text(eval_xpath(result, thumbnail_src_xpath)), - 'title': extract_text(eval_xpath(result, title_xpath)), - } + author = extract_text(eval_xpath(result, author_xpath)) + + res.add( + res.types.Image( + template='images.html', + url=result.get('href'), + img_src=img_src or "", + thumbnail_src=thumbnail_src or "", + title=result.get('aria-label'), + author=author or "", + ) ) nextpage_url = extract_text(eval_xpath(dom, cursor_xpath)) - if nextpage_url: - results.append( - { - 'engine_data': nextpage_url.replace("http://", "https://"), - 'key': 'nextpage', - } + cursor = urllib.parse.parse_qs(urllib.parse.urlparse(nextpage_url or '').query).get('cursor', [None])[0] + if cursor: + res.add( + res.types.LegacyResult( + engine_data=cursor, + key='cursor', + ) ) - return results + return res diff --git a/searx/settings.yml b/searx/settings.yml index a7cc04d23..2b6991749 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -761,6 +761,7 @@ engines: engine: deviantart shortcut: da timeout: 3.0 + inactive: true - name: devicons engine: devicons |