Browse Source

[gelbooru] improve and fix pagination (#2230, #2232)

Use 'id:<POSTID' as a tag instead of going through pages with 'pid'.

Something similar was already implemented in 93cef784,
but that got broken again in 3085aac4.
master
Mike Fährmann 5 months ago
parent
commit
e2be199124
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
  1. 4
      CHANGELOG.md
  2. 33
      gallery_dl/extractor/gelbooru.py
  3. 2
      gallery_dl/version.py

4
CHANGELOG.md

@ -1,9 +1,11 @@
# Changelog
## Unreleased
## 1.20.3 - 2022-01-26
### Fixes
- [kemonoparty] fix DMs extraction ([#2008](https://github.com/mikf/gallery-dl/issues/2008))
- [twitter] fix crash ob Tweets with deleted quotes ([#2225](https://github.com/mikf/gallery-dl/issues/2225))
- [twitter] fix crash on Tweets with deleted quotes ([#2225](https://github.com/mikf/gallery-dl/issues/2225))
- [twitter] fix crash on suspended Tweets without `legacy` entry ([#2216](https://github.com/mikf/gallery-dl/issues/2216))
- [twitter] fix crash on unified cards without `type`
- [twitter] prevent crash on invalid/deleted Retweets ([#2225](https://github.com/mikf/gallery-dl/issues/2225))

33
gallery_dl/extractor/gelbooru.py

@ -33,28 +33,20 @@ class GelbooruBase():
def _pagination(self, params):
params["pid"] = self.page_start
params["limit"] = self.per_page
limit = self.per_page // 2
post = None
while True:
try:
posts = self._api_request(params)
except ValueError:
if "tags" not in params or post is None:
raise
taglist = [tag for tag in params["tags"].split()
if not tag.startswith("id:<")]
taglist.append("id:<" + str(post.attrib["id"]))
params["tags"] = " ".join(taglist)
params["pid"] = 0
continue
post = None
posts = self._api_request(params)
for post in posts:
yield post
if len(posts) < self.per_page:
if len(posts) < limit:
return
params["pid"] += 1
if "pid" in params:
del params["pid"]
params["tags"] = "{} id:<{}".format(self.tags, post["id"])
@staticmethod
def _file_url(post):
@ -81,9 +73,12 @@ class GelbooruTagExtractor(GelbooruBase,
("https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", {
"count": 5,
}),
("https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", {
"options": (("api", False),),
"count": 5,
("https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", {
"range": "196-204",
"url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf",
"pattern": r"https://img\d\.gelbooru\.com"
r"/images/../../[0-9a-f]{32}\.jpg",
"count": 9,
}),
)

2
gallery_dl/version.py

@ -6,4 +6,4 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
__version__ = "1.20.3"
__version__ = "1.20.4-dev"

Loading…
Cancel
Save