486 lines
18 KiB
Python
486 lines
18 KiB
Python
#!/usr/bin/env python3
|
||
"""Импорт карт и артов с официального сайта en.cf-vanguard.com.
|
||
|
||
Один источник и для параметров, и для картинок:
|
||
|
||
* список сетов: /cardlist/?change_ex -> код сета -> id раскрытия;
|
||
* список карт: /cardlist/cardsearch/?expansion=N&view=text&sort=no&page=P;
|
||
* карточка: /cardlist/?cardno=BT01/001EN&expansion=5;
|
||
* картинка: /wordpress/wp-content/images/cardlist/bt01/BT01_001EN.png.
|
||
|
||
Пишет тот же формат, что `tools/import_cards.py`: ``content/sets/{SET}.yaml``
|
||
(``set`` / ``cards`` / ``printings``), и складывает арты в ``CardArt`` как
|
||
``{imageKey}.png``, не трогая уже существующие файлы.
|
||
|
||
Примеры:
|
||
|
||
python tools/import_official.py --list-sets
|
||
python tools/import_official.py --sets BT01,TD01,TD02
|
||
python tools/import_official.py --sets BT01 --no-art --dry-run
|
||
python tools/import_official.py --sets TD01 --out .tmp/sets --art-dir .tmp/art
|
||
|
||
Зависимостей нет — только стандартная библиотека.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import html as html_mod
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
BASE = "https://en.cf-vanguard.com"
|
||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/126 Safari/537.36"
|
||
|
||
DEFAULT_SETS = "BT01,TD01,TD02"
|
||
|
||
# Триггеры базовой эры; порядок важен для разбора поля gift.
|
||
TRIGGERS = ("Critical", "Draw", "Stand", "Heal")
|
||
|
||
# Выгрузка Fandom и старая версия скрипта использовали эти же имена.
|
||
# Официальный сайт иногда теряет маркер AUTO у Dragonic Overlord.
|
||
TEXT_FIXES = {"Own(VC/RC)": "[AUTO](VC/RC)"}
|
||
|
||
|
||
# ---------------------------------------------------------------- сеть
|
||
|
||
|
||
def fetch(url: str, retries: int = 3, timeout: int = 60) -> str:
|
||
last: Exception | None = None
|
||
for attempt in range(retries):
|
||
try:
|
||
request = urllib.request.Request(url, headers={"User-Agent": UA})
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
return response.read().decode("utf-8", errors="replace")
|
||
except (urllib.error.URLError, TimeoutError) as exc: # noqa: PERF203
|
||
last = exc
|
||
time.sleep(1.5 * (attempt + 1))
|
||
raise RuntimeError(f"не удалось загрузить {url}: {last}")
|
||
|
||
|
||
def fetch_bytes(url: str, retries: int = 3, timeout: int = 90) -> bytes:
|
||
last: Exception | None = None
|
||
for attempt in range(retries):
|
||
try:
|
||
request = urllib.request.Request(url, headers={"User-Agent": UA})
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
return response.read()
|
||
except (urllib.error.URLError, TimeoutError) as exc: # noqa: PERF203
|
||
last = exc
|
||
time.sleep(1.5 * (attempt + 1))
|
||
raise RuntimeError(f"не удалось скачать {url}: {last}")
|
||
|
||
|
||
def discover_sets() -> dict[str, int]:
|
||
"""Код сета -> id раскрытия, по странице выбора продуктов."""
|
||
page = fetch(BASE + "/cardlist/?change_ex")
|
||
pattern = re.compile(r'expansion=(\d+)"[\s\S]{0,300}?alt="\[([A-Z0-9]+)\]')
|
||
result: dict[str, int] = {}
|
||
for match in pattern.finditer(page):
|
||
code = match.group(2)
|
||
if code not in result:
|
||
result[code] = int(match.group(1))
|
||
return result
|
||
|
||
|
||
def max_page(page: str) -> int:
|
||
match = re.search(r"max_page = (\d+)", page)
|
||
return int(match.group(1)) if match else 1
|
||
|
||
|
||
def results_count(page: str) -> int | None:
|
||
match = re.search(r"([\d,]+)\s+Results", page)
|
||
return int(match.group(1).replace(",", "")) if match else None
|
||
|
||
|
||
def list_codes(expansion: int) -> list[str]:
|
||
"""Все коды принтов сета в порядке выдачи сайта."""
|
||
first = fetch(f"{BASE}/cardlist/cardsearch/?expansion={expansion}&view=text&sort=no&page=1")
|
||
pages = max_page(first)
|
||
codes: list[str] = []
|
||
seen: set[str] = set()
|
||
|
||
def collect(page: str) -> None:
|
||
for code in re.findall(r'cardno=([A-Z0-9]+/[A-Z0-9]+)(?:&|")', page):
|
||
if code not in seen:
|
||
seen.add(code)
|
||
codes.append(code)
|
||
|
||
collect(first)
|
||
for page_no in range(2, pages + 1):
|
||
collect(fetch(
|
||
f"{BASE}/cardlist/cardsearch/?expansion={expansion}"
|
||
f"&view=text&sort=no&page={page_no}"
|
||
))
|
||
|
||
expected = results_count(first)
|
||
if expected is not None and expected != len(codes):
|
||
print(
|
||
f" ! ожидалось {expected} карт, собрано {len(codes)}",
|
||
file=sys.stderr,
|
||
)
|
||
return codes
|
||
|
||
|
||
# ---------------------------------------------------------------- разбор
|
||
|
||
|
||
def strip_tags(raw: str) -> str:
|
||
value = re.sub(r"<br\s*/?>", "\n", raw)
|
||
value = re.sub(r"<[^>]+>", "", value)
|
||
value = html_mod.unescape(value)
|
||
value = value.replace("\\<", "<").replace("\\>", ">")
|
||
value = re.sub(r"[ \t]+", " ", value)
|
||
return value.strip()
|
||
|
||
|
||
def field(page: str, css_class: str, tag: str = "div") -> str | None:
|
||
match = re.search(
|
||
r'<%s class="%s">([\s\S]*?)</%s>' % (tag, re.escape(css_class), tag), page
|
||
)
|
||
if not match:
|
||
return None
|
||
value = strip_tags(match.group(1))
|
||
return None if value in ("-", "—", "") else value
|
||
|
||
|
||
def number_or(value: str | None, default: int = 0) -> int:
|
||
if not value:
|
||
return default
|
||
digits = re.sub(r"[^\d]", "", value)
|
||
return int(digits) if digits else default
|
||
|
||
|
||
def parse_trigger(gift: str | None) -> str:
|
||
if not gift:
|
||
return "None"
|
||
for trigger in TRIGGERS:
|
||
if gift.startswith(trigger):
|
||
return trigger
|
||
return "None"
|
||
|
||
|
||
def slug(name: str) -> str:
|
||
value = name.lower().replace("&", " and ")
|
||
value = re.sub(r"[^a-z0-9]+", "-", value)
|
||
return value.strip("-")
|
||
|
||
|
||
def parse_card(page: str) -> dict:
|
||
"""Параметры карты из страницы /cardlist/?cardno=..."""
|
||
name = field(page, "face", tag="span")
|
||
if not name:
|
||
raise ValueError("не найдено имя карты")
|
||
|
||
kind = field(page, "type") or ""
|
||
clan = (field(page, "group") or "").replace(" ", "")
|
||
text = field(page, "effect")
|
||
if text:
|
||
for source, target in TEXT_FIXES.items():
|
||
text = text.replace(source, target)
|
||
|
||
return {
|
||
"name": name,
|
||
"type": "TriggerUnit" if kind.lower().startswith("trigger") else "NormalUnit",
|
||
"clan": clan,
|
||
"grade": number_or(field(page, "grade")),
|
||
"power": number_or(field(page, "power")),
|
||
"critical": number_or(field(page, "critical"), 1),
|
||
"shield": number_or(field(page, "shield")),
|
||
"trigger": parse_trigger(field(page, "gift")),
|
||
"rarity": field(page, "rarity") or "",
|
||
"race": field(page, "race") or "",
|
||
"nation": field(page, "nation") or "",
|
||
"skill": field(page, "skill") or "",
|
||
"flavor": field(page, "flavor"),
|
||
"text": text,
|
||
}
|
||
|
||
|
||
def parse_detail(expansion: int, code: str) -> dict:
|
||
page = fetch(f"{BASE}/cardlist/?cardno={code}&expansion={expansion}")
|
||
card = parse_card(page)
|
||
number = field(page, "number") or code
|
||
if number != code:
|
||
print(f" ! {code}: сайт вернул номер {number}", file=sys.stderr)
|
||
card["code"] = number
|
||
return card
|
||
|
||
|
||
# ---------------------------------------------------------------- арт
|
||
|
||
|
||
def art_url(code: str) -> str:
|
||
set_code, rest = code.split("/", 1)
|
||
return (
|
||
f"{BASE}/wordpress/wp-content/images/cardlist/"
|
||
f"{set_code.lower()}/{set_code}_{rest}.png"
|
||
)
|
||
|
||
|
||
def image_key(printing_id: str) -> str:
|
||
return printing_id.replace("/", "_").replace("EN", "").lower()
|
||
|
||
|
||
def existing_art(art_dir: Path, key: str) -> Path | None:
|
||
for extension in (".png", ".jpg", ".jpeg"):
|
||
candidate = art_dir / (key + extension)
|
||
if candidate.exists():
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def download_art(code: str, art_dir: Path, force: bool = False) -> str:
|
||
key = image_key(code)
|
||
existing = existing_art(art_dir, key)
|
||
if existing is not None and not force:
|
||
return "skip"
|
||
|
||
# Сначала качаем: при сетевой ошибке старый арт остаётся на месте.
|
||
data = fetch_bytes(art_url(code))
|
||
art_dir.mkdir(parents=True, exist_ok=True)
|
||
target = art_dir / f"{key}.png"
|
||
|
||
# --force-art: убираем старый файл другого расширения и его .meta, иначе
|
||
# в Resources окажется два ассета с одним именем (Unity считает это ошибкой).
|
||
if existing is not None and existing != target and existing.exists():
|
||
Path(str(existing) + ".meta").unlink(missing_ok=True)
|
||
existing.unlink()
|
||
|
||
target.write_bytes(data)
|
||
return f"{len(data) // 1024} KiB"
|
||
|
||
|
||
# ---------------------------------------------------------------- YAML
|
||
|
||
|
||
def yaml_str(value: str) -> str:
|
||
escaped = (
|
||
value.replace("\\", "\\\\")
|
||
.replace('"', '\\"')
|
||
.replace("\n", "\\n")
|
||
.replace("\t", "\\t")
|
||
)
|
||
return f'"{escaped}"'
|
||
|
||
|
||
# Поля, которых нет на сайте: их нельзя терять при повторном импорте.
|
||
MANUAL_FIELDS = ("nameRu", "textRu", "snipeBackRow", "boost")
|
||
|
||
|
||
def read_manual(path: Path) -> dict[str, dict[str, str]]:
|
||
"""Ручные поля из уже существующего YAML сета, по nameId."""
|
||
result: dict[str, dict[str, str]] = {}
|
||
if not path.exists():
|
||
return result
|
||
current: str | None = None
|
||
for raw in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||
line = raw.strip()
|
||
if line.startswith("- nameId:"):
|
||
current = line.split(":", 1)[1].strip()
|
||
result[current] = {}
|
||
continue
|
||
if line == "printings:":
|
||
current = None
|
||
continue
|
||
if current and ":" in line:
|
||
key, value = line.split(":", 1)
|
||
key = key.strip()
|
||
if key in MANUAL_FIELDS:
|
||
result[current][key] = value.strip()
|
||
return result
|
||
|
||
|
||
def existing_name_ids(out_dir: Path, exclude: set[str]) -> set[str]:
|
||
"""nameId, уже определённые в других YAML-сетах этой папки."""
|
||
names: set[str] = set()
|
||
if not out_dir.exists():
|
||
return names
|
||
for path in out_dir.glob("*.yaml"):
|
||
if path.stem in exclude:
|
||
continue
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
names.update(re.findall(r"^\s*-\s*nameId:\s*(\S+)", text, re.M))
|
||
return names
|
||
|
||
|
||
def write_set(
|
||
path: Path,
|
||
set_id: str,
|
||
cards: list[dict],
|
||
printings: list[dict],
|
||
manual: dict[str, dict[str, str]] | None = None,
|
||
) -> None:
|
||
manual = manual or {}
|
||
with path.open("w", encoding="utf-8", newline="\n") as handle:
|
||
handle.write(f"set: {set_id}\n\ncards:\n")
|
||
for card in cards:
|
||
handle.write(f" - nameId: {card['nameId']}\n")
|
||
handle.write(f" nameEn: {yaml_str(card['nameEn'])}\n")
|
||
handle.write(f" type: {card['type']}\n")
|
||
handle.write(f" grade: {card['grade']}\n")
|
||
handle.write(f" power: {card['power']}\n")
|
||
handle.write(f" shield: {card['shield']}\n")
|
||
handle.write(f" critical: {card['critical']}\n")
|
||
handle.write(f" trigger: {card['trigger']}\n")
|
||
handle.write(f" clan: {card['clan']}\n")
|
||
extra = manual.get(card["nameId"], {})
|
||
for key in MANUAL_FIELDS:
|
||
if key in extra:
|
||
handle.write(f" {key}: {extra[key]}\n")
|
||
if card["textEn"]:
|
||
handle.write(f" textEn: {yaml_str(card['textEn'])}\n")
|
||
if card["sentinel"]:
|
||
handle.write(" sentinel: true\n")
|
||
if card["restraint"]:
|
||
handle.write(" restraint: true\n")
|
||
handle.write("\nprintings:\n")
|
||
for printing in printings:
|
||
handle.write(f" - printingId: {printing['printingId']}\n")
|
||
handle.write(f" nameId: {printing['nameId']}\n")
|
||
handle.write(f" rarity: {printing['rarity']}\n")
|
||
if printing.get("imageKey"):
|
||
handle.write(f" imageKey: {printing['imageKey']}\n")
|
||
if printing.get("isParallel"):
|
||
handle.write(" isParallel: true\n")
|
||
handle.write(f" basePrintingId: {printing['basePrintingId']}\n")
|
||
|
||
|
||
# ---------------------------------------------------------------- сборка
|
||
|
||
|
||
def build_set(set_id: str, expansion: int, skip_defs: set[str], args) -> tuple[list[dict], list[dict]]:
|
||
codes = list_codes(expansion)
|
||
if args.limit:
|
||
codes = codes[: args.limit]
|
||
print(f"{set_id} (expansion={expansion}): {len(codes)} принтов")
|
||
|
||
rows: list[dict] = []
|
||
for index, code in enumerate(codes, 1):
|
||
card = parse_detail(expansion, code)
|
||
card["code"] = code
|
||
card["nameId"] = slug(card["name"])
|
||
card["nameEn"] = card["name"]
|
||
card["textEn"] = card["text"]
|
||
card["sentinel"] = bool(card["text"] and "Sentinel" in card["text"])
|
||
card["restraint"] = bool(card["text"] and "Restraint" in card["text"])
|
||
card["imageKey"] = image_key(code)
|
||
rows.append(card)
|
||
print(f" [{index}/{len(codes)}] {code} {card['name']} "
|
||
f"G{card['grade']} {card['trigger']} {card['rarity']}")
|
||
if args.delay:
|
||
time.sleep(args.delay)
|
||
|
||
# арт
|
||
if not args.no_art:
|
||
downloaded = 0
|
||
for card in rows:
|
||
try:
|
||
status = download_art(card["code"], args.art_dir, args.force_art)
|
||
except Exception as exc: # noqa: BLE001
|
||
print(f" ! арт {card['code']}: {exc}", file=sys.stderr)
|
||
continue
|
||
if status != "skip":
|
||
downloaded += 1
|
||
print(f" art {card['code']} -> {card['imageKey']}.png ({status})")
|
||
print(f" арт: скачано {downloaded}, уже было {len(rows) - downloaded}")
|
||
|
||
# определения без повторов между сетами
|
||
definitions: dict[str, dict] = {}
|
||
for card in rows:
|
||
definitions.setdefault(card["nameId"], card)
|
||
definitions = {k: v for k, v in definitions.items() if k not in skip_defs}
|
||
skip_defs.update(definitions)
|
||
|
||
# параллельные принты (BT01/S01EN и т.п.)
|
||
base_by_name = {
|
||
row["nameId"]: row["code"] for row in rows if "/S" not in row["code"]
|
||
}
|
||
printings: list[dict] = []
|
||
for row in rows:
|
||
entry = {
|
||
"printingId": row["code"],
|
||
"nameId": row["nameId"],
|
||
"rarity": row["rarity"] or "C",
|
||
"imageKey": row["imageKey"],
|
||
}
|
||
if "/S" in row["code"]:
|
||
entry["isParallel"] = True
|
||
entry["basePrintingId"] = base_by_name.get(row["nameId"])
|
||
entry["rarity"] = "SP"
|
||
printings.append(entry)
|
||
|
||
for card in definitions.values():
|
||
if card["clan"] and not re.fullmatch(r"[A-Za-z]+", card["clan"]):
|
||
print(f" ! клан без маппинга: {card['clan']}", file=sys.stderr)
|
||
|
||
return list(definitions.values()), printings
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--sets", default=DEFAULT_SETS, help="коды сетов через запятую")
|
||
parser.add_argument("--out", default=str(ROOT / "content" / "sets"), help="папка YAML")
|
||
parser.add_argument(
|
||
"--art-dir",
|
||
default=str(ROOT / "client" / "Assets" / "Vanguard" / "Resources" / "CardArt"),
|
||
help="папка артов",
|
||
)
|
||
parser.add_argument("--no-art", action="store_true", help="не скачивать арты")
|
||
parser.add_argument("--force-art", action="store_true", help="перезаписать арты")
|
||
parser.add_argument("--dry-run", action="store_true", help="ничего не писать")
|
||
parser.add_argument("--list-sets", action="store_true", help="показать сеты и выйти")
|
||
parser.add_argument("--delay", type=float, default=0.25, help="пауза между запросами")
|
||
parser.add_argument("--limit", type=int, default=0, help="только первые N карт сета")
|
||
args = parser.parse_args()
|
||
|
||
args.out = Path(args.out)
|
||
args.art_dir = Path(args.art_dir)
|
||
|
||
mapping = discover_sets()
|
||
if args.list_sets:
|
||
for code, expansion in sorted(mapping.items(), key=lambda item: item[1], reverse=True):
|
||
print(f"{code} = {expansion}")
|
||
return 0
|
||
|
||
sets = [value.strip().upper() for value in args.sets.split(",") if value.strip()]
|
||
unknown = [value for value in sets if value not in mapping]
|
||
if unknown:
|
||
print(f"неизвестные сеты: {', '.join(unknown)}", file=sys.stderr)
|
||
print("доступные: " + ", ".join(sorted(mapping)), file=sys.stderr)
|
||
return 2
|
||
|
||
if not args.dry_run:
|
||
args.out.mkdir(parents=True, exist_ok=True)
|
||
|
||
skip_defs: set[str] = set()
|
||
if not args.dry_run:
|
||
# при запуске только одного сета не дублировать определения из других файлов
|
||
skip_defs.update(existing_name_ids(args.out, set(sets)))
|
||
total_cards = 0
|
||
total_printings = 0
|
||
|
||
for set_id in sets:
|
||
path = args.out / f"{set_id}.yaml"
|
||
manual = read_manual(path)
|
||
cards, printings = build_set(set_id, mapping[set_id], skip_defs, args)
|
||
total_cards += len(cards)
|
||
total_printings += len(printings)
|
||
if args.dry_run:
|
||
print(f" dry-run: {path} ({len(cards)} определений, {len(printings)} принтов)")
|
||
continue
|
||
write_set(path, set_id, cards, printings, manual)
|
||
print(f" записано {path}: {len(cards)} определений, {len(printings)} принтов")
|
||
|
||
print(f"ИТОГО: {total_cards} определений, {total_printings} принтов")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|