import asyncio import logging import math import re import threading import weakref from collections.abc import Awaitable, Callable from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta from decimal import Decimal, InvalidOperation from html import unescape from typing import Any from zoneinfo import ZoneInfo import httpx from .lunar_support import solar_to_lunar_text SHANGHAI_TZ = ZoneInfo("Asia/Shanghai") WEATHER_URL = "https://api.open-meteo.com/v1/forecast" GOLD_URL = "https://www.sge.com.cn/sjzx/yshqbg" GOLD_HISTORY_URL = ( "https://www.sge.com.cn/sjzx/quotation_daily_new" "?start_date={start_date}&end_date={end_date}" ) AIR_URL = "https://air-quality-api.open-meteo.com/v1/air-quality" OIL_BASE_URL = "https://cx.sinopecsales.com" OIL_MAIN_PATH = "/yjkqiantai/core/main" OIL_SWITCH_PATH = "/yjkqiantai/data/switchProvince" OIL_INIT_PATH = "/yjkqiantai/data/initMainData" ASHARE_URL = "https://hq.sinajs.cn/list=sh000001" WEATHER_SOURCE = "Open-Meteo" GOLD_SOURCE = "上海黄金交易所" AIR_SOURCE = "Open-Meteo 空气质量" OIL_SOURCE = "中国石化" ASHARE_SOURCE = "新浪财经" ASHARE_HEADERS = {"Referer": "https://finance.sina.com.cn", "User-Agent": "Mozilla/5.0"} REQUEST_HEADERS = {"User-Agent": "dodo/0.1 (+https://dodo.bboy.app)"} WEEKDAYS = ("星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日") SOURCE_TIMEOUT_SECONDS = 3.0 GOLD_SOURCE_TIMEOUT_SECONDS = 6.0 OIL_SOURCE_TIMEOUT_SECONDS = 6.0 TOTAL_TIMEOUT_SECONDS = 6.5 WEATHER_TTL = timedelta(minutes=15) WEATHER_STALE_TTL = timedelta(hours=6) GOLD_TTL = timedelta(minutes=5) GOLD_STALE_TTL = timedelta(days=7) AIR_TTL = timedelta(hours=1) AIR_STALE_TTL = timedelta(days=3) OIL_TTL = timedelta(hours=6) OIL_STALE_TTL = timedelta(days=30) ASHARE_TTL = timedelta(minutes=10) ASHARE_STALE_TTL = timedelta(days=7) logger = logging.getLogger(__name__) _TAG_RE = re.compile(r"<[^>]+>") _DATE_RE = re.compile(r"上海黄金交易所\s*(\d{4})年(\d{2})月(\d{2})日\s*延时行情") _ROW_RE = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) _CELL_RE = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) @dataclass class CacheEntry: value: dict[str, Any] fetched_at: datetime @dataclass class LoopRuntime: inflight: dict[tuple[int, str], asyncio.Task[dict[str, Any]]] _cache: dict[str, CacheEntry] = {} _generation = 0 _state_guard = threading.Lock() _runtimes: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, LoopRuntime] = ( weakref.WeakKeyDictionary() ) def _runtime(loop: asyncio.AbstractEventLoop | None = None) -> LoopRuntime: active_loop = loop or asyncio.get_running_loop() with _state_guard: runtime = _runtimes.get(active_loop) if runtime is None: runtime = LoopRuntime(inflight={}) _runtimes[active_loop] = runtime return runtime def clear_cache() -> None: global _generation with _state_guard: _generation += 1 _cache.clear() def _text(fragment: str) -> str: return " ".join(unescape(_TAG_RE.sub(" ", fragment)).split()) def parse_sge_au9999(html: str) -> tuple[Decimal, str]: date_match = _DATE_RE.search(_text(html)) if date_match is None: raise ValueError("SGE market date not found") try: year, month, day = map(int, date_match.groups()) market_date = date(year, month, day).isoformat() except ValueError as exc: raise ValueError("invalid SGE market date") from exc rows = [[_text(cell) for cell in _CELL_RE.findall(row)] for row in _ROW_RE.findall(html)] header = next((cells for cells in rows if "合约" in cells and "最新价" in cells), None) if header is None: raise ValueError("SGE quote table header not found") contract_index = header.index("合约") price_index = header.index("最新价") required_length = max(contract_index, price_index) + 1 for cells in rows[rows.index(header) + 1 :]: if len(cells) < required_length or cells[contract_index] != "Au99.99": continue try: price = Decimal(cells[price_index].replace(",", "")) except InvalidOperation as exc: raise ValueError("invalid Au99.99 latest price") from exc if not price.is_finite() or price <= 0: raise ValueError("invalid Au99.99 latest price") return price, market_date raise ValueError("Au99.99 quote not found") def parse_sge_au9999_history(html: str) -> tuple[Decimal, str]: rows = [[_text(cell) for cell in _CELL_RE.findall(row)] for row in _ROW_RE.findall(html)] header = next( (cells for cells in rows if "日期" in cells and "合约" in cells and "收盘价" in cells), None, ) if header is None: raise ValueError("SGE history table header not found") date_index = header.index("日期") contract_index = header.index("合约") price_index = header.index("收盘价") required_length = max(date_index, contract_index, price_index) + 1 quotes = [] for cells in rows[rows.index(header) + 1 :]: if len(cells) < required_length or cells[contract_index] != "Au99.99": continue try: market_date = date.fromisoformat(cells[date_index]).isoformat() price = Decimal(cells[price_index].replace(",", "")) except (InvalidOperation, ValueError): continue if not price.is_finite() or price <= 0: continue quotes.append((market_date, price)) if not quotes: raise ValueError("Au99.99 history quote not found") market_date, price = max(quotes) return price, market_date async def fetch_weather(request: Callable[..., Awaitable[dict[str, Any]]]) -> dict[str, Any]: payload = await request( WEATHER_URL, params={ "latitude": 29.88, "longitude": 121.55, "current": "temperature_2m,apparent_temperature,weather_code", "hourly": "precipitation_probability", "timezone": "Asia/Shanghai", }, ) if payload.get("timezone") != "Asia/Shanghai": raise ValueError("weather timezone must be Asia/Shanghai") current = payload["current"] if not isinstance(current, dict): raise TypeError("weather current must be an object") def finite_number(name: str, minimum: float, maximum: float) -> int | float: value = current[name] if isinstance(value, bool) or not isinstance(value, (int, float)): raise TypeError(f"weather {name} must be numeric") if not math.isfinite(value) or not minimum <= value <= maximum: raise ValueError(f"weather {name} out of range") return value temperature = finite_number("temperature_2m", -100, 100) apparent_temperature = finite_number("apparent_temperature", -100, 100) weather_code = current["weather_code"] if isinstance(weather_code, bool) or not isinstance(weather_code, int): raise TypeError("weather weather_code must be an integer") if not 0 <= weather_code <= 99: raise ValueError("weather weather_code out of range") raw_time = current["time"] if not isinstance(raw_time, str): raise TypeError("weather time must be a string") observed_at = datetime.fromisoformat(raw_time) if observed_at.tzinfo is not None: if observed_at.utcoffset() != SHANGHAI_TZ.utcoffset(observed_at): raise ValueError("weather time must use Asia/Shanghai") observed_at = observed_at.astimezone(SHANGHAI_TZ) else: observed_at = observed_at.replace(tzinfo=SHANGHAI_TZ) precipitation_probability = _precipitation_probability(payload, observed_at) return { "temperature_c": temperature, "apparent_temperature_c": apparent_temperature, "weather_code": weather_code, "precipitation_probability": precipitation_probability, "observed_at": observed_at.isoformat(), "source": WEATHER_SOURCE, } def _precipitation_probability(payload: dict[str, Any], observed_at: datetime) -> int | None: hourly = payload.get("hourly") if not isinstance(hourly, dict): raise TypeError("weather hourly must be an object") times = hourly.get("time") probabilities = hourly.get("precipitation_probability") if not isinstance(times, list) or not isinstance(probabilities, list): raise TypeError("weather hourly precipitation_probability must be a list") if len(times) != len(probabilities) or not times or not all(isinstance(value, str) for value in times): raise ValueError("weather hourly precipitation_probability must align with time") current_hour = observed_at.strftime("%Y-%m-%dT%H:00") try: index = times.index(current_hour) except ValueError: return None window = probabilities[index : index + 3] valid: list[float] = [] for value in window: if value is None: continue if isinstance(value, bool) or not isinstance(value, (int, float)): raise TypeError("weather precipitation_probability must be numeric or null") if not math.isfinite(value) or not 0 <= value <= 100: raise ValueError("weather precipitation_probability out of range") valid.append(float(value)) return round(max(valid)) if valid else None async def fetch_gold(request: Callable[..., Awaitable[str]]) -> dict[str, Any]: html = await request(GOLD_URL) try: price, market_date = parse_sge_au9999(html) except ValueError: date_match = _DATE_RE.search(_text(html)) if date_match is None: raise try: quote_date = date(*map(int, date_match.groups())) except ValueError as exc: raise ValueError("invalid SGE market date") from exc start_date = quote_date - timedelta(days=31) history_url = GOLD_HISTORY_URL.format( start_date=start_date.isoformat(), end_date=quote_date.isoformat() ) price, market_date = parse_sge_au9999_history(await request(history_url)) return { "contract": "Au99.99", "latest_price": price, "currency": "CNY", "unit": "gram", "market_date": market_date, "delayed": True, "source": GOLD_SOURCE, } _PM25_BREAKPOINTS = ( (0.0, 35.0, 0.0, 50.0), (35.0, 75.0, 50.0, 100.0), (75.0, 115.0, 100.0, 150.0), (115.0, 150.0, 150.0, 200.0), (150.0, 250.0, 200.0, 300.0), (250.0, 350.0, 300.0, 400.0), (350.0, 500.0, 400.0, 500.0), ) _PM10_BREAKPOINTS = ( (0.0, 50.0, 0.0, 50.0), (50.0, 150.0, 50.0, 100.0), (150.0, 250.0, 100.0, 150.0), (250.0, 350.0, 150.0, 200.0), (350.0, 420.0, 200.0, 300.0), (420.0, 500.0, 300.0, 400.0), (500.0, 600.0, 400.0, 500.0), ) def _sub_index(concentration: float, breakpoints: tuple[tuple[float, float, float, float], ...]) -> float: if concentration <= 0: return 0.0 for low, high, index_low, index_high in breakpoints: if concentration <= high: return (index_high - index_low) / (high - low) * (concentration - low) + index_low return 500.0 def china_aqi(pm2_5: float, pm10: float) -> int: """HJ 633-2012 AQI from PM2.5 / PM10 concentrations (μg/m³), capped at 500.""" return round(max(_sub_index(pm2_5, _PM25_BREAKPOINTS), _sub_index(pm10, _PM10_BREAKPOINTS))) def aqi_level(aqi: int) -> str: if aqi <= 50: return "优" if aqi <= 100: return "良" if aqi <= 150: return "轻度污染" if aqi <= 200: return "中度污染" if aqi <= 300: return "重度污染" return "严重污染" async def fetch_air(request: Callable[..., Awaitable[dict[str, Any]]]) -> dict[str, Any]: payload = await request( AIR_URL, params={ "latitude": 29.88, "longitude": 121.55, "current": "pm2_5,pm10", "timezone": "Asia/Shanghai", }, ) if payload.get("timezone") != "Asia/Shanghai": raise ValueError("air quality timezone must be Asia/Shanghai") current = payload["current"] if not isinstance(current, dict): raise TypeError("air quality current must be an object") def concentration(name: str) -> float: value = current[name] if isinstance(value, bool) or not isinstance(value, (int, float)): raise TypeError(f"air quality {name} must be numeric") if not math.isfinite(value) or not 0 <= value <= 2000: raise ValueError(f"air quality {name} out of range") return float(value) pm2_5 = concentration("pm2_5") pm10 = concentration("pm10") raw_time = current["time"] if not isinstance(raw_time, str): raise TypeError("air quality time must be a string") observed_at = datetime.fromisoformat(raw_time) if observed_at.tzinfo is not None: if observed_at.utcoffset() != SHANGHAI_TZ.utcoffset(observed_at): raise ValueError("air quality time must use Asia/Shanghai") observed_at = observed_at.astimezone(SHANGHAI_TZ) else: observed_at = observed_at.replace(tzinfo=SHANGHAI_TZ) aqi = china_aqi(pm2_5, pm10) return { "aqi": aqi, "level": aqi_level(aqi), "pm2_5": pm2_5, "pm10": pm10, "observed_at": observed_at.isoformat(), "source": AIR_SOURCE, } async def _request_json(client: httpx.AsyncClient, url: str, **kwargs: Any) -> dict[str, Any]: response = await client.get(url, **kwargs) response.raise_for_status() return response.json() async def _request_text(client: httpx.AsyncClient, url: str, **kwargs: Any) -> str: response = await client.get(url, **kwargs) response.raise_for_status() return response.text async def _request_bytes(client: httpx.AsyncClient, url: str, **kwargs: Any) -> bytes: response = await client.get(url, **kwargs) response.raise_for_status() return response.content def parse_sina_index(text: str) -> dict[str, Any]: match = re.search(r'"([^"]*)"', text) if match is None: raise ValueError("sina quote not found") fields = match.group(1).split(",") if len(fields) < 5: raise ValueError("sina quote is too short") if fields[0] != "上证指数": raise ValueError("unexpected sina symbol") def number(index: int) -> Decimal: try: value = Decimal(fields[index].replace(",", "")) except InvalidOperation as exc: raise ValueError("invalid sina number") from exc if not value.is_finite() or value <= 0: raise ValueError("invalid sina number") return value previous_close = number(2) price = number(3) as_of = next((field for field in fields if re.fullmatch(r"\d{4}-\d{2}-\d{2}", field)), None) if as_of is None: raise ValueError("sina quote date not found") try: date.fromisoformat(as_of) except ValueError as exc: raise ValueError("invalid sina quote date") from exc change_percent = float((price - previous_close) / previous_close * 100) return { "name": fields[0], "price": price, "prev_close": previous_close, "change_percent": round(change_percent, 2), "as_of": as_of, "source": ASHARE_SOURCE, } async def fetch_ashare(request: Callable[..., Awaitable[bytes]]) -> dict[str, Any]: raw = await request(ASHARE_URL, headers=ASHARE_HEADERS) try: text = raw.decode("gbk") except UnicodeDecodeError as exc: raise ValueError("sina quote is not decodable as GBK") from exc return parse_sina_index(text) def parse_oil_payload(payload: dict[str, Any]) -> dict[str, Any]: if not isinstance(payload, dict): raise TypeError("sinopec payload must be an object") data = payload.get("data") if not isinstance(data, dict): raise TypeError("sinopec data must be an object") province = data.get("provinceData") if not isinstance(province, dict): raise TypeError("sinopec provinceData must be an object") def price(key: str) -> Decimal: if key not in province: raise ValueError(f"sinopec {key} missing") try: value = Decimal(str(province[key])) except InvalidOperation as exc: raise ValueError(f"sinopec {key} invalid") from exc if not value.is_finite() or not Decimal("0.01") <= value <= Decimal(100): raise ValueError(f"sinopec {key} out of range") return value effective_at = None raw_effective = province.get("START_DATE") if isinstance(raw_effective, str) and raw_effective: try: parsed = datetime.strptime(raw_effective, "%Y-%m-%d %H:%M:%S").replace(tzinfo=SHANGHAI_TZ) except ValueError: effective_at = None else: effective_at = parsed.isoformat() return { "gas_92": price("GAS_92"), "gas_95": price("GAS_95"), "gas_98": price("AIPAO_GAS_98"), "diesel_0": price("CHECHAI_0"), "effective_at": effective_at, "source": OIL_SOURCE, } async def fetch_oil(client: httpx.AsyncClient) -> dict[str, Any]: main = await client.get(f"{OIL_BASE_URL}{OIL_MAIN_PATH}") main.raise_for_status() if client.cookies.get("SESSION") is None: raise ValueError("sinopec session cookie missing") switch = await client.post(f"{OIL_BASE_URL}{OIL_SWITCH_PATH}", json={"provinceId": "33"}) switch.raise_for_status() init = await client.get( f"{OIL_BASE_URL}{OIL_INIT_PATH}", headers={"Referer": f"{OIL_BASE_URL}{OIL_MAIN_PATH}"}, ) init.raise_for_status() return parse_oil_payload(init.json()) def _serializable(value: dict[str, Any]) -> dict[str, Any]: return {key: str(item) if isinstance(item, Decimal) else item for key, item in value.items()} def _source_timeout(name: str) -> float: if name == "gold": return GOLD_SOURCE_TIMEOUT_SECONDS if name == "oil": return OIL_SOURCE_TIMEOUT_SECONDS return SOURCE_TIMEOUT_SECONDS async def _refresh_source( name: str, fetcher: Callable[[], Awaitable[dict[str, Any]]], fetched_at: datetime, generation: int, ) -> dict[str, Any]: value = await asyncio.wait_for(fetcher(), timeout=_source_timeout(name)) with _state_guard: if generation == _generation: newest = _cache.get(name) if newest is None or newest.fetched_at <= fetched_at: _cache[name] = CacheEntry(value=value, fetched_at=fetched_at) return value def _source_error_code(exc: BaseException) -> str: if isinstance(exc, TimeoutError): return "timeout" if isinstance(exc, (KeyError, TypeError, ValueError, InvalidOperation)): return "invalid_upstream_response" return "upstream_unavailable" def _finish_inflight( runtime: LoopRuntime, key: tuple[int, str], task: asyncio.Task[dict[str, Any]], ) -> None: # Calling exception() marks failures as retrieved even when every shielded waiter # was cancelled. This synchronous callback cannot create an untracked cleanup task. if not task.cancelled(): try: task.exception() except asyncio.CancelledError: pass if runtime.inflight.get(key) is task: runtime.inflight.pop(key, None) async def _cached_source( name: str, fetcher: Callable[[], Awaitable[dict[str, Any]]], now: datetime, fresh_for: timedelta, stale_for: timedelta, ) -> tuple[dict[str, Any] | None, str | None]: runtime = _runtime() with _state_guard: generation = _generation key = (generation, name) with _state_guard: cached = _cache.get(name) if generation == _generation else None if cached is not None and now - cached.fetched_at <= fresh_for: return {**_serializable(cached.value), "stale": False}, None task = runtime.inflight.get(key) if task is None: task = asyncio.create_task(_refresh_source(name, fetcher, now, generation)) runtime.inflight[key] = task task.add_done_callback( lambda completed, active=runtime, task_key=key: _finish_inflight( active, task_key, completed ) ) try: value = await asyncio.shield(task) except Exception as exc: error_code = _source_error_code(exc) logger.warning("Today environment source %s failed (%s)", name, error_code, exc_info=exc) with _state_guard: generation_is_current = generation == _generation if generation_is_current and cached is not None and now - cached.fetched_at <= stale_for: return {**_serializable(cached.value), "stale": True}, error_code return None, error_code return {**_serializable(value), "stale": False}, None async def _default_weather_fetcher() -> dict[str, Any]: async with httpx.AsyncClient( timeout=httpx.Timeout(SOURCE_TIMEOUT_SECONDS), headers={"User-Agent": "dodo/0.1 (+https://dodo.bboy.app)"}, ) as client: return await fetch_weather(lambda url, **kwargs: _request_json(client, url, **kwargs)) async def _default_gold_fetcher() -> dict[str, Any]: async with httpx.AsyncClient( timeout=httpx.Timeout(SOURCE_TIMEOUT_SECONDS), headers={"User-Agent": "dodo/0.1 (+https://dodo.bboy.app)"}, ) as client: return await fetch_gold(lambda url, **kwargs: _request_text(client, url, **kwargs)) async def _default_air_fetcher() -> dict[str, Any]: async with httpx.AsyncClient( timeout=httpx.Timeout(SOURCE_TIMEOUT_SECONDS), headers=REQUEST_HEADERS, ) as client: return await fetch_air(lambda url, **kwargs: _request_json(client, url, **kwargs)) async def _default_oil_fetcher() -> dict[str, Any]: async with httpx.AsyncClient( timeout=httpx.Timeout(OIL_SOURCE_TIMEOUT_SECONDS), headers={"User-Agent": "Mozilla/5.0"}, ) as client: return await fetch_oil(client) async def _default_ashare_fetcher() -> dict[str, Any]: async with httpx.AsyncClient( timeout=httpx.Timeout(SOURCE_TIMEOUT_SECONDS), headers=REQUEST_HEADERS, ) as client: return await fetch_ashare(lambda url, **kwargs: _request_bytes(client, url, **kwargs)) async def get_environment( *, now: datetime | None = None, weather_fetcher: Callable[[], Awaitable[dict[str, Any]]] | None = None, gold_fetcher: Callable[[], Awaitable[dict[str, Any]]] | None = None, ) -> dict[str, Any]: current = now or datetime.now(UTC) local_date = current.astimezone(SHANGHAI_TZ).date() weather_task = _cached_source( "weather", weather_fetcher or _default_weather_fetcher, current, WEATHER_TTL, WEATHER_STALE_TTL ) gold_task = _cached_source( "gold", gold_fetcher or _default_gold_fetcher, current, GOLD_TTL, GOLD_STALE_TTL ) weather_result, gold_result = await asyncio.wait_for( asyncio.gather(weather_task, gold_task), timeout=TOTAL_TIMEOUT_SECONDS ) weather, weather_error = weather_result gold, gold_error = gold_result errors = {} if weather_error: errors["weather"] = weather_error if gold_error: errors["gold"] = gold_error return { "date": { "solar_date": local_date.isoformat(), "weekday": WEEKDAYS[local_date.weekday()], "lunar": solar_to_lunar_text(local_date), "timezone": "Asia/Shanghai", }, "weather": weather, "gold": gold, "errors": errors, } async def get_environment_extras( *, now: datetime | None = None, air_fetcher: Callable[[], Awaitable[dict[str, Any]]] | None = None, oil_fetcher: Callable[[], Awaitable[dict[str, Any]]] | None = None, ashare_fetcher: Callable[[], Awaitable[dict[str, Any]]] | None = None, ) -> dict[str, Any]: """Air quality, Zhejiang oil prices, and the SSE index for the Today strip.""" current = now or datetime.now(UTC) sources = ( ("air", air_fetcher or _default_air_fetcher, AIR_TTL, AIR_STALE_TTL), ("oil", oil_fetcher or _default_oil_fetcher, OIL_TTL, OIL_STALE_TTL), ("ashare", ashare_fetcher or _default_ashare_fetcher, ASHARE_TTL, ASHARE_STALE_TTL), ) results = await asyncio.wait_for( asyncio.gather( *( _cached_source(name, fetcher, current, fresh_for, stale_for) for name, fetcher, fresh_for, stale_for in sources ) ), timeout=TOTAL_TIMEOUT_SECONDS, ) values: dict[str, Any] = {} errors: dict[str, str] = {} for (name, *_), (value, error) in zip(sources, results): values[name] = value if error: errors[name] = error return {**values, "errors": errors}