diff --git a/README.md b/README.md index 61925bc..e8c1a1d 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ api.available_category() ## Examples - [Generate a source-linked news briefing](examples/source_linked_briefing/README.md) from a live Search API response or a deterministic offline fixture. +- [Build a company news monitor](examples/company_news_monitor/README.md) with + a JSON watchlist, bounded search window, local state, and deterministic fixture. ## Authentication diff --git a/examples/company_news_monitor/README.md b/examples/company_news_monitor/README.md new file mode 100644 index 0000000..3d7f247 --- /dev/null +++ b/examples/company_news_monitor/README.md @@ -0,0 +1,115 @@ +# Company news monitor + +This example runs a small company, competitor, or industry watchlist against a +bounded Currents Search API window. It writes: + +- `report.md`, a source-linked change report for review; +- `report.json`, the same new articles as structured data; +- a local state file containing article keys seen across runs. + +Currents provides search results and article metadata. This script owns the +watchlist, date window, local state, deduplication, and report formatting. It is +not a managed monitor or alert service. + +## Run the checked-in fixture + +Install this repository's dependencies, then run: + +```bash +python examples/company_news_monitor/monitor.py \ + --fixture examples/company_news_monitor/fixtures/search_responses.json \ + --state-file company-monitor-state.json \ + --output-dir company-monitor-output +``` + +The fixture uses fictional companies and `example.com` URLs. It needs no network +request, credentials, or customer data. Its fixed timestamps make the first run +deterministic. Delete the generated state file to reproduce that first run. + +Run the same command again with the same state file. The second report contains +no new articles. + +## Configure a watchlist + +Each watch requires a unique `name` and exactly one search field. Use `keywords` +for standard search or `query` for Boolean syntax. `language` defaults to `en`. +The optional `domain` narrows that watch to one publisher domain. + +```json +{ + "watches": [ + { + "name": "Competitor names", + "query": "\"Northstar Battery\" OR \"Atlas Storage\"", + "language": "en" + }, + { + "name": "Industry policy", + "keywords": "\"grid storage\" regulation", + "language": "en", + "domain": "example.com" + } + ] +} +``` + +Use the smallest watchlist that represents a real decision. Search does not +perform entity resolution, so ambiguous company names need additional terms. + +## Run a live window + +Create a [Currents API key](https://currentsapi.services/en/register?utm_source=github&utm_medium=referral&utm_campaign=content-build-company-news-monitor-currents-search-api), +then set it in your environment: + +```bash +export CURRENTS_API_KEY="your-api-key" +``` + +Run an explicit UTC window: + +```bash +python examples/company_news_monitor/monitor.py \ + --watchlist examples/company_news_monitor/watchlist.json \ + --start-date 2026-08-04T09:00:00Z \ + --end-date 2026-08-05T09:00:00Z \ + --page-size 20 \ + --state-file company-monitor-state.json \ + --output-dir company-monitor-output +``` + +Live mode sends the watch's `keywords` or `query`, plus `language`, `start_date`, +`end_date`, `page_number`, and `page_size`. It also sends `domain` when the +watch defines one. + +Use a window and page size allowed by your Currents plan. Search windows, +lookback, page sizes, and retrievable result counts can vary by plan. + +## Deduplication and state + +The report treats the publisher URL and article ID as aliases. A match on either +alias merges the article and its watch names. The state file stores both aliases +when both are present. + +The state file prevents the same article from appearing as new on later runs. +Keep a separate state file for each monitor. Back it up if missing alerts would +matter to your application. + +## Limitations + +- The script runs once. Your application must schedule it. +- The first run treats every in-window result as new. +- One page is requested per watch. Increase coverage only within plan limits. +- Names and Boolean terms can miss aliases or match unrelated entities. +- Results reflect the Currents index and are not exhaustive. +- Currents does not verify article claims or score their truth. +- API access does not grant publisher-content republication rights. +- Add retries, locking, state recovery, observability, and human review before + using this pattern in a production workflow. + +## Test + +Run the focused offline tests: + +```bash +python -m pytest tests/test_company_news_monitor_example.py +``` diff --git a/examples/company_news_monitor/fixtures/search_responses.json b/examples/company_news_monitor/fixtures/search_responses.json new file mode 100644 index 0000000..4eac5e5 --- /dev/null +++ b/examples/company_news_monitor/fixtures/search_responses.json @@ -0,0 +1,45 @@ +{ + "_fixture_generated_at": "2026-08-05T09:00:00Z", + "_fixture_start_date": "2026-08-04T09:00:00Z", + "_fixture_end_date": "2026-08-05T09:00:00Z", + "responses": { + "Battery competitors": { + "status": "ok", + "news": [ + { + "id": "fictional-factory-1", + "title": "Northstar Battery opens a pilot recycling line", + "description": "A fictional company begins pilot operations.", + "url": "https://example.com/business/northstar-pilot-line", + "published": "2026-08-05T07:30:00Z" + }, + { + "id": "fictional-joint-project-1", + "title": "Atlas Storage joins a grid demonstration", + "description": "A fictional storage supplier joins a demonstration.", + "url": "https://example.com/energy/atlas-grid-project", + "published": "2026-08-04T14:00:00Z" + } + ] + }, + "Industry policy": { + "status": "ok", + "news": [ + { + "id": "fictional-policy-1", + "title": "Regulator publishes a grid storage consultation", + "description": "A fictional regulator opens a consultation period.", + "url": "https://example.com/policy/grid-storage-consultation", + "published": "2026-08-05T08:00:00Z" + }, + { + "id": "fictional-joint-project-1", + "title": "Atlas Storage joins a grid demonstration", + "description": "A fictional storage supplier joins a demonstration.", + "url": "https://example.com/energy/atlas-grid-project", + "published": "2026-08-04T14:00:00Z" + } + ] + } + } +} diff --git a/examples/company_news_monitor/monitor.py b/examples/company_news_monitor/monitor.py new file mode 100644 index 0000000..f7b8388 --- /dev/null +++ b/examples/company_news_monitor/monitor.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python3 +"""Build a source-linked company news change report.""" + +import argparse +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote, urlsplit + +import requests + + +SEARCH_URL = "https://api.currentsapi.services/v1/search" + + +def parse_args(): + example_dir = Path(__file__).resolve().parent + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--watchlist", + type=Path, + default=example_dir / "watchlist.json", + help="Read company and competitor searches from JSON.", + ) + parser.add_argument( + "--fixture", + type=Path, + help="Read saved Search API responses instead of calling the network.", + ) + parser.add_argument("--start-date", help="Set the inclusive RFC 3339 window start.") + parser.add_argument("--end-date", help="Set the inclusive RFC 3339 window end.") + parser.add_argument( + "--state-file", + type=Path, + default=Path("company-news-monitor-state.json"), + help="Store article keys already included in earlier reports.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("company-news-monitor-output"), + help="Write report.md and report.json here.", + ) + parser.add_argument( + "--generated-at", + help="Override the report timestamp. Fixtures provide a stable default.", + ) + parser.add_argument( + "--page-size", + type=int, + default=20, + help="Request this many results per watch in live mode.", + ) + return parser.parse_args() + + +def parse_timestamp(value, label): + if not isinstance(value, str) or not value: + raise ValueError("{} must be a non-empty RFC 3339 timestamp".format(label)) + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("{} must be an RFC 3339 timestamp".format(label)) from exc + if parsed.tzinfo is None: + raise ValueError("{} must include a timezone".format(label)) + return parsed.astimezone(timezone.utc) + + +def format_timestamp(value): + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def load_json(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def load_watchlist(path): + payload = load_json(path) + if not isinstance(payload, dict) or not isinstance(payload.get("watches"), list): + raise ValueError("watchlist must contain a watches list") + + watches = [] + names = set() + for item in payload["watches"]: + if not isinstance(item, dict): + raise ValueError("every watch must be an object") + name = item.get("name") + keywords = item.get("keywords") + query = item.get("query") + language = item.get("language", "en") + domain = item.get("domain") + if not isinstance(name, str) or not name.strip(): + raise ValueError("every watch requires a non-empty name") + if name in names: + raise ValueError("watch names must be unique") + search_fields = { + key: value + for key, value in (("keywords", keywords), ("query", query)) + if value is not None + } + if len(search_fields) != 1: + raise ValueError("every watch requires exactly one of keywords or query") + search_field, search_value = next(iter(search_fields.items())) + if not isinstance(search_value, str) or not search_value.strip(): + raise ValueError( + "watch {} must be a non-empty string".format(search_field) + ) + if not isinstance(language, str) or not language.strip(): + raise ValueError("watch language must be a non-empty string") + if domain is not None and ( + not isinstance(domain, str) or not domain.strip() + ): + raise ValueError("watch domain must be a non-empty string") + names.add(name) + watch = { + "name": name, + "language": language, + "domain": domain, + search_field: search_value, + } + watches.append(watch) + if not watches: + raise ValueError("watchlist must contain at least one watch") + return watches + + +def resolve_window(args, fixture): + start_value = args.start_date + end_value = args.end_date + if fixture: + start_value = start_value or fixture.get("_fixture_start_date") + end_value = end_value or fixture.get("_fixture_end_date") + start = parse_timestamp(start_value, "start_date") + end = parse_timestamp(end_value, "end_date") + if start > end: + raise ValueError("start_date must not be after end_date") + return start, end + + +def resolve_generated_at(args, fixture): + if args.generated_at: + return format_timestamp(parse_timestamp(args.generated_at, "generated_at")) + if fixture: + return format_timestamp( + parse_timestamp( + fixture.get("_fixture_generated_at"), + "_fixture_generated_at", + ) + ) + return format_timestamp(datetime.now(timezone.utc)) + + +def validate_search_response(response): + if not isinstance(response, dict): + raise ValueError("Search API response must be an object") + if response.get("status") != "ok": + raise ValueError("Search API response status must be 'ok'") + if not isinstance(response.get("news"), list): + raise ValueError("Search API response news must be a list") + if any(not isinstance(article, dict) for article in response["news"]): + raise ValueError("every news item must be an object") + + +def fixture_response(fixture, watch): + responses = fixture.get("responses") + if not isinstance(responses, dict): + raise ValueError("fixture must contain a responses object") + if watch["name"] not in responses: + raise ValueError( + "fixture is missing a response for watch '{}'".format(watch["name"]) + ) + response = responses[watch["name"]] + validate_search_response(response) + return response + + +def fetch_live_response(watch, start, end, page_size): + api_key = os.environ.get("CURRENTS_API_KEY") + if not api_key: + raise ValueError("CURRENTS_API_KEY is required for live mode") + if page_size < 1: + raise ValueError("page_size must be positive") + + search_fields = [ + key for key in ("keywords", "query") if watch.get(key) is not None + ] + if len(search_fields) != 1: + raise ValueError("watch requires exactly one of keywords or query") + search_field = search_fields[0] + search_value = watch[search_field] + if not isinstance(search_value, str) or not search_value.strip(): + raise ValueError("watch {} must be a non-empty string".format(search_field)) + + params = { + search_field: search_value, + "language": watch["language"], + "start_date": format_timestamp(start), + "end_date": format_timestamp(end), + "page_number": 1, + "page_size": page_size, + } + if watch["domain"]: + params["domain"] = watch["domain"] + + response = requests.get( + SEARCH_URL, + headers={"Authorization": "Bearer {}".format(api_key)}, + params=params, + timeout=20, + ) + response.raise_for_status() + payload = response.json() + validate_search_response(payload) + return payload + + +def normalize_article(article, start, end): + title = article.get("title") + url = article.get("url") + published = article.get("published") + article_id = article.get("id") + description = article.get("description") + + for label, value in ( + ("title", title), + ("url", url), + ("published", published), + ("id", article_id), + ("description", description), + ): + if value is not None and not isinstance(value, str): + raise ValueError("news item {} must be a string".format(label)) + if not url: + raise ValueError("news item url must be present") + parsed_url = urlsplit(url) + if ( + parsed_url.scheme.lower() not in {"http", "https"} + or not parsed_url.netloc + or any(character.isspace() or ord(character) < 32 for character in url) + ): + raise ValueError("news item url must be an HTTP or HTTPS URL") + published_at = parse_timestamp(published, "news item published") + if published_at < start or published_at > end: + return None + + keys = ["url:{}".format(url)] + if article_id: + keys.append("id:{}".format(article_id)) + return { + "keys": keys, + "title": title or "Untitled", + "description": description or "", + "url": url, + "published": format_timestamp(published_at), + } + + +def load_state(path): + if not path.exists(): + return set() + payload = load_json(path) + keys = payload.get("seen_article_keys") if isinstance(payload, dict) else None + if not isinstance(keys, list) or any(not isinstance(key, str) for key in keys): + raise ValueError("state file must contain a seen_article_keys list") + return set(keys) + + +def collect_articles(watches, responses, start, end, seen_keys): + by_alias = {} + collected = [] + watch_counts = {} + discovered_keys = set() + + for watch in watches: + response = responses[watch["name"]] + validate_search_response(response) + matched = 0 + for raw_article in response["news"]: + article = normalize_article(raw_article, start, end) + if article is None: + continue + matched += 1 + keys = tuple(article["keys"]) + key_set = set(keys) + discovered_keys.update(keys) + existing_articles = [] + for key in keys: + existing = by_alias.get(key) + if existing is not None and all( + existing is not candidate for candidate in existing_articles + ): + existing_articles.append(existing) + if existing_articles: + existing_articles.sort(key=lambda item: item["_order"]) + target = existing_articles[0] + for duplicate in existing_articles[1:]: + target["keys"] = sorted( + set(target["keys"]) | set(duplicate["keys"]) + ) + target["watches"] = sorted( + set(target["watches"]) | set(duplicate["watches"]), + key=str.casefold, + ) + target["_suppressed"] = ( + target.get("_suppressed", False) + or duplicate.get("_suppressed", False) + ) + duplicate["_suppressed"] = True + for alias in duplicate["keys"]: + by_alias[alias] = target + target["keys"] = sorted(set(target["keys"]) | key_set) + if watch["name"] not in target["watches"]: + target["watches"].append(watch["name"]) + if key_set & seen_keys: + target["_suppressed"] = True + for key in target["keys"]: + by_alias[key] = target + continue + if key_set & seen_keys: + continue + article["watches"] = [watch["name"]] + article["_suppressed"] = False + article["_order"] = len(collected) + collected.append(article) + for key in keys: + by_alias[key] = article + watch_counts[watch["name"]] = matched + + articles = [ + article for article in collected if not article.get("_suppressed", False) + ] + for article in articles: + article["watches"].sort(key=str.casefold) + articles.sort(key=lambda item: item["title"].casefold()) + articles.sort(key=lambda item: item["published"], reverse=True) + return articles, watch_counts, discovered_keys + + +def escape_markdown_text(value): + normalized = " ".join(value.split()) + return re.sub(r"([\\`*_\[\]{}()#+\-.!|<>~])", r"\\\1", normalized) + + +def markdown_url(value): + encoded = quote( + value, + safe=":/?#[]@!$&'()*+,;=%-._~", + ) + return "<{}>".format(encoded) + + +def render_report(generated_at, start, end, articles, watch_counts): + lines = [ + "# Company News Change Report", + "", + "Generated at: {}".format(generated_at), + "Window: {} to {}".format(format_timestamp(start), format_timestamp(end)), + "", + ] + if not articles: + lines.append("No new matching articles.") + for article in articles: + lines.append( + "- [{}]({})".format( + escape_markdown_text(article["title"]), + markdown_url(article["url"]), + ) + ) + lines.append(" - Published: {}".format(article["published"])) + lines.append( + " - Watches: {}".format( + ", ".join(escape_markdown_text(name) for name in article["watches"]) + ) + ) + if article["description"]: + lines.append(" - {}".format(escape_markdown_text(article["description"]))) + + structured = { + "generated_at": generated_at, + "window": { + "start_date": format_timestamp(start), + "end_date": format_timestamp(end), + }, + "watch_match_counts": watch_counts, + "new_articles": [ + { + key: value + for key, value in article.items() + if key not in {"keys", "_suppressed", "_order"} + } + for article in articles + ], + } + return "\n".join(lines) + "\n", structured + + +def main(): + args = parse_args() + try: + watches = load_watchlist(args.watchlist) + fixture = load_json(args.fixture) if args.fixture else None + start, end = resolve_window(args, fixture) + generated_at = resolve_generated_at(args, fixture) + responses = {} + for watch in watches: + responses[watch["name"]] = ( + fixture_response(fixture, watch) + if fixture + else fetch_live_response(watch, start, end, args.page_size) + ) + seen_keys = load_state(args.state_file) + articles, watch_counts, discovered_keys = collect_articles( + watches, + responses, + start, + end, + seen_keys, + ) + markdown, structured = render_report( + generated_at, + start, + end, + articles, + watch_counts, + ) + except (OSError, ValueError, json.JSONDecodeError, requests.RequestException) as exc: + raise SystemExit("error: {}".format(exc)) + + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "report.md").write_text(markdown, encoding="utf-8") + (args.output_dir / "report.json").write_text( + json.dumps(structured, indent=2) + "\n", + encoding="utf-8", + ) + args.state_file.parent.mkdir(parents=True, exist_ok=True) + args.state_file.write_text( + json.dumps( + {"seen_article_keys": sorted(seen_keys | discovered_keys)}, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print("Wrote {}".format(args.output_dir)) + + +if __name__ == "__main__": + main() diff --git a/examples/company_news_monitor/watchlist.json b/examples/company_news_monitor/watchlist.json new file mode 100644 index 0000000..6bb2542 --- /dev/null +++ b/examples/company_news_monitor/watchlist.json @@ -0,0 +1,15 @@ +{ + "watches": [ + { + "name": "Battery competitors", + "query": "\"Northstar Battery\" OR \"Atlas Storage\"", + "language": "en" + }, + { + "name": "Industry policy", + "keywords": "\"grid storage\" regulation", + "language": "en", + "domain": "example.com" + } + ] +} diff --git a/examples/source_linked_briefing/README.md b/examples/source_linked_briefing/README.md index 9e4893e..24b39b0 100644 --- a/examples/source_linked_briefing/README.md +++ b/examples/source_linked_briefing/README.md @@ -31,7 +31,8 @@ For another saved Search API response, add `_fixture_generated_at` at the top le ## Run a live search -Create a [free Currents API key](https://currentsapi.services/en/register), then export it: +Create a [free Currents API key](https://currentsapi.services/en/register?utm_source=github&utm_medium=referral&utm_campaign=content-build-source-linked-news-briefing-currents-search-api), +then export it: ```bash export CURRENTS_API_KEY="your-api-key" diff --git a/tests/test_company_news_monitor_example.py b/tests/test_company_news_monitor_example.py new file mode 100644 index 0000000..031260b --- /dev/null +++ b/tests/test_company_news_monitor_example.py @@ -0,0 +1,525 @@ +import importlib.util +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import pytest + + +EXAMPLE = Path("examples/company_news_monitor/monitor.py") +WATCHLIST = Path("examples/company_news_monitor/watchlist.json") +FIXTURE = Path("examples/company_news_monitor/fixtures/search_responses.json") + + +def load_example_module(): + spec = importlib.util.spec_from_file_location("company_news_monitor", EXAMPLE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def run_fixture(tmp_path, state_file=None): + state_file = state_file or tmp_path / "state.json" + output_dir = tmp_path / "output" + result = subprocess.run( + [ + sys.executable, + str(EXAMPLE), + "--watchlist", + str(WATCHLIST), + "--fixture", + str(FIXTURE), + "--state-file", + str(state_file), + "--output-dir", + str(output_dir), + ], + capture_output=True, + check=False, + text=True, + ) + return result, output_dir, state_file + + +def test_checked_in_fixture_runs_without_network_or_credentials(tmp_path): + env_key = os.environ.pop("CURRENTS_API_KEY", None) + try: + result, output_dir, state_file = run_fixture(tmp_path) + finally: + if env_key is not None: + os.environ["CURRENTS_API_KEY"] = env_key + + assert result.returncode == 0, result.stderr + report = json.loads((output_dir / "report.json").read_text(encoding="utf-8")) + markdown = (output_dir / "report.md").read_text(encoding="utf-8") + assert len(report["new_articles"]) == 3 + assert "https://example.com/business/northstar-pilot-line" in markdown + assert "2026-08-05T07:30:00Z" in markdown + assert state_file.exists() + + +def test_watchlist_parser_rejects_duplicate_names(tmp_path): + module = load_example_module() + path = tmp_path / "watchlist.json" + path.write_text( + json.dumps( + { + "watches": [ + {"name": "Same", "keywords": "one", "language": "en"}, + {"name": "Same", "keywords": "two", "language": "en"}, + ] + } + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="unique"): + module.load_watchlist(path) + + +@pytest.mark.parametrize( + "watch", + [ + {"name": "Missing search", "language": "en"}, + { + "name": "Ambiguous search", + "keywords": "Northstar Battery", + "query": '"Northstar Battery" OR "Atlas Storage"', + "language": "en", + }, + ], +) +def test_watchlist_parser_requires_exactly_one_search_field(tmp_path, watch): + module = load_example_module() + path = tmp_path / "watchlist.json" + path.write_text(json.dumps({"watches": [watch]}), encoding="utf-8") + + with pytest.raises(ValueError, match="exactly one of keywords or query"): + module.load_watchlist(path) + + +def test_date_boundaries_are_inclusive_and_outside_articles_are_removed(): + module = load_example_module() + start = datetime(2026, 8, 4, tzinfo=timezone.utc) + end = datetime(2026, 8, 5, tzinfo=timezone.utc) + watches = [ + {"name": "Window", "keywords": "test", "language": "en", "domain": None} + ] + responses = { + "Window": { + "status": "ok", + "news": [ + { + "id": "start", + "title": "At start", + "url": "https://example.com/start", + "published": "2026-08-04T00:00:00Z", + }, + { + "id": "end", + "title": "At end", + "url": "https://example.com/end", + "published": "2026-08-05T00:00:00Z", + }, + { + "id": "before", + "title": "Before", + "url": "https://example.com/before", + "published": "2026-08-03T23:59:59Z", + }, + { + "id": "after", + "title": "After", + "url": "https://example.com/after", + "published": "2026-08-05T00:00:01Z", + }, + ], + } + } + + articles, counts, keys = module.collect_articles( + watches, responses, start, end, set() + ) + + assert {article["title"] for article in articles} == {"At start", "At end"} + assert counts == {"Window": 2} + assert keys == { + "id:start", + "url:https://example.com/start", + "id:end", + "url:https://example.com/end", + } + + +def test_same_url_with_optional_id_drift_is_reported_once(): + module = load_example_module() + start = datetime(2026, 8, 4, tzinfo=timezone.utc) + end = datetime(2026, 8, 5, tzinfo=timezone.utc) + watches = [ + {"name": "Company", "keywords": "test", "language": "en", "domain": None}, + {"name": "Policy", "keywords": "test", "language": "en", "domain": None}, + ] + shared_with_id = { + "id": "shared", + "title": "Shared result", + "url": "https://example.com/shared", + "published": "2026-08-04T12:00:00Z", + } + shared_without_id = { + key: value for key, value in shared_with_id.items() if key != "id" + } + responses = { + "Company": {"status": "ok", "news": [shared_without_id]}, + "Policy": {"status": "ok", "news": [shared_with_id]}, + } + + articles, _, _ = module.collect_articles(watches, responses, start, end, set()) + + assert len(articles) == 1 + assert articles[0]["watches"] == ["Company", "Policy"] + + +def test_same_id_with_url_variants_is_reported_once(): + module = load_example_module() + start = datetime(2026, 8, 4, tzinfo=timezone.utc) + end = datetime(2026, 8, 5, tzinfo=timezone.utc) + watches = [ + {"name": "Company", "keywords": "test", "language": "en", "domain": None}, + {"name": "Policy", "keywords": "test", "language": "en", "domain": None}, + ] + responses = { + "Company": { + "status": "ok", + "news": [ + { + "id": "shared", + "title": "Shared result", + "url": "https://example.com/shared", + "published": "2026-08-04T12:00:00Z", + } + ], + }, + "Policy": { + "status": "ok", + "news": [ + { + "id": "shared", + "title": "Shared result", + "url": "https://example.com/shared?ref=feed", + "published": "2026-08-04T12:00:00Z", + } + ], + }, + } + + articles, _, keys = module.collect_articles( + watches, responses, start, end, set() + ) + + assert len(articles) == 1 + assert articles[0]["watches"] == ["Company", "Policy"] + assert keys == { + "id:shared", + "url:https://example.com/shared", + "url:https://example.com/shared?ref=feed", + } + + +def test_transitive_alias_bridge_keeps_first_article_deterministically(): + module = load_example_module() + start = datetime(2026, 8, 4, tzinfo=timezone.utc) + end = datetime(2026, 8, 5, tzinfo=timezone.utc) + watches = [ + {"name": "First", "keywords": "test", "language": "en", "domain": None}, + {"name": "Second", "keywords": "test", "language": "en", "domain": None}, + {"name": "Bridge", "keywords": "test", "language": "en", "domain": None}, + ] + responses = { + "First": { + "status": "ok", + "news": [ + { + "title": "First record", + "url": "https://example.com/a", + "published": "2026-08-04T12:00:00Z", + } + ], + }, + "Second": { + "status": "ok", + "news": [ + { + "id": "shared", + "title": "Second record", + "url": "https://example.com/b", + "published": "2026-08-04T12:00:00Z", + } + ], + }, + "Bridge": { + "status": "ok", + "news": [ + { + "id": "shared", + "title": "Bridge record", + "url": "https://example.com/a", + "published": "2026-08-04T12:00:00Z", + } + ], + }, + } + + articles, _, _ = module.collect_articles(watches, responses, start, end, set()) + + assert len(articles) == 1 + assert articles[0]["title"] == "First record" + assert articles[0]["url"] == "https://example.com/a" + assert articles[0]["watches"] == ["Bridge", "First", "Second"] + + +def test_reverse_transitive_alias_bridge_keeps_first_article(): + module = load_example_module() + start = datetime(2026, 8, 4, tzinfo=timezone.utc) + end = datetime(2026, 8, 5, tzinfo=timezone.utc) + watches = [ + {"name": "First", "keywords": "test", "language": "en", "domain": None}, + {"name": "Second", "keywords": "test", "language": "en", "domain": None}, + {"name": "Bridge", "keywords": "test", "language": "en", "domain": None}, + ] + responses = { + "First": { + "status": "ok", + "news": [ + { + "id": "shared", + "title": "First record", + "url": "https://example.com/a", + "published": "2026-08-04T12:00:00Z", + } + ], + }, + "Second": { + "status": "ok", + "news": [ + { + "title": "Second record", + "url": "https://example.com/b", + "published": "2026-08-04T12:00:00Z", + } + ], + }, + "Bridge": { + "status": "ok", + "news": [ + { + "id": "shared", + "title": "Bridge record", + "url": "https://example.com/b", + "published": "2026-08-04T12:00:00Z", + } + ], + }, + } + + articles, _, _ = module.collect_articles(watches, responses, start, end, set()) + + assert len(articles) == 1 + assert articles[0]["title"] == "First record" + assert articles[0]["url"] == "https://example.com/a" + assert articles[0]["watches"] == ["Bridge", "First", "Second"] + + +def test_state_suppresses_same_url_when_id_appears_later(): + module = load_example_module() + start = datetime(2026, 8, 4, tzinfo=timezone.utc) + end = datetime(2026, 8, 5, tzinfo=timezone.utc) + watches = [ + {"name": "Company", "keywords": "test", "language": "en", "domain": None} + ] + response_without_id = { + "Company": { + "status": "ok", + "news": [ + { + "title": "Same result", + "url": "https://example.com/same", + "published": "2026-08-04T12:00:00Z", + } + ], + } + } + first_articles, _, first_keys = module.collect_articles( + watches, response_without_id, start, end, set() + ) + assert len(first_articles) == 1 + + response_with_id = { + "Company": { + "status": "ok", + "news": [ + { + "id": "same", + "title": "Same result", + "url": "https://example.com/same", + "published": "2026-08-04T12:00:00Z", + } + ], + } + } + later_articles, _, later_keys = module.collect_articles( + watches, response_with_id, start, end, first_keys + ) + + assert later_articles == [] + assert later_keys == { + "id:same", + "url:https://example.com/same", + } + + +def test_second_run_uses_state_and_reports_no_new_articles(tmp_path): + first, _, state_file = run_fixture(tmp_path / "first", tmp_path / "state.json") + assert first.returncode == 0, first.stderr + + second_dir = tmp_path / "second" + second, output_dir, _ = run_fixture(second_dir, state_file) + + assert second.returncode == 0, second.stderr + report = json.loads((output_dir / "report.json").read_text(encoding="utf-8")) + assert report["new_articles"] == [] + assert "No new matching articles." in ( + output_dir / "report.md" + ).read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + ("search_field", "search_value"), + [ + ("keywords", '"grid storage" regulation'), + ("query", '"Northstar Battery" OR "Atlas Storage"'), + ], +) +def test_live_mode_sends_matching_documented_search_parameter( + monkeypatch, search_field, search_value +): + module = load_example_module() + captured = {} + + class FakeResponse: + def raise_for_status(self): + return None + + def json(self): + return {"status": "ok", "news": []} + + def fake_get(url, headers, params, timeout): + captured.update( + {"url": url, "headers": headers, "params": params, "timeout": timeout} + ) + return FakeResponse() + + monkeypatch.setenv("CURRENTS_API_KEY", "test-key") + monkeypatch.setattr(module.requests, "get", fake_get) + watch = { + "name": "Policy", + "language": "en", + "domain": "example.com", + search_field: search_value, + } + start = datetime(2026, 8, 4, 9, tzinfo=timezone.utc) + end = datetime(2026, 8, 5, 9, tzinfo=timezone.utc) + + payload = module.fetch_live_response(watch, start, end, 20) + + assert payload == {"status": "ok", "news": []} + assert captured["url"] == module.SEARCH_URL + assert captured["headers"] == {"Authorization": "Bearer test-key"} + assert captured["params"] == { + search_field: search_value, + "language": "en", + "start_date": "2026-08-04T09:00:00Z", + "end_date": "2026-08-05T09:00:00Z", + "page_number": 1, + "page_size": 20, + "domain": "example.com", + } + assert captured["timeout"] == 20 + + +@pytest.mark.parametrize( + "url", + [ + "javascript:alert(1)", + "ftp://example.com/report", + "https://example.com/report\n- injected", + ], +) +def test_article_url_requires_safe_http_or_https_url(url): + module = load_example_module() + start = datetime(2026, 8, 4, tzinfo=timezone.utc) + end = datetime(2026, 8, 5, tzinfo=timezone.utc) + + with pytest.raises(ValueError, match="HTTP or HTTPS"): + module.normalize_article( + { + "title": "Unsafe URL", + "url": url, + "published": "2026-08-04T12:00:00Z", + }, + start, + end, + ) + + +def test_report_escapes_hostile_publisher_metadata(): + module = load_example_module() + start = datetime(2026, 8, 4, tzinfo=timezone.utc) + end = datetime(2026, 8, 5, tzinfo=timezone.utc) + articles = [ + { + "keys": ["url:https://example.com/report_(final)"], + "title": "[Trusted](https://attacker.example)