{ "schema": 1, "run_id": "clientsflow-new2000-20261004", "captured_at_utc": "2026-10-04T06:50:03.517517+00:00", "captured_at_Europe_Budapest": "2026-10-04T08:50:03.518211+02:00", "production_scope": "Read-only audit. No paid/provider service calls, restart, source edits, imports, sends, monitoring or production state mutation. Public documentation was read separately. Audit files written only under assigned audit/output paths.", "state_at_last_persisted_update": { "active_calls": 0, "actual_cost_usd": 5.4769407, "available_frozen_candidates": 2127, "cap_usd": 100.0, "category_counts": { "construction_services": 74, "excluded": 5, "other_services": 33, "unresolved": 687 }, "committed": 5.51666002, "eligible_review_count": 91, "error": "", "excluded": 5, "held": 630, "incomplete": 0, "max_concurrency": 8, "model": "openai/gpt-6-luna", "nonexcluded_personalized_count": 91, "observed_peak_calls": 7, "personalized_including_held": 107, "pid": 1275030, "prefix_count": 0, "processing_counts": { "complete": 113, "email_held": 614, "model_held": 6, "scrape_unavailable": 58, "uncertain_paid_held": 2, "worker_held": 6 }, "raw_processed": 799, "ready_import_count": 91, "reasoning_effort": "max", "remaining_budget_usd": 94.48333998, "reserved": 0.03971932, "review_url": "https://get-leads2.cfd-staging.com/prepare1200/?run=clientsflow-new2000-20261004", "run_actual_usd": 5.4769407, "run_id": "clientsflow-new2000-20261004", "schema": 1, "script_version": "1c31b19311d85428db194f772783072536b04b59b4a54a4cea8e01e40aae3432", "source_remaining": 1328, "starttime": "2026-10-03T23:36:15.815985+00:00", "status": "running", "supervision": "Parent only: T+60min assessment and one check two hours after first refinement verification. No recurring AI supervision.", "target": 2000, "unit": "get-leads-new2000-20261004.service", "updated_at": "2026-10-04T05:47:20.599647+00:00" }, "snapshot_warning": "State timestamp is 05:47:20.599647 UTC. It is 62m42.918s older than the audit observation. Service active is not proof of healthy throughput. Separate ledger/process snapshots carry their own timestamps.", "summary": "The strongest current cause is memory throttling plus a shared budget-lock queue. Earlier OOM downtime, batch barriers, full history scans and weak contact yield compound it. The one combined model request is normally a small part of the pipeline, though Max reasoning exhausted the output cap in six saved calls.", "stages": [ { "number": 1, "name": "Open the durable run and reserve one producer", "code": [ "new2000.py:368-386", "new2000.py:156-194", "new2000.py:37-74" ], "plain_english": "Resume the existing run, take run/source producer locks, preserve the older paid ledger, load the separate USD100 ledger, and confirm provider balances. Previously completed rows and paid responses are reused. Only review preparation is authorized.", "cost": "No generation purchase in this stage. Local work and provider account reads.", "concurrency": "One main process. Shared whole-ledger reads and writes serialize on budget.lock.", "validation": "Run identity, cap and ledger identity must agree. Required provider access must exist. Duplicate paid dispatch is not allowed." }, { "number": 2, "name": "Rebuild the do-not-use history", "code": [ "production_pool.py:114-169", "scale1200.py:152-198", "new2000.py:76-101", "scale1200.py:473-490" ], "plain_english": "Read Instantly membership, blocklist, sent email, manual email, campaigns and lists, then the Notion CRM. Add prior review companies, prior paid/reserved identities and local lead-bank suppression. Connect aliases so a different email or domain for the same known company also stays excluded.", "cost": "No model or paid verification call. API reads and local processing.", "concurrency": "Serial main-loop barrier. Runs initially and when the oldest proof is at least one hour old, and again at final acceptance.", "validation": "Every paginated snapshot must reach its end. Previous exclusions remain merged. The main loop does not feed another worker batch until this finishes." }, { "number": 3, "name": "Select unused source companies", "code": [ "new2000.py:134-154", "new2000.py:176-183", "scale1200.py:209-222", "new2000.py:361-417" ], "plain_english": "The first launch loaded unused old raw pools and Hungarian lead-bank companies. The current wrapper prefers newly acquired Maps companies when the fresh queue is empty, then returns to older unused candidates after the fresh plan is exhausted. Construction and evidenced team/project/location mentions sort first. Those phrases are priority signals, not a measured company-size qualification.", "cost": "Old raw records are reused free.", "concurrency": "Selection and full candidate/exclusion scans are serial. Work is chosen in batches of at most eight.", "validation": "Email/domain/name/company duplicates, all historical prepared/paid identities, unproved email-to-company identity and invalid domains are excluded before append." }, { "number": 4, "name": "Acquire more Maps records when the fresh queue is empty", "code": [ "new2000.py:28-31", "new2000.py:260-267", "new2000.py:287-329" ], "plain_english": "Run the installed Google Places actor for one Hungarian query/city combination. It requests up to 200 places with websites, disables paid contact enrichment and rejects social-only, closed, non-Hungarian or previously used companies. The deterministic plan contains 225 query/city combinations.", "cost": "Paid Apify actor, from the same USD100 total. Per-actor reservation USD1.61. Existing actor receipts and completed query folders are reused.", "concurrency": "One actor at a time, polled every 15 seconds. Candidate-generation work does not overlap this acquisition function.", "validation": "No blind actor retry if a start outcome is ambiguous. Record charges and retain unresolved reservations. Maximum dataset retrieval is 1000 records, and reaching that bound is rejected." }, { "number": 5, "name": "Find a real contact on each Maps company website", "code": [ "new2000.py:273-285", "new2000.py:330-336", "fresh100.py:133-154", "run_trial.py:341-422" ], "plain_english": "Fetch the company homepage and observed contact/about links. Take a visible email from a matching company page, prefer its business domain and common role mailbox, and keep the page quote/hash proving the connection. A free-mail address is accepted only when it was shown on the company-owned page.", "cost": "Free website HTTP requests. The resulting scrape cache is reused downstream.", "concurrency": "Up to eight free website crawls. The actor batch must finish crawling and filtering before it appends any candidates and returns to model processing.", "validation": "Usable same-company page, a visible syntactically valid email on that page, and no identity/history conflict. No guessed email address." }, { "number": 6, "name": "Check email deliverability before buying current personalization", "code": [ "new2000.py:206-231", "production_pool.py:410-421" ], "plain_english": "Send the exact candidate email to MillionVerifier once, or reuse this run's saved verification. Only a completed ok result without a provider error proceeds to the model path. Catch-all, invalid, unknown or uncertain addresses are held. The current implementation still scrapes held companies for review evidence even though their paid personalization will not be bought.", "cost": "Paid verification credits, conservatively accounted at USD0.0039 each. Repeat reads of the saved verification do not buy another check. This is accounting allocation, not proof of a new cash purchase.", "concurrency": "Inside the same pool of up to eight candidate workers, with 20-second provider verification and 45-second HTTP timeout.", "validation": "Return email must match and provider status must be recognized. Held candidates do not count toward 2000." }, { "number": 7, "name": "Load current website evidence and build the one combined request", "code": [ "scale1200.py:436-466", "run_trial.py:341-422", "fresh100.py:133-154", "pilot100.py:363-374" ], "plain_english": "Recheck current exclusions, take the company source lock and reuse the run-local scrape or fetch it. Save full evidence, then give the model only bounded exact text with the company name and domain. A saved prepared request is immutable on resume.", "cost": "Free local/cache/HTTP work. No payment if pages are unusable or input bounds fail.", "concurrency": "Runs in the candidate worker before its model call. It is not a separate permanent scrape worker pool.", "validation": "Scraper seeks up to three usable pages, with up to eight requests, a nominal 35-second crawl deadline and up to a 12-second fallback. Source page text is capped to 18000 characters per page for the model. The assembly code permits at most four evidence pages, but all 127 actual payloads had one to three. Full request hard limit is 100000 bytes. Timeouts are per operation, not a verified hard total wall-time bound." }, { "number": 8, "name": "Make one combined Luna request", "code": [ "scale1200.py:253-267", "scale1200.py:345-388", "endtoend100.py:457-550" ], "plain_english": "In one paid call, classify the business, write the short Hungarian activity fragment, choose a generic or evidence-backed personal greeting, choose cég or vállalkozás, and return exact supporting page quotes. The other Python files are imported helpers, not separate model agents or separate paid classification/name/legal calls.", "cost": "Paid OpenRouter request using openai/gpt-6-luna, reasoning max, Azure only. Saved paid results are reused instead of bought again.", "concurrency": "At most eight model calls through both a bounded semaphore and cross-process slot files. Those slots share the eight candidate workers. Having eight configured slots does not mean eight requests stay busy.", "validation": "Reserve worst-case spend before dispatch. Response model, actual provider and actual cost must match. HTTP timeout is 120 seconds. Uncertain dispatch is held without automatic paid retry." }, { "number": 9, "name": "Validate and normalize the returned row in Python", "code": [ "fresh100.py:95-116", "endtoend100.py:219-278", "pilot100.py:377-429", "scale1200.py:403-434" ], "plain_english": "Require exactly the schema fields. Check the activity quote is present in the saved page, check the short fragment shape, and verify person and legal-form quotes. Repair page IDs, whitespace and Unicode representation only by recovering actual source text. If person evidence is weak, use Szép napot. If legal-form evidence is weak, use vállalkozás.", "cost": "No second model judge and no extra paid refinement.", "concurrency": "Local work and budget-ledger/SQLite writes inside each candidate worker.", "validation": "The validator enforces schema, quote presence, fragment length and certain name/legal rules. Category choice, business identity interpretation and Hungarian naturalness still rely on the single model plus human review. It is not a 100% semantic quality proof. Six saved responses hit the entire 12000-token output cap with reasoning only, returned null content and became model-held TypeErrors." }, { "number": 10, "name": "Decide whether the row counts and save progress", "code": [ "new2000.py:196-204", "new2000.py:233-255", "scale1200.py:471", "new2000.py:418-428" ], "plain_english": "For supported construction or other-service personalization, reuse the email receipt, check the current exclusion snapshot and write a separate eligibility receipt. Count the row only if it has a generation ID, nonempty personalization, an accepted category, ok email, matching email and the ready flags/receipt.", "cost": "Reuses existing paid receipts. No email send or campaign import.", "concurrency": "Every completed future asks the main thread to reload rows and the ledger and write progress. The next batch waits for the slowest worker in the current batch.", "validation": "INSTANTLY_READY is an internal review/import-eligibility label. Rows still await human review. It does not prove an import, send, reply or sale. The final write can lag behind the finished model response." }, { "number": 11, "name": "Final target acceptance, when 2000 is actually reached", "code": [ "new2000.py:342-349", "new2000.py:393-399", "scale1200.py:492-516" ], "plain_english": "Force a fresh history scan, recheck eligible rows, require exactly 2000 distinct source-backed eligible companies, then create the final CSV and an immutable hashed target receipt.", "cost": "Local validation and refreshed read-only history. Existing email results are reused by email_attempt.", "concurrency": "Serial final acceptance.", "validation": "Exact target, no duplicate identities/history conflicts, source quote present, domain and email receipts match. This stage has not completed at the audited snapshot." } ], "prompt_inventory": { "captured_at": "2026-10-04T06:49:56.638144+00:00", "prepared_count": 127, "full_request_hash_unique_count": 127, "payload_hash_mismatches": [], "request_template_saved_sha256": "3a1182f3fcc64cdb72b27516ee9c82bef08ad215f0be0876ad67976b4a3b8d0e", "request_template_calculated_sha256": "3a1182f3fcc64cdb72b27516ee9c82bef08ad215f0be0876ad67976b4a3b8d0e", "template_static_hash": "67b62e8b760aa21498272a592fef5a5c52593003d36480c5cd98e5bf29cfa378", "input_shape": { "candidate_id": "source identifier", "company_name": "source company name", "domain": "candidate domain", "pages": [ { "page_id": "p1...pn", "url": "source page URL", "text": "exact bounded scraped text" } ] }, "temperature": "absent from saved request, no explicit value", "private_inputs_excluded": true, "run_id": "clientsflow-new2000-20261004", "endpoint": "https://openrouter.ai/api/v1/chat/completions", "http_timeout_seconds": 120, "metadata_header": { "X-OpenRouter-Metadata": "enabled" }, "one_combined_model_request_per_candidate": true, "request_reconstruction": "Copy static_payload, then append '\\nINPUT\\n' + json.dumps(company_input, ensure_ascii=False) to the user content. The source code does not send the empty template INPUT {} as the company request.", "actual_limits": { "prepared_pages": { "1": 9, "2": 20, "3": 98 }, "max_page_text_characters": 18000, "largest_prepared_payload_bytes": 58580, "hard_request_byte_limit": 100000, "prepared_count": 127 }, "prompt_byte_hash_encoding": "UTF-8, without label/delimiter or added final newline", "object_hash_format": "SHA-256 of UTF-8 json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(comma, colon))", "scope": "Existing saved request settings only. No provider requests made by this audit." }, "prompt_variant_statistics": { "static_payload_sha256": "67b62e8b760aa21498272a592fef5a5c52593003d36480c5cd98e5bf29cfa378", "system_sha256": "ce9e2297f8808b9cb635302d758104c58154950eda41963366432c057b358440", "user_static_sha256": "a577c2b2b3f457cec0a651bbf068c76ffccd74892f477dce3220f1dcc3181a3c", "response_format_sha256": "69d432d6e7e4ccc5f8299aebc9e19218aaa9b75a02430d5642a17167fb2bb0a0", "prepared_count": 127, "ledger_attempt_count": 121, "dispatched_count": 120, "completed_count": 119, "returned_models": { "openai/gpt-6-luna": 119 }, "returned_providers": { "Azure": 119 }, "first_prepared_at": "2026-10-03T23:43:43.852976+00:00", "last_prepared_at": "2026-10-04T06:34:11.204568+00:00" }, "wrapper_versions": { "current_wrapper": { "path": "/home/matt/.local/share/get-leads-review/scripts/new2000.py", "sha256": "1c31b19311d85428db194f772783072536b04b59b4a54a4cea8e01e40aae3432", "bytes": 35004, "mtime_utc": "2026-10-04T00:44:23.022634+00:00" }, "prior_retained_wrapper": { "path": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/check1/new2000-before.py", "sha256": "10b9e487e3fdf0c68b841bd611ecaf356814234ecf8938567df7986743658926", "bytes": 34054, "mtime_utc": "2026-10-04T00:44:22.990633+00:00" }, "current_wrapper_started_at": "2026-10-04T00:44:23.674963+00:00", "model_dispatches_before_current_wrapper_start": 75, "model_dispatches_at_or_after_current_wrapper_start": 45, "model_reservation_without_dispatch": 1, "count_limit": "Dispatch counts are time partitions, not per-attempt embedded script attestations. The saved prepared files do not store a script hash. Two recoverable wrapper versions are evidenced. Earlier live edits/restarts cannot be assigned byte-for-byte solely from current retained files.", "behavior_change": "Retained prior wrapper prioritizes unused existing candidates. Current wrapper prefers fresh Maps candidates/acquisition ahead of remaining legacy candidates. The extracted static model prompt and settings did not change across 127 prepared payloads.", "historical_receipts": { "launch-bundle.json": { "script_sha256": "10b9e487e3fdf0c68b841bd611ecaf356814234ecf8938567df7986743658926", "first_paid_dispatch_at": "2026-10-03T23:43:44.068294+00:00", "state": { "active_calls": 1, "eligible_review_count": 9, "raw_processed": 261, "script_version": "10b9e487e3fdf0c68b841bd611ecaf356814234ecf8938567df7986743658926", "starttime": "2026-10-03T23:36:15.815985+00:00", "updated_at": "2026-10-04T00:03:57.715460+00:00" } }, "check1/state.json.before": { "active_calls": 0, "committed": 2.892008375, "eligible_review_count": 48, "processing_counts": { "complete": 69, "email_held": 559, "model_held": 6, "scrape_unavailable": 54 }, "raw_processed": 688, "script_version": "10b9e487e3fdf0c68b841bd611ecaf356814234ecf8938567df7986743658926", "starttime": "2026-10-03T23:36:15.815985+00:00", "status": "running", "updated_at": "2026-10-04T00:37:31.485962+00:00" }, "check2/state-before.json": { "active_calls": 1, "committed": 4.543944669999999, "eligible_review_count": 80, "processing_counts": { "complete": 101, "email_held": 600, "model_held": 6, "pending": 1, "scrape_unavailable": 58 }, "raw_processed": 765, "script_version": "1c31b19311d85428db194f772783072536b04b59b4a54a4cea8e01e40aae3432", "starttime": "2026-10-03T23:36:15.815985+00:00", "status": "running", "updated_at": "2026-10-04T01:17:35.802093+00:00" } } }, "source_hashes": [ { "path": "/home/matt/.local/share/get-leads-review/scripts/new2000.py", "sha256": "1c31b19311d85428db194f772783072536b04b59b4a54a4cea8e01e40aae3432", "bytes": 35004, "mtime_utc": "2026-10-04T00:44:23.022634+00:00" }, { "path": "/home/matt/.local/share/get-leads-review/scripts/scale1200.py", "sha256": "7f87384bde7d17691a6ea5612f6be611e7172eaecbea2b2b2dc9aeafa3741c98", "bytes": 51551, "mtime_utc": "2026-09-28T00:05:13.890188+00:00" }, { "path": "/home/matt/.local/share/get-leads-review/scripts/fresh100.py", "sha256": "968d6065f3c4958a3ce98ae7da3db8753ead6e5687f4a402a7040e5925107564", "bytes": 22020, "mtime_utc": "2026-09-27T22:40:27.948148+00:00" }, { "path": "/home/matt/.local/share/get-leads-review/scripts/endtoend100.py", "sha256": "f5de51f78c3df1b11044925285b234e86a6301a07dd2c55107cf60e98a634006", "bytes": 38311, "mtime_utc": "2026-09-25T11:29:40.331715+00:00" }, { "path": "/home/matt/.local/share/get-leads-review/scripts/pilot100.py", "sha256": "defd41f974656c277bc28c2bb90c667ca4045fc728157c658f2e19d35ddb2a53", "bytes": 45650, "mtime_utc": "2026-09-25T06:58:51.266960+00:00" }, { "path": "/home/matt/.local/share/get-leads-review/scripts/production_pool.py", "sha256": "c177d480f8dc3928b2ba2b5fef1ae1d774aa0a615f67f1be7f90fb28984da7f4", "bytes": 36568, "mtime_utc": "2026-09-23T08:04:12.212535+00:00" }, { "path": "/home/matt/.local/share/get-leads-review/scripts/run_trial.py", "sha256": "f9dd6872257d85365e51cb8a699bf1d58c6e206210e1ca8373ceae65ee9793f3", "bytes": 67939, "mtime_utc": "2026-09-25T08:01:29.537213+00:00" }, { "path": "/home/matt/.local/share/get-leads-review/scripts/review_server.py", "sha256": "29a382cbfd0a28c3a14640929cb7bc37c44e9f00f5847b5849a21036b2770003", "bytes": 47717, "mtime_utc": "2026-10-03T23:32:36.247822+00:00" }, { "path": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/check1/new2000-before.py", "sha256": "10b9e487e3fdf0c68b841bd611ecaf356814234ecf8938567df7986743658926", "bytes": 34054, "mtime_utc": "2026-10-04T00:44:22.990633+00:00" }, { "path": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/new2000-check1.py", "sha256": "1c31b19311d85428db194f772783072536b04b59b4a54a4cea8e01e40aae3432", "bytes": 35004, "mtime_utc": "2026-10-04T00:40:07.835837+00:00" }, { "path": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/check2/resource_guard.py", "sha256": "3cef9aade371893052e2a7d653b3bed143b8ddd50c9411532bb572fe71d168a9", "bytes": 2360, "mtime_utc": "2026-10-04T03:00:37.761899+00:00" }, { "path": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/request-template.json", "sha256": "01589daf6071f0623545c6947ae93c416efe0a92af65176d7a2da293fd8a0f05", "bytes": 6728, "mtime_utc": "2026-10-04T03:05:53.254569+00:00" } ], "timing": { "wall_since_state_start_seconds": 26027.701532, "state_snapshot_age_at_audit_seconds": 3762.91787, "service_outages": [ { "reason": "Stopped/restarted", "from": "2026-10-03T23:47:30.425419+00:00", "to": "2026-10-03T23:50:48.805831+00:00", "seconds": 198.380412 }, { "reason": "Stopped/restarted", "from": "2026-10-03T23:58:55.095828+00:00", "to": "2026-10-03T23:59:16.328541+00:00", "seconds": 21.232713 }, { "reason": "Stop timed out and process killed", "from": "2026-10-04T00:43:34.472538+00:00", "to": "2026-10-04T00:44:23.674963+00:00", "seconds": 49.202425 }, { "reason": "OOM killed, then manual recovery start", "from": "2026-10-04T01:17:45.438451+00:00", "to": "2026-10-04T03:05:52.534031+00:00", "seconds": 6487.09558 } ], "service_outage_total_seconds": 6755.91113, "history_snapshot_intervals": [ { "directory": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004", "started_at": "2026-10-03T23:36:16.341234+00:00", "last_checked_at": "2026-10-03T23:43:30.557017+00:00", "span_seconds": 434.215783, "page_reads": 261, "complete": true }, { "directory": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/history-refresh/20261004T003731", "started_at": "2026-10-04T00:37:32.076035+00:00", "last_checked_at": "2026-10-04T00:43:32.440058+00:00", "span_seconds": 360.364023, "page_reads": 236, "complete": false }, { "directory": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/history-refresh/20261004T004424", "started_at": "2026-10-04T00:44:25.176218+00:00", "last_checked_at": "2026-10-04T00:51:21.632712+00:00", "span_seconds": 416.456494, "page_reads": 261, "complete": true }, { "directory": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/history-refresh/20261004T030554", "started_at": "2026-10-04T03:05:54.627158+00:00", "last_checked_at": "2026-10-04T03:12:52.189849+00:00", "span_seconds": 417.562691, "page_reads": 261, "complete": true }, { "directory": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/history-refresh/20261004T041934", "started_at": "2026-10-04T04:19:34.832875+00:00", "last_checked_at": "2026-10-04T04:26:35.644346+00:00", "span_seconds": 420.811471, "page_reads": 261, "complete": true }, { "directory": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004/history-refresh/20261004T054013", "started_at": "2026-10-04T05:40:14.030868+00:00", "last_checked_at": "2026-10-04T05:47:15.761253+00:00", "span_seconds": 421.730385, "page_reads": 261, "complete": true } ], "history_snapshot_total_span_seconds": 2471.140847, "history_page_reads": 1541, "actor_seconds": { "n": 16, "min": 25.046, "median": 40.114000000000004, "p90_nearest_rank": 83.496, "max": 192.359, "sum": 828.62, "mean": 51.78875 }, "post_actor_to_local_done_seconds": { "n": 16, "min": 13.566059, "median": 26.160281, "p90_nearest_rank": 57.394773, "max": 93.971994, "sum": 557.048872, "mean": 34.8155545 }, "email_dispatch_to_receipt_seconds": { "n": 795, "min": 0.104017, "median": 2.047974, "p90_nearest_rank": 20.201975, "max": 24.60467, "sum": 3937.907363, "mean": 4.953342594968554 }, "model_dispatch_to_validated_result_seconds": { "n": 113, "min": 8.066341, "median": 19.750749, "p90_nearest_rank": 39.843982, "max": 3334.668007, "sum": 12962.004519, "mean": 114.70800459292036 }, "model_dispatch_to_validated_result_before_restart_seconds": { "n": 101, "min": 8.066341, "median": 18.194465, "p90_nearest_rank": 34.092763, "max": 54.205027, "sum": 2182.954332 }, "model_dispatch_to_validated_result_after_restart_seconds": { "n": 12, "min": 15.282961, "median": 320.38896800000003, "p90_nearest_rank": 2530.233313, "max": 3334.668007, "sum": 10779.050187 }, "provider_generation_time_raw": { "n": 119, "min": 7033, "median": 18960, "p90_nearest_rank": 38999, "max": 56891, "sum": 2719538 }, "provider_generation_time_units": "Saved receipts do not label units. Public API reference lists the field without an explicit unit. Do not present a conversion as directly documented latency.", "last_model_dispatch_at": "2026-10-04T05:12:46.253563+00:00", "last_validated_model_result_at": "2026-10-04T05:40:11.657699+00:00", "not_instrumented": [ "Exact HTTP response arrival versus local settlement duration", "Per-company scrape start/end duration", "Time spent reconstructing exclusion sets", "Time waiting for budget.lock", "Complete model slot utilization timeline", "Exact overlap-adjusted end-to-end time attribution" ], "non_additivity": "Per-request times run concurrently and include accounting. They must not be added to outage/history wall time to claim a total breakdown." }, "process_profile": { "captured_at": "2026-10-04T06:50:03.517087+00:00", "unit": { "Restart": "on-abnormal", "MainPID": "1275030", "NRestarts": "0", "ExecMainStartTimestamp": "Sun 2026-10-04 03:05:52 UTC", "ControlGroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/get-leads-new2000-20261004.service", "MemoryCurrent": "842039296", "MemoryPeak": "849448960", "CPUUsageNSec": "926300880000", "MemoryHigh": "805306368", "MemoryMax": "1610612736", "MemorySwapMax": "268435456", "ActiveState": "active", "SubState": "running" }, "samples": [ { "at": "2026-10-04T06:49:58.509289+00:00", "status": { "State": "S (sleeping)", "VmHWM": "837484 kB", "VmRSS": "828232 kB", "VmSwap": "72428 kB", "Threads": "9", "voluntary_ctxt_switches": "123360", "nonvoluntary_ctxt_switches": "1201817" }, "user_cpu_ticks": 25419, "system_cpu_ticks": 67207, "ticks_per_second": 100, "io": { "rchar": "3715416946", "wchar": "3295137808", "syscr": "261570", "syscw": "265200", "read_bytes": "119652544512", "write_bytes": "3299131392", "cancelled_write_bytes": "200704" }, "threads": [ { "tid": 1275030, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488809, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488810, "comm": "python3", "wchan": "futex_wait_queue" }, { "tid": 1488811, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488812, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488813, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488814, "comm": "python3", "wchan": "mem_cgroup_handle_over_high" }, { "tid": 1488815, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488816, "comm": "python3", "wchan": "futex_wait_queue" } ], "fd_counts": { "other_file": 1, "socket": 2, "runtime_file": 30 }, "cgroup_memory_events": "low 0\nhigh 6087393\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", "cgroup_memory_pressure": "some avg10=63.75 avg60=67.02 avg300=68.60 total=6568590863\nfull avg10=63.75 avg60=67.02 avg300=68.60 total=6564688658\n", "cgroup_cpu_pressure": "some avg10=0.00 avg60=0.00 avg300=0.00 total=75462408\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=71720232\n", "cgroup_io_pressure": "some avg10=0.00 avg60=0.00 avg300=0.01 total=49325034\nfull avg10=0.00 avg60=0.00 avg300=0.01 total=48835272\n", "host_memory_pressure": "some avg10=36.78 avg60=39.75 avg300=40.22 total=6845666778\nfull avg10=29.96 avg60=34.05 avg300=34.56 total=5119311067\n" }, { "at": "2026-10-04T06:50:03.511197+00:00", "status": { "State": "S (sleeping)", "VmHWM": "837484 kB", "VmRSS": "828468 kB", "VmSwap": "72332 kB", "Threads": "9", "voluntary_ctxt_switches": "123360", "nonvoluntary_ctxt_switches": "1201817" }, "user_cpu_ticks": 25421, "system_cpu_ticks": 67208, "ticks_per_second": 100, "io": { "rchar": "3715416946", "wchar": "3295137808", "syscr": "261570", "syscw": "265200", "read_bytes": "119652556800", "write_bytes": "3299131392", "cancelled_write_bytes": "200704" }, "threads": [ { "tid": 1275030, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488809, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488810, "comm": "python3", "wchan": "futex_wait_queue" }, { "tid": 1488811, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488812, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488813, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488814, "comm": "python3", "wchan": "mem_cgroup_handle_over_high" }, { "tid": 1488815, "comm": "python3", "wchan": "locks_lock_inode_wait" }, { "tid": 1488816, "comm": "python3", "wchan": "futex_wait_queue" } ], "fd_counts": { "other_file": 1, "socket": 2, "runtime_file": 30 }, "cgroup_memory_events": "low 0\nhigh 6088022\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", "cgroup_memory_pressure": "some avg10=62.74 avg60=66.45 avg300=68.44 total=6571632066\nfull avg10=62.74 avg60=66.45 avg300=68.44 total=6567729861\n", "cgroup_cpu_pressure": "some avg10=0.00 avg60=0.00 avg300=0.00 total=75465454\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=71723278\n", "cgroup_io_pressure": "some avg10=0.00 avg60=0.00 avg300=0.01 total=49325877\nfull avg10=0.00 avg60=0.00 avg300=0.01 total=48836115\n", "host_memory_pressure": "some avg10=35.45 avg60=39.10 avg300=40.07 total=6847435948\nfull avg10=28.97 avg60=33.39 avg300=34.40 total=5120683015\n" } ], "sample_elapsed_seconds": 5.001908, "cpu_seconds_during_sample": 0.03, "host_memory": { "MemTotal": "7937220 kB", "MemAvailable": "2917340 kB", "SwapTotal": "6291448 kB", "SwapFree": "224 kB" }, "process_locks": [ "9: FLOCK ADVISORY WRITE 1275030 08:01:828698 0 EOF", "12: FLOCK ADVISORY WRITE 1275030 08:01:912809 0 EOF", "13: FLOCK ADVISORY WRITE 1275030 08:01:825627 0 EOF", "14: FLOCK ADVISORY WRITE 1275030 08:01:777232 0 EOF", "15: FLOCK ADVISORY WRITE 1275030 08:01:779416 0 EOF", "16: FLOCK ADVISORY WRITE 1275030 08:01:825626 0 EOF", "44: FLOCK ADVISORY WRITE 1275030 08:01:828751 0 EOF", "45: FLOCK ADVISORY WRITE 1275030 08:01:828743 0 EOF", "79: FLOCK ADVISORY WRITE 1275030 08:01:825638 0 EOF", "79: -> FLOCK ADVISORY WRITE 1275030 08:01:825638 0 EOF", "79: -> FLOCK ADVISORY WRITE 1275030 08:01:825638 0 EOF", "79: -> FLOCK ADVISORY WRITE 1275030 08:01:825638 0 EOF", "79: -> FLOCK ADVISORY WRITE 1275030 08:01:825638 0 EOF", "79: -> FLOCK ADVISORY WRITE 1275030 08:01:825638 0 EOF", "79: -> FLOCK ADVISORY WRITE 1275030 08:01:825638 0 EOF", "80: FLOCK ADVISORY WRITE 1275030 08:01:828739 0 EOF", "81: FLOCK ADVISORY WRITE 1275030 08:01:828708 0 EOF", "124: FLOCK ADVISORY WRITE 1275030 08:01:828741 0 EOF" ], "limits": "One bounded /proc sample only. No attach, signals, tracing, stop, restart, provider calls, or production writes." }, "known_failure_counts": { "model_output_token_cap_null_content": 6, "sqlite_locked_workers": 6, "uncertain_paid_rows_after_OOM": 2, "persisted_scrape_unavailable_rows": 58, "persisted_email_held_rows": 614 }, "ledger_attempt_states": { "model": { "completed": 119, "uncertain": 1, "pending": 1 }, "apify": { "completed": 16 }, "millionverifier": { "completed": 793, "uncertain": 3 } }, "cost_evidence": { "ledger_completed_model_cost_usd": 0.36424070000000003, "ledger_email_allocations_usd": 3.1005, "ledger_completed_actor_cost_usd": 2.0512, "note": "Model and actor receipts have actual provider amounts. MillionVerifier uses fixed per-check credit allocation, and uncertain exposure remains reserved. The separately observed dashboard total is not refreshed by this audit." }, "ranked_bottlenecks_and_future_improvements": [ { "rank": 1, "finding": "Current resource throttling and the shared budget lock are the clearest immediate bottleneck.", "evidence": "At 06:49:58-06:50:03 UTC, process used 0.03 CPU seconds in 5.00 seconds. Six threads including the main thread waited in locks_lock_inode_wait, one waited in mem_cgroup_handle_over_high, two in futex. /proc/locks identified six queued waiters on inode825638, the run budget.lock. MemoryCurrent842039296 bytes exceeded MemoryHigh 805306368. Cgroup memory full PSI averaged66.45% over 60s, and memory.high events increased 629 during the 5-second sample.", "interpretation": "The worker holding or working through memory-heavy shared state can delay every budget reader and progress writer. This is direct resource/lock evidence, not proof of a permanent deadlock or attribution to one Python allocation.", "future_improvement": "Profile allocations in an isolated replay, share immutable exclusion snapshots across workers, and replace repeated whole-ledger parse/rewrites with bounded transaction records or one ledger owner. Set resource limits from measured working set and host headroom. Do not merely add workers." }, { "rank": 2, "finding": "A real OOM outage removed 1h48m07s of runtime.", "evidence": "Journal OOM kill 01:17:45.438451 UTC to recovered start03:05:52.534031 UTC. Earlier stop/restart gaps add 4m28.815s.", "interpretation": "The prior restart protection addressed availability after failure, but the current memory.high throttling remains visible.", "future_improvement": "Preserve the existing receipt-aware recovery while first reducing the working set and confirming sustained progress under the actual resource limit." }, { "rank": 3, "finding": "Eight candidate threads do not maintain eight model requests. The entire batch waits at barriers.", "evidence": "new2000.py:418-428 exits ThreadPoolExecutor only after every future completes. History scans and Maps acquisition run on the main loop outside that pool. Email checks, website work and local persistence occupy the same worker slots. State records peak 7, and the current FD inventory showed no Luna model-slot file open.", "interpretation": "A slow email/scrape/lock worker can leave paid model capacity unused. No complete slot-occupancy time series exists, so idle percentage is not known.", "future_improvement": "Use separately bounded source, verification, evidence and model queues with a rolling executor, retaining the same eight-call model ceiling and exact budget/reservation rules." }, { "rank": 4, "finding": "History refresh is a repeated full scan, costing about 7 minutes per complete pass.", "evidence": "Five complete saved passes plus one interrupted pass contain 1541 page receipts and 2471.141 seconds of non-overlapping fetch intervals. A typical full pass has 261 page reads. Sent-history alone takes 286-291 seconds with a 3.1-second inter-page delay.", "interpretation": "History correctness is necessary, but repeating all old pages blocks production. The interrupted 00:37 scan was redone in a new 00:44 directory.", "future_improvement": "Preserve the same history clearance while using a durable cursor/change watermark where supported, one shared immutable snapshot, and a final fresh reconciliation. Keep incomplete snapshots unusable." }, { "rank": 5, "finding": "Poor contact yield and free scraping of emails already held create avoidable work.", "evidence": "At the ledger snapshot, 793 completed MillionVerifier entries contained 163 ok, 276 catch_all, 178 invalid, 176 unknown, plus 3 uncertain attempts. The persisted review state held 614 rows before personalization. Current process_candidate still calls fresh.scrape on that held path. Sixteen paid Maps runs returned 512 raw records and appended 96 eligible-to-process candidates.", "interpretation": "Raw Maps records and verified contacts are different stages. The 96 appended rows are not 96 final ready leads. A source with many catch-all/unknown contacts cannot satisfy the strict ok-only target regardless of model speed.", "future_improvement": "Prioritize source/city combinations using verified contact yield and defer optional held-row website evidence. Retain existing scrape caches and deduplication." }, { "rank": 6, "finding": "Six paid model calls produced no JSON because Max reasoning consumed all 12000 completion tokens.", "evidence": "All six validation failures have finish_reason=length, native_finish_reason=max_output_tokens, completion_tokens=12000, reasoning_tokens=12000, and content=null. The Python error is json.loads(None), reported as a TypeError. Overall 549334 of 570099 completion tokens were reasoning tokens, 96.36%.", "interpretation": "The literal JSON parsing error obscures the actual output-cap exhaustion. This affected 6/119 completed paid responses, 5.04%, but does not explain the multi-hour slowdown alone.", "future_improvement": "Handle this finish reason explicitly and expose it in the review UI. Any output-limit/prompt/model tradeoff should be evaluated with a small authorized benchmark while preserving the requested Luna Max setting unless changed by the user. Do not automatically rebuy uncertain calls." }, { "rank": 7, "finding": "Six candidate rows are held as terminal outcomes after SQLite busy errors.", "evidence": "Current saved rows contain 6 OperationalError: database is locked. review_server.py uses a 15-second busy timeout and the wrapper marks exceptions terminal.", "interpretation": "Short-lived contention becomes a lost candidate for this run. It is separate from the current budget flock queue.", "future_improvement": "Serialize review writes or add a narrow retry of known local transactions with preserved paid receipts. Instrument the write queue and keep a single owner for each row." } ], "source_of_truth_paths": { "scripts": "/home/matt/.local/share/get-leads-review/scripts", "run": "/home/matt/.local/state/get-leads-review/clientsflow-new2000-20261004", "private_compact_evidence": "/Users/agency/Documents/Agty/ColdEmail/.tmp/get-leads-funnel-audit-20261004/runtime", "remote_compact_evidence": "/home/matt/.local/state/get-leads-review/audits/funnel-20261004/runtime" }, "limitations": [ "The code paths, receipts and one bounded process sample establish facts, not a continuous profiler trace.", "The full source funnel belongs to the companion source/stage audit. Ledger attempt counts and review-row counts are different denominators.", "No exact completion ETA can be supported.", "No 100% language or business-fit accuracy claim is made.", "No bulk company/contact inputs or credentials are included in publishable files." ], "public_documentation_check": { "url": "https://openrouter.ai/docs/api/api-reference/generations/get-generation", "finding": "The public reference lists generation_time. Its rendered field lacks explicit units, so raw metadata is retained without claiming documented seconds." } }