{
  "commit": "84a145aae7360e7d8d8b4d6db6ec05ca7bee6104",
  "example": "examples/fiserv-premier-11.0.0-prm",
  "files": [
    "premier_demo/host.py",
    "premier_demo/rasa_adapter.py"
  ],
  "fileSha256Original": {
    "host": "6fc8ce9ef580ee752cddce0bde93087ee2110294043f325080ca0f9069702cfc",
    "adapter": "8c08c2558c270cd61e1d795150253c4c46d024565839247f9061045658de9954"
  },
  "filesRestoredIdentical": true,
  "suiteBefore": 154,
  "suiteWithNewTests": 158,
  "skipped": 0,
  "command": "uv run --locked --group models python -m unittest discover -s tests -v",
  "environment": "A real temporary directory, the unmodified official Fiserv Premier ZIP in place.",
  "scope": "Offline unit tests of the example code. No model, speech service or bank. The reply-policy layer (lib/reply_policy.py) was not changed. Counts are distinct failing test names.",
  "newTests": "Merged in RasaHQ/rasa-community-resources pull request 255 at commit 0e90ad8f3083c697d000913b9d4a54cf559a87c9 (tests/test_flow.py only).",
  "rows": [
    {
      "change": "baseline",
      "failingBefore": 0,
      "testsBefore": [],
      "failingWithNewTests": 0,
      "testsWithNewTests": []
    },
    {
      "change": "Drop the review-time hold (review_transfer)",
      "failingBefore": 1,
      "testsBefore": [
        "test_unknown_outcome_holds_the_principal_across_sessions_and_restart"
      ],
      "failingWithNewTests": 1,
      "testsWithNewTests": [
        "test_unknown_outcome_holds_the_principal_across_sessions_and_restart"
      ]
    },
    {
      "change": "Drop the send-time hold (Journal.mark_pending)",
      "failingBefore": 0,
      "testsBefore": [],
      "failingWithNewTests": 2,
      "testsWithNewTests": [
        "test_a_review_made_before_an_unknown_outcome_cannot_be_sent_after_it",
        "test_the_journal_refuses_a_second_marker_while_an_outcome_is_unknown"
      ]
    },
    {
      "change": "The journal stops holding after an unknown (Journal.held)",
      "failingBefore": 1,
      "testsBefore": [
        "test_unknown_outcome_holds_the_principal_across_sessions_and_restart"
      ],
      "failingWithNewTests": 3,
      "testsWithNewTests": [
        "test_a_review_made_before_an_unknown_outcome_cannot_be_sent_after_it",
        "test_the_journal_refuses_a_second_marker_while_an_outcome_is_unknown",
        "test_unknown_outcome_holds_the_principal_across_sessions_and_restart"
      ]
    },
    {
      "change": "Drop both host holds",
      "failingBefore": 1,
      "testsBefore": [
        "test_unknown_outcome_holds_the_principal_across_sessions_and_restart"
      ],
      "failingWithNewTests": 3,
      "testsWithNewTests": [
        "test_a_review_made_before_an_unknown_outcome_cannot_be_sent_after_it",
        "test_the_journal_refuses_a_second_marker_while_an_outcome_is_unknown",
        "test_unknown_outcome_holds_the_principal_across_sessions_and_restart"
      ]
    },
    {
      "change": "Drop the unknown-first wording rule (adapter)",
      "failingBefore": 1,
      "testsBefore": [
        "test_an_unknown_earlier_outcome_outranks_an_expired_verification"
      ],
      "failingWithNewTests": 1,
      "testsWithNewTests": [
        "test_an_unknown_earlier_outcome_outranks_an_expired_verification"
      ]
    },
    {
      "change": "Forget the journal status after a restart (adapter)",
      "failingBefore": 3,
      "testsBefore": [
        "test_a_status_that_was_never_learned_is_never_cleared_after_a_restart",
        "test_an_unknown_earlier_outcome_outranks_an_expired_verification",
        "test_an_unknown_outcome_held_across_a_restart_is_never_retried"
      ],
      "failingWithNewTests": 3,
      "testsWithNewTests": [
        "test_a_status_that_was_never_learned_is_never_cleared_after_a_restart",
        "test_an_unknown_earlier_outcome_outranks_an_expired_verification",
        "test_an_unknown_outcome_held_across_a_restart_is_never_retried"
      ]
    },
    {
      "change": "Drop every hold and both wording rules",
      "failingBefore": 4,
      "testsBefore": [
        "test_a_status_that_was_never_learned_is_never_cleared_after_a_restart",
        "test_an_unknown_earlier_outcome_outranks_an_expired_verification",
        "test_an_unknown_outcome_held_across_a_restart_is_never_retried",
        "test_unknown_outcome_holds_the_principal_across_sessions_and_restart"
      ],
      "failingWithNewTests": 6,
      "testsWithNewTests": [
        "test_a_review_made_before_an_unknown_outcome_cannot_be_sent_after_it",
        "test_a_status_that_was_never_learned_is_never_cleared_after_a_restart",
        "test_an_unknown_earlier_outcome_outranks_an_expired_verification",
        "test_an_unknown_outcome_held_across_a_restart_is_never_retried",
        "test_the_journal_refuses_a_second_marker_while_an_outcome_is_unknown",
        "test_unknown_outcome_holds_the_principal_across_sessions_and_restart"
      ]
    }
  ],
  "typedRuns": {
    "revision": "84a145aae7360e7d8d8b4d6db6ec05ca7bee6104",
    "seed": 0,
    "channel": "Inspector, typed, real model, no microphone",
    "makeTest": {
      "tests": 154,
      "failures": 0,
      "skips": 0,
      "seconds": 71.433
    },
    "cases": [
      {
        "case": "over-balance confirm",
        "inputs": [
          "Verify me.",
          "What is my available checking balance?",
          "Move 2000 dollars from checking-A to bills-A.",
          "confirm transfer"
        ],
        "finalReply": "This transfer was refused, and nothing was sent.",
        "journal": [
          [
            "refused",
            1
          ]
        ],
        "note": "HTTP 422 seen in the server log"
      },
      {
        "case": "expired verification",
        "verifyToRequestSeconds": 123,
        "finalReply": "Your verification has expired, so this transfer was not reviewed and nothing was sent. Say verify me, then ask for the transfer again.",
        "reviewResult": {
          "state": "refused",
          "reason": "assurance_required"
        }
      },
      {
        "case": "outside scope",
        "inputs": [
          "Verify me.",
          "Move 25 dollars from checking to savings."
        ],
        "finalReply": "That transfer was not reviewed, and nothing was sent: only transfers between your own listed accounts are supported.",
        "reviewResult": {
          "state": "refused",
          "reason": "account_unavailable"
        },
        "journal": []
      },
      {
        "case": "lost reply (--fault transfer-response-lost)",
        "inputs": [
          "Verify me.",
          "Move 25 dollars from checking to bills.",
          "Confirm transfer.",
          "Move 25 dollars from checking to bills."
        ],
        "replyAfterConfirm": "The earlier transfer outcome is unknown. Do not retry. Check the original transfer status.",
        "replyAfterNewRequest": "The earlier transfer outcome is unknown. Do not retry. Check the original transfer status.",
        "journal": [
          [
            1,
            "unknown",
            1
          ]
        ],
        "note": "two submit-tool calls one second apart; the second returned unknown with reason no_retry_check_original_status; after almost five minutes the review tool returned refused/assurance_required and the caller still heard the unknown sentence"
      },
      {
        "case": "restart after unknown",
        "replyAfterNewRequest": "The earlier transfer outcome is unknown. Do not retry. Check the original transfer status.",
        "journal": [
          [
            1,
            "unknown",
            1
          ]
        ],
        "precedenceObserved": {
          "secondsAfterVerification": 80.727,
          "secondsAfterVerificationClock": "tool-call log: verification result 07:02:54.529Z to final request 07:04:15.256Z; the typed-run log gives 81.484 seconds (07:02:53.575Z to 07:04:15.059Z); an earlier version of this file said 69.171, which the raw logs do not support",
          "reply": "The earlier transfer outcome is unknown. Do not retry. Check the original transfer status.",
          "limit": "The model sent that final reply without calling a banking tool in the turn; the reply is observed, the backend path is not claimed."
        }
      },
      {
        "case": "reset",
        "command": "uv run --locked python -m premier_demo.reset_fixture --reason \"starting over\" --acknowledge-unresolved",
        "exit": 0,
        "stdout": "New fictional exercise started (epoch 2); the audit row was written.",
        "journalAfter": [
          [
            1,
            "historical_unknown",
            1
          ],
          [
            2,
            "accepted",
            1
          ]
        ]
      }
    ],
    "limits": "Typed runs only. No spoken run, no response times. Ctrl+C stop of the launcher was not claimed as a zero exit."
  }
}