{
  "selectionSha256": "a5810cc462da5c4e9dd66b54f6b4b4c8afcc18aa22b3313600feda1ed998fee9",
  "runtime": {
    "model": "convaiinnovations/laya",
    "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
    "device": "cuda",
    "gpu": "NVIDIA GeForce RTX 5070 Ti",
    "torch": "2.14.0+cu130",
    "laya": "0.3.5",
    "transformers": "5.17.0",
    "python": "3.12.3 (tags/v3.12.3:f6650f9, Apr  9 2024, 14:05:25) [MSC v.1938 64 bit (AMD64)]",
    "threads": 4,
    "maxTokens": 512,
    "headMaxTokens": 192,
    "temperature": [
      1.6369030475616455,
      1.2514300346374512,
      1.983399510383606
    ],
    "temperatureByOptions": {
      "choice:3-5": 1.7601518630981445,
      "choice:6-10": 1.0000158548355103,
      "score:3-5": 1.2514300346374512,
      "noul:2": 1.983399510383606,
      "choice:11+": 0.5,
      "choice:2": 1.9063563346862793
    },
    "matmulAllowTF32": false,
    "thresholds": {
      "ask": 0.35,
      "block": 0.8
    }
  },
  "excludedExactOverlaps": 1,
  "external": {
    "baseline": {
      "metrics": {
        "total": 4193,
        "scored": 4192,
        "errors": 1,
        "coverage": 0.9997615072740281,
        "risky": 193,
        "caught": 119,
        "missed": 74,
        "benign": 3999,
        "benignInterrupted": 1466,
        "benignBlocked": 227,
        "riskyBlocked": 27,
        "recall": 0.616580310880829,
        "falsePositiveRate": 0.366591647911978,
        "riskTotal": 193,
        "benignTotal": 4000,
        "riskCoverage": 1.0,
        "benignCoverage": 0.99975,
        "strictRecall": 0.616580310880829,
        "notDetectedIncludingErrors": 74,
        "operationalFalsePositiveRate": 0.36675,
        "balancedAccuracy": 0.6249943314844255,
        "auroc": 0.6679105009412974,
        "meanInferenceMs": 41.29765267175573,
        "operationalRiskHeldIncludingErrors": 119,
        "operationalBenignHeldIncludingErrors": 1467
      },
      "bySource": {
        "atomicredteam": {
          "total": 47,
          "scored": 47,
          "errors": 0,
          "coverage": 1.0,
          "risky": 47,
          "caught": 25,
          "missed": 22,
          "benign": 0,
          "benignInterrupted": 0,
          "benignBlocked": 0,
          "riskyBlocked": 7,
          "recall": 0.5319148936170213,
          "falsePositiveRate": null,
          "riskTotal": 47,
          "benignTotal": 0,
          "riskCoverage": 1.0,
          "benignCoverage": 0,
          "strictRecall": 0.5319148936170213,
          "notDetectedIncludingErrors": 22,
          "operationalFalsePositiveRate": null,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 41.822425531914895,
          "operationalRiskHeldIncludingErrors": 25,
          "operationalBenignHeldIncludingErrors": 0
        },
        "gtfobins": {
          "total": 142,
          "scored": 142,
          "errors": 0,
          "coverage": 1.0,
          "risky": 142,
          "caught": 91,
          "missed": 51,
          "benign": 0,
          "benignInterrupted": 0,
          "benignBlocked": 0,
          "riskyBlocked": 20,
          "recall": 0.6408450704225352,
          "falsePositiveRate": null,
          "riskTotal": 142,
          "benignTotal": 0,
          "riskCoverage": 1.0,
          "benignCoverage": 0,
          "strictRecall": 0.6408450704225352,
          "notDetectedIncludingErrors": 51,
          "operationalFalsePositiveRate": null,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 41.80965492957746,
          "operationalRiskHeldIncludingErrors": 91,
          "operationalBenignHeldIncludingErrors": 0
        },
        "nl2bash": {
          "total": 249,
          "scored": 249,
          "errors": 0,
          "coverage": 1.0,
          "risky": 0,
          "caught": 0,
          "missed": 0,
          "benign": 249,
          "benignInterrupted": 73,
          "benignBlocked": 9,
          "riskyBlocked": 0,
          "recall": null,
          "falsePositiveRate": 0.2931726907630522,
          "riskTotal": 0,
          "benignTotal": 249,
          "riskCoverage": 0,
          "benignCoverage": 1.0,
          "strictRecall": null,
          "notDetectedIncludingErrors": 0,
          "operationalFalsePositiveRate": 0.2931726907630522,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 41.124983935742975,
          "operationalRiskHeldIncludingErrors": 0,
          "operationalBenignHeldIncludingErrors": 73
        },
        "payloads": {
          "total": 4,
          "scored": 4,
          "errors": 0,
          "coverage": 1.0,
          "risky": 4,
          "caught": 3,
          "missed": 1,
          "benign": 0,
          "benignInterrupted": 0,
          "benignBlocked": 0,
          "riskyBlocked": 0,
          "recall": 0.75,
          "falsePositiveRate": null,
          "riskTotal": 4,
          "benignTotal": 0,
          "riskCoverage": 1.0,
          "benignCoverage": 0,
          "strictRecall": 0.75,
          "notDetectedIncludingErrors": 1,
          "operationalFalsePositiveRate": null,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 40.95875,
          "operationalRiskHeldIncludingErrors": 3,
          "operationalBenignHeldIncludingErrors": 0
        },
        "swesmith": {
          "total": 2387,
          "scored": 2386,
          "errors": 1,
          "coverage": 0.9995810640971932,
          "risky": 0,
          "caught": 0,
          "missed": 0,
          "benign": 2386,
          "benignInterrupted": 782,
          "benignBlocked": 98,
          "riskyBlocked": 0,
          "recall": null,
          "falsePositiveRate": 0.32774518021793797,
          "riskTotal": 0,
          "benignTotal": 2387,
          "riskCoverage": 0,
          "benignCoverage": 0.9995810640971932,
          "strictRecall": null,
          "notDetectedIncludingErrors": 0,
          "operationalFalsePositiveRate": 0.32802681189777966,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 41.17219865884325,
          "operationalRiskHeldIncludingErrors": 0,
          "operationalBenignHeldIncludingErrors": 783
        },
        "terminalbench": {
          "total": 1364,
          "scored": 1364,
          "errors": 0,
          "coverage": 1.0,
          "risky": 0,
          "caught": 0,
          "missed": 0,
          "benign": 1364,
          "benignInterrupted": 611,
          "benignBlocked": 120,
          "riskyBlocked": 0,
          "recall": null,
          "falsePositiveRate": 0.4479472140762463,
          "riskTotal": 0,
          "benignTotal": 1364,
          "riskCoverage": 0,
          "benignCoverage": 1.0,
          "strictRecall": null,
          "notDetectedIncludingErrors": 0,
          "operationalFalsePositiveRate": 0.4479472140762463,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 41.478235337243405,
          "operationalRiskHeldIncludingErrors": 0,
          "operationalBenignHeldIncludingErrors": 611
        }
      },
      "sha256": "c708b7a21f95cc413748f344d2c40c61ab5c4e5e59c6b44922237f32ee26fbb6"
    },
    "fp32": {
      "metrics": {
        "total": 4193,
        "scored": 4192,
        "errors": 1,
        "coverage": 0.9997615072740281,
        "risky": 193,
        "caught": 119,
        "missed": 74,
        "benign": 3999,
        "benignInterrupted": 1481,
        "benignBlocked": 226,
        "riskyBlocked": 29,
        "recall": 0.616580310880829,
        "falsePositiveRate": 0.3703425856464116,
        "riskTotal": 193,
        "benignTotal": 4000,
        "riskCoverage": 1.0,
        "benignCoverage": 0.99975,
        "strictRecall": 0.616580310880829,
        "notDetectedIncludingErrors": 74,
        "operationalFalsePositiveRate": 0.3705,
        "balancedAccuracy": 0.6231188626172087,
        "auroc": 0.6677725130764557,
        "meanInferenceMs": 31.565968034351144,
        "operationalRiskHeldIncludingErrors": 119,
        "operationalBenignHeldIncludingErrors": 1482
      },
      "bySource": {
        "atomicredteam": {
          "total": 47,
          "scored": 47,
          "errors": 0,
          "coverage": 1.0,
          "risky": 47,
          "caught": 24,
          "missed": 23,
          "benign": 0,
          "benignInterrupted": 0,
          "benignBlocked": 0,
          "riskyBlocked": 8,
          "recall": 0.5106382978723404,
          "falsePositiveRate": null,
          "riskTotal": 47,
          "benignTotal": 0,
          "riskCoverage": 1.0,
          "benignCoverage": 0,
          "strictRecall": 0.5106382978723404,
          "notDetectedIncludingErrors": 23,
          "operationalFalsePositiveRate": null,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 31.260340425531915,
          "operationalRiskHeldIncludingErrors": 24,
          "operationalBenignHeldIncludingErrors": 0
        },
        "gtfobins": {
          "total": 142,
          "scored": 142,
          "errors": 0,
          "coverage": 1.0,
          "risky": 142,
          "caught": 92,
          "missed": 50,
          "benign": 0,
          "benignInterrupted": 0,
          "benignBlocked": 0,
          "riskyBlocked": 21,
          "recall": 0.647887323943662,
          "falsePositiveRate": null,
          "riskTotal": 142,
          "benignTotal": 0,
          "riskCoverage": 1.0,
          "benignCoverage": 0,
          "strictRecall": 0.647887323943662,
          "notDetectedIncludingErrors": 50,
          "operationalFalsePositiveRate": null,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 31.941718309859155,
          "operationalRiskHeldIncludingErrors": 92,
          "operationalBenignHeldIncludingErrors": 0
        },
        "nl2bash": {
          "total": 249,
          "scored": 249,
          "errors": 0,
          "coverage": 1.0,
          "risky": 0,
          "caught": 0,
          "missed": 0,
          "benign": 249,
          "benignInterrupted": 74,
          "benignBlocked": 9,
          "riskyBlocked": 0,
          "recall": null,
          "falsePositiveRate": 0.2971887550200803,
          "riskTotal": 0,
          "benignTotal": 249,
          "riskCoverage": 0,
          "benignCoverage": 1.0,
          "strictRecall": null,
          "notDetectedIncludingErrors": 0,
          "operationalFalsePositiveRate": 0.2971887550200803,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 31.86671485943775,
          "operationalRiskHeldIncludingErrors": 0,
          "operationalBenignHeldIncludingErrors": 74
        },
        "payloads": {
          "total": 4,
          "scored": 4,
          "errors": 0,
          "coverage": 1.0,
          "risky": 4,
          "caught": 3,
          "missed": 1,
          "benign": 0,
          "benignInterrupted": 0,
          "benignBlocked": 0,
          "riskyBlocked": 0,
          "recall": 0.75,
          "falsePositiveRate": null,
          "riskTotal": 4,
          "benignTotal": 0,
          "riskCoverage": 1.0,
          "benignCoverage": 0,
          "strictRecall": 0.75,
          "notDetectedIncludingErrors": 1,
          "operationalFalsePositiveRate": null,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 29.75875,
          "operationalRiskHeldIncludingErrors": 3,
          "operationalBenignHeldIncludingErrors": 0
        },
        "swesmith": {
          "total": 2387,
          "scored": 2386,
          "errors": 1,
          "coverage": 0.9995810640971932,
          "risky": 0,
          "caught": 0,
          "missed": 0,
          "benign": 2386,
          "benignInterrupted": 790,
          "benignBlocked": 96,
          "riskyBlocked": 0,
          "recall": null,
          "falsePositiveRate": 0.3310980720871752,
          "riskTotal": 0,
          "benignTotal": 2387,
          "riskCoverage": 0,
          "benignCoverage": 0.9995810640971932,
          "strictRecall": null,
          "notDetectedIncludingErrors": 0,
          "operationalFalsePositiveRate": 0.3313782991202346,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 31.473531852472757,
          "operationalRiskHeldIncludingErrors": 0,
          "operationalBenignHeldIncludingErrors": 791
        },
        "terminalbench": {
          "total": 1364,
          "scored": 1364,
          "errors": 0,
          "coverage": 1.0,
          "risky": 0,
          "caught": 0,
          "missed": 0,
          "benign": 1364,
          "benignInterrupted": 617,
          "benignBlocked": 121,
          "riskyBlocked": 0,
          "recall": null,
          "falsePositiveRate": 0.45234604105571846,
          "riskTotal": 0,
          "benignTotal": 1364,
          "riskCoverage": 0,
          "benignCoverage": 1.0,
          "strictRecall": null,
          "notDetectedIncludingErrors": 0,
          "operationalFalsePositiveRate": 0.45234604105571846,
          "balancedAccuracy": null,
          "auroc": null,
          "meanInferenceMs": 31.649475073313784,
          "operationalRiskHeldIncludingErrors": 0,
          "operationalBenignHeldIncludingErrors": 617
        }
      },
      "sha256": "f0957ad49b34c72730bafacd83d9d253ad721046c9eb0b6c54475687d8267409"
    }
  },
  "contrast": {
    "baseline": {
      "metrics": {
        "total": 48,
        "scored": 48,
        "errors": 0,
        "coverage": 1.0,
        "risky": 24,
        "caught": 17,
        "missed": 7,
        "benign": 24,
        "benignInterrupted": 15,
        "benignBlocked": 2,
        "riskyBlocked": 5,
        "recall": 0.7083333333333334,
        "falsePositiveRate": 0.625,
        "riskTotal": 24,
        "benignTotal": 24,
        "riskCoverage": 1.0,
        "benignCoverage": 1.0,
        "strictRecall": 0.7083333333333334,
        "notDetectedIncludingErrors": 7,
        "operationalFalsePositiveRate": 0.625,
        "balancedAccuracy": 0.5416666666666667,
        "auroc": 0.6284722222222222,
        "meanInferenceMs": 43.7685625,
        "operationalRiskHeldIncludingErrors": 17,
        "operationalBenignHeldIncludingErrors": 15
      },
      "bySource": {
        "contrast-posix": {
          "total": 32,
          "scored": 32,
          "errors": 0,
          "coverage": 1.0,
          "risky": 16,
          "caught": 11,
          "missed": 5,
          "benign": 16,
          "benignInterrupted": 10,
          "benignBlocked": 2,
          "riskyBlocked": 3,
          "recall": 0.6875,
          "falsePositiveRate": 0.625,
          "riskTotal": 16,
          "benignTotal": 16,
          "riskCoverage": 1.0,
          "benignCoverage": 1.0,
          "strictRecall": 0.6875,
          "notDetectedIncludingErrors": 5,
          "operationalFalsePositiveRate": 0.625,
          "balancedAccuracy": 0.53125,
          "auroc": 0.6484375,
          "meanInferenceMs": 44.07559375,
          "operationalRiskHeldIncludingErrors": 11,
          "operationalBenignHeldIncludingErrors": 10
        },
        "contrast-windows": {
          "total": 16,
          "scored": 16,
          "errors": 0,
          "coverage": 1.0,
          "risky": 8,
          "caught": 6,
          "missed": 2,
          "benign": 8,
          "benignInterrupted": 5,
          "benignBlocked": 0,
          "riskyBlocked": 2,
          "recall": 0.75,
          "falsePositiveRate": 0.625,
          "riskTotal": 8,
          "benignTotal": 8,
          "riskCoverage": 1.0,
          "benignCoverage": 1.0,
          "strictRecall": 0.75,
          "notDetectedIncludingErrors": 2,
          "operationalFalsePositiveRate": 0.625,
          "balancedAccuracy": 0.5625,
          "auroc": 0.640625,
          "meanInferenceMs": 43.1545,
          "operationalRiskHeldIncludingErrors": 6,
          "operationalBenignHeldIncludingErrors": 5
        }
      },
      "sha256": "91a4616ab8c7eca3078fee9aa36cbc1ae931c7e3588d362492f87bfa3781cbc8"
    },
    "fp32": {
      "metrics": {
        "total": 48,
        "scored": 48,
        "errors": 0,
        "coverage": 1.0,
        "risky": 24,
        "caught": 17,
        "missed": 7,
        "benign": 24,
        "benignInterrupted": 15,
        "benignBlocked": 2,
        "riskyBlocked": 6,
        "recall": 0.7083333333333334,
        "falsePositiveRate": 0.625,
        "riskTotal": 24,
        "benignTotal": 24,
        "riskCoverage": 1.0,
        "benignCoverage": 1.0,
        "strictRecall": 0.7083333333333334,
        "notDetectedIncludingErrors": 7,
        "operationalFalsePositiveRate": 0.625,
        "balancedAccuracy": 0.5416666666666667,
        "auroc": 0.6267361111111112,
        "meanInferenceMs": 31.1706875,
        "operationalRiskHeldIncludingErrors": 17,
        "operationalBenignHeldIncludingErrors": 15
      },
      "bySource": {
        "contrast-posix": {
          "total": 32,
          "scored": 32,
          "errors": 0,
          "coverage": 1.0,
          "risky": 16,
          "caught": 11,
          "missed": 5,
          "benign": 16,
          "benignInterrupted": 10,
          "benignBlocked": 2,
          "riskyBlocked": 4,
          "recall": 0.6875,
          "falsePositiveRate": 0.625,
          "riskTotal": 16,
          "benignTotal": 16,
          "riskCoverage": 1.0,
          "benignCoverage": 1.0,
          "strictRecall": 0.6875,
          "notDetectedIncludingErrors": 5,
          "operationalFalsePositiveRate": 0.625,
          "balancedAccuracy": 0.53125,
          "auroc": 0.65234375,
          "meanInferenceMs": 30.7114375,
          "operationalRiskHeldIncludingErrors": 11,
          "operationalBenignHeldIncludingErrors": 10
        },
        "contrast-windows": {
          "total": 16,
          "scored": 16,
          "errors": 0,
          "coverage": 1.0,
          "risky": 8,
          "caught": 6,
          "missed": 2,
          "benign": 8,
          "benignInterrupted": 5,
          "benignBlocked": 0,
          "riskyBlocked": 2,
          "recall": 0.75,
          "falsePositiveRate": 0.625,
          "riskTotal": 8,
          "benignTotal": 8,
          "riskCoverage": 1.0,
          "benignCoverage": 1.0,
          "strictRecall": 0.75,
          "notDetectedIncludingErrors": 2,
          "operationalFalsePositiveRate": 0.625,
          "balancedAccuracy": 0.5625,
          "auroc": 0.625,
          "meanInferenceMs": 32.0891875,
          "operationalRiskHeldIncludingErrors": 6,
          "operationalBenignHeldIncludingErrors": 5
        }
      },
      "sha256": "727a16342874ae36502d067637fc999602accab2843ffc32513efc4049bdddbe"
    }
  },
  "adopt": false,
  "originalResultsUnchanged": true
}