{ "protocol": "ISL-PRESENTATIONS-V1-DRAFT", "status": "Draft benchmark; no results yet", "check_count": 60, "area_count": 12, "overall_rating": null, "pilot_is_separate": true, "scoring": "Draft: each check 0–4 against a criterion-specific rubric locked before execution; category points = weight × check-score sum / 20. Do not publish an overall score until required coverage is complete. Not tested is not zero. Unsupported features must be documented; any applicability/weight change is fixed before comparison runs.", "repetition": "Three task variants, three fresh runs per variant for generation-dependent checks; publish every run. Use two blinded reviewers for subjective checks and resolve disagreements with recorded evidence.", "evidence": "Record date, tier, settings, model/build if disclosed, prompt/input fixtures, output IDs/files, screenshots and every grade. Purchases, account permissions and support contact require appropriate owner authorization.", "areas": [ { "id": "instructions", "title": "Instruction following", "proposed_weight": 10, "checks": [ { "id": "INSTRUCTIONS-01", "title": "Exact slide-count control", "method": "Generate the same six-slide brief using the explicit count setting and Auto separately, three fresh runs each.", "required_evidence": "Rendered count, cover/agenda behavior, settings and every output.", "status": "Not tested" }, { "id": "INSTRUCTIONS-02", "title": "Required headings and order", "method": "Provide six exact headings; compare every rendered title and position with the fixture.", "required_evidence": "Ordered title list and position-match percentage.", "status": "Not tested" }, { "id": "INSTRUCTIONS-03", "title": "Required content placement", "method": "Request a table, chart and limitations on specified slides; count covers and agenda consistently.", "required_evidence": "Rendered positions and screenshots of each required element.", "status": "Not tested" }, { "id": "INSTRUCTIONS-04", "title": "Formatting constraints", "method": "Request a fixed bullet count, word limit and prohibited phrases on designated slides.", "required_evidence": "Bullet counts, word counts and prohibited-phrase occurrences.", "status": "Not tested" }, { "id": "INSTRUCTIONS-05", "title": "Conflicting instructions", "method": "Supply a deliberately conflicting brief and a clear priority rule; observe whether the tool follows the rule or requests clarification.", "required_evidence": "Conflict resolution and unsupported decisions, with original prompt.", "status": "Not tested" } ] }, { "id": "facts", "title": "Factual accuracy", "proposed_weight": 15, "checks": [ { "id": "FACTS-01", "title": "Source-fact preservation", "method": "Supply a fictional brief with 20 locked facts, names, dates and amounts; compare output with an answer key.", "required_evidence": "Correct facts out of 20, omissions and alterations.", "status": "Not tested" }, { "id": "FACTS-02", "title": "Unsupported claims", "method": "Use a brief with intentionally limited information and prohibit invented facts.", "required_evidence": "Every unsupported factual claim and its slide location.", "status": "Not tested" }, { "id": "FACTS-03", "title": "Missing-information handling", "method": "Omit figures needed for a conclusion and explicitly prohibit estimation.", "required_evidence": "Whether gaps are disclosed, questions are asked or numbers invented.", "status": "Not tested" }, { "id": "FACTS-04", "title": "Citation and link validity", "method": "Provide a set of controlled source URLs and require source attribution; inspect cited links and supporting passages.", "required_evidence": "Working links, supported claims and fabricated citations.", "status": "Not tested" }, { "id": "FACTS-05", "title": "PDF extraction accuracy", "method": "Upload an authorized synthetic PDF containing known paragraphs, tables and footnotes, when the tested tier supports it.", "required_evidence": "Extracted facts versus fixture; missing or distorted table/footnote content.", "status": "Not tested" } ] }, { "id": "math", "title": "Calculations and data integrity", "proposed_weight": 10, "checks": [ { "id": "MATH-01", "title": "Growth calculations", "method": "Use locked positive, negative and zero-baseline examples with known answers.", "required_evidence": "Correct calculations, units, rounding and zero-baseline handling.", "status": "Not tested" }, { "id": "MATH-02", "title": "Totals and percentages", "method": "Provide a synthetic dataset with totals and percentage breakdowns that must reconcile.", "required_evidence": "Absolute errors and whether percentages/totals reconcile.", "status": "Not tested" }, { "id": "MATH-03", "title": "Money and unit consistency", "method": "Mix monthly/annual costs, two currencies and thousands/millions with explicit conversion rules.", "required_evidence": "Unit labels, currency labels and conversion accuracy.", "status": "Not tested" }, { "id": "MATH-04", "title": "Chart-to-table agreement", "method": "Require a chart and table from the identical dataset.", "required_evidence": "Every chart label/value compared with the source table.", "status": "Not tested" }, { "id": "MATH-05", "title": "Targets versus actual results", "method": "Mix achieved results, forecasts and goals, explicitly labeled in the fixture.", "required_evidence": "Misclassification count and visibility of forecast/target labels.", "status": "Not tested" } ] }, { "id": "writing", "title": "Writing and communication", "proposed_weight": 8, "checks": [ { "id": "WRITING-01", "title": "Audience adaptation", "method": "Generate the same brief for executives and beginners; evaluate against a rubric fixed before generation.", "required_evidence": "Blind rubric scores for terminology, explanation and audience fit.", "status": "Not tested" }, { "id": "WRITING-02", "title": "Narrative flow", "method": "Use a problem-evidence-options-recommendation brief and inspect the causal sequence.", "required_evidence": "Sequence adherence, transitions and unexplained conclusions.", "status": "Not tested" }, { "id": "WRITING-03", "title": "Conciseness and readability", "method": "Specify slide word limits and readability requirements on a fixed source brief.", "required_evidence": "Words per slide, repeated content and predefined readability checks.", "status": "Not tested" }, { "id": "WRITING-04", "title": "Technical explanation", "method": "Provide a technical passage with an answer key and ask for an accessible explanation.", "required_evidence": "Key concepts retained and meaning-changing simplifications.", "status": "Not tested" }, { "id": "WRITING-05", "title": "Multilingual fidelity", "method": "Translate the same locked brief into specified languages and use qualified reviewers.", "required_evidence": "Name/number preservation and reviewer-graded meaning, fluency and terminology.", "status": "Not tested" } ] }, { "id": "design", "title": "Visual design and layout", "proposed_weight": 8, "checks": [ { "id": "DESIGN-01", "title": "Overflow and clipping", "method": "Generate short, medium and long text fixtures; inspect at fixed display dimensions.", "required_evidence": "Clipped text, collisions and elements outside slide bounds.", "status": "Not tested" }, { "id": "DESIGN-02", "title": "Typography and hierarchy", "method": "Assess title/body/caption styling using a predeclared readability rubric.", "required_evidence": "Font sizes, hierarchy consistency and unreadable labels.", "status": "Not tested" }, { "id": "DESIGN-03", "title": "Theme consistency", "method": "Generate multiple slides with the same chosen theme and review spacing, color and alignment.", "required_evidence": "Style deviations across the deck, supported by screenshots.", "status": "Not tested" }, { "id": "DESIGN-04", "title": "Brand adherence", "method": "When supported, supply a test logo, palette and fonts owned by iSync.", "required_evidence": "Exact palette/font/logo placement checks and unsupported controls.", "status": "Not tested" }, { "id": "DESIGN-05", "title": "Image relevance", "method": "Use five synthetic scenarios and grade supplied/generated imagery against the brief.", "required_evidence": "Relevant images, misleading imagery and documented provenance where available.", "status": "Not tested" } ] }, { "id": "charts", "title": "Charts, tables and diagrams", "proposed_weight": 7, "checks": [ { "id": "CHARTS-01", "title": "Table completeness", "method": "Request a table with fixed rows, columns, labels and values.", "required_evidence": "Missing cells, changed values and visual legibility.", "status": "Not tested" }, { "id": "CHARTS-02", "title": "Chart suitability", "method": "Use time-series, categorical and proportion datasets with permitted chart types defined in advance.", "required_evidence": "Chart-type match and whether the chosen representation misleads.", "status": "Not tested" }, { "id": "CHARTS-03", "title": "Axes and labels", "method": "Require units, labels, scales and source notes for each chart.", "required_evidence": "Correct axes, omitted units and misleading truncation.", "status": "Not tested" }, { "id": "CHARTS-04", "title": "Dense data readability", "method": "Supply a controlled larger table and inspect both displayed and exported slides.", "required_evidence": "Readable values, overlap and handling of overflow.", "status": "Not tested" }, { "id": "CHARTS-05", "title": "Diagram relationship accuracy", "method": "Request a flowchart from a locked process with branches and exceptions.", "required_evidence": "Correct nodes, edges, branch labels and omitted exceptions.", "status": "Not tested" } ] }, { "id": "editing", "title": "Editing and user control", "proposed_weight": 7, "checks": [ { "id": "EDITING-01", "title": "Outline editing", "method": "Change a heading and reorder a section before generating the deck.", "required_evidence": "Whether final output preserves the saved edits and order.", "status": "Not tested" }, { "id": "EDITING-02", "title": "Slide-level revision", "method": "Request a change to one slide while explicitly preserving all others.", "required_evidence": "Target change completed and unintended changes elsewhere.", "status": "Not tested" }, { "id": "EDITING-03", "title": "Regeneration stability", "method": "Regenerate a selected element with facts and numbers declared invariant.", "required_evidence": "Preserved facts, retained layout and unsupported changes.", "status": "Not tested" }, { "id": "EDITING-04", "title": "Undo and recovery", "method": "When offered, make and undo a sequence of edits on a disposable test deck.", "required_evidence": "Recoverable states, restored content and lost edits.", "status": "Not tested" }, { "id": "EDITING-05", "title": "Save and reopen", "method": "Save an authorized test deck, close the view and reopen it through the product UI.", "required_evidence": "Content, styling and notes retained after reopening.", "status": "Not tested" } ] }, { "id": "exports", "title": "Exports and compatibility", "proposed_weight": 8, "checks": [ { "id": "EXPORTS-01", "title": "PowerPoint fidelity", "method": "Export an authorized test deck to PPTX on a tier that permits it; inspect it in PowerPoint or another declared viewer.", "required_evidence": "Text, layout, charts, notes and editability versus browser output.", "status": "Not tested" }, { "id": "EXPORTS-02", "title": "PDF fidelity", "method": "Export the same deck to PDF and inspect every page.", "required_evidence": "Page count, clipped elements, fonts and text selection.", "status": "Not tested" }, { "id": "EXPORTS-03", "title": "Google Slides fidelity", "method": "If available, import/export the test deck through an authorized Google Slides account.", "required_evidence": "Layout and editability differences from the source deck.", "status": "Not tested" }, { "id": "EXPORTS-04", "title": "Font and media portability", "method": "Open exported files on a second declared environment without the original fonts installed.", "required_evidence": "Fallback fonts, missing images and broken media.", "status": "Not tested" }, { "id": "EXPORTS-05", "title": "Notes and link preservation", "method": "Include known speaker notes and links, then inspect each supported export.", "required_evidence": "Notes retained, links functional and content in the correct layer.", "status": "Not tested" } ] }, { "id": "reliability", "title": "Speed and repeatability", "proposed_weight": 7, "checks": [ { "id": "RELIABILITY-01", "title": "Generation time", "method": "Measure from submission to usable completed deck across repeated locked tasks.", "required_evidence": "Instrumented timing, median, range and environment/tier.", "status": "Not tested" }, { "id": "RELIABILITY-02", "title": "Fresh-run consistency", "method": "Run three predefined task variants three times each without selecting the best output.", "required_evidence": "All nine results and variation in checklist scores.", "status": "Not tested" }, { "id": "RELIABILITY-03", "title": "Failure and retry behavior", "method": "Record normal failures during the planned runs and exercise documented retry controls.", "required_evidence": "Failure rate, error messages, recovery and duplicate outputs.", "status": "Not tested" }, { "id": "RELIABILITY-04", "title": "Long-brief handling", "method": "Use increasingly long synthetic briefs within documented product limits.", "required_evidence": "Completion, omissions, truncation and time by input length.", "status": "Not tested" }, { "id": "RELIABILITY-05", "title": "Version-aware retesting", "method": "Repeat the locked suite after a documented product update or at the scheduled retest date.", "required_evidence": "Configuration/version evidence and comparable historical outputs.", "status": "Not tested" } ] }, { "id": "usability", "title": "Usability and workflow", "proposed_weight": 4, "checks": [ { "id": "USABILITY-01", "title": "First-run completion", "method": "Have a first-time tester complete a declared task without assistance.", "required_evidence": "Time, errors, help required and completed steps.", "status": "Not tested" }, { "id": "USABILITY-02", "title": "Settings discoverability", "method": "Ask testers to locate slide count, theme, language and export controls that exist on the tier.", "required_evidence": "Task completion and navigation steps.", "status": "Not tested" }, { "id": "USABILITY-03", "title": "Mobile workflow", "method": "Generate, inspect and edit where supported on a declared mobile viewport/device.", "required_evidence": "Usable controls, blocked steps and display defects.", "status": "Not tested" }, { "id": "USABILITY-04", "title": "Plan-limit clarity", "method": "Compare displayed generation/download limits with actual behavior on the authorized tier.", "required_evidence": "Clear limits, blocked actions and unexpected restrictions.", "status": "Not tested" }, { "id": "USABILITY-05", "title": "Support response", "method": "With owner authorization, submit one ordinary support question about an observed issue.", "required_evidence": "Response time, relevance and whether the proposed fix works.", "status": "Not tested" } ] }, { "id": "accessibility", "title": "Accessibility", "proposed_weight": 4, "checks": [ { "id": "ACCESSIBILITY-01", "title": "Keyboard navigation", "method": "Complete the declared core workflow using only the keyboard.", "required_evidence": "Reachable controls, focus visibility, traps and completion.", "status": "Not tested" }, { "id": "ACCESSIBILITY-02", "title": "Screen-reader labels", "method": "Inspect the core UI and generated document with a named screen reader.", "required_evidence": "Meaningful control names, reading order and missing labels.", "status": "Not tested" }, { "id": "ACCESSIBILITY-03", "title": "Text contrast", "method": "Measure text/background combinations on generated slides against a declared contrast threshold.", "required_evidence": "Measured ratios and failures by element.", "status": "Not tested" }, { "id": "ACCESSIBILITY-04", "title": "Reading order and alternatives", "method": "Inspect exported reading order and alternative text where supported.", "required_evidence": "Logical reading sequence and informative image alternatives.", "status": "Not tested" }, { "id": "ACCESSIBILITY-05", "title": "Zoom and small-screen legibility", "method": "Inspect the UI at 200% zoom and the deck on a small screen.", "required_evidence": "Lost controls, overlap and unreadable content.", "status": "Not tested" } ] }, { "id": "privacy", "title": "Privacy controls and documentation", "proposed_weight": 2, "checks": [ { "id": "PRIVACY-01", "title": "Disclosure consistency", "method": "Compare the operator identity, privacy terms and in-product data notices.", "required_evidence": "Contradictions or unclear statements, labeled as documentation findings.", "status": "Not tested" }, { "id": "PRIVACY-02", "title": "Sharing-default visibility", "method": "Create a synthetic deck and inspect default sharing controls and public/private labels.", "required_evidence": "Who can access the deck under each available setting.", "status": "Not tested" }, { "id": "PRIVACY-03", "title": "Deletion-control behavior", "method": "Use only an explicitly disposable synthetic deck and the documented deletion workflow.", "required_evidence": "UI confirmation and subsequent accessibility under the same account.", "status": "Not tested" }, { "id": "PRIVACY-04", "title": "Retention and training options", "method": "Inspect available data-retention and model-training preferences without assuming vendor claims are verified.", "required_evidence": "Displayed choices, changes applied and remaining uncertainty.", "status": "Not tested" }, { "id": "PRIVACY-05", "title": "Sensitive-data guidance", "method": "Inspect warnings and instructions about submitting private material; submit only synthetic fixtures.", "required_evidence": "Clear user guidance and observable controls; no security-assurance conclusion.", "status": "Not tested" } ] } ] }