Show field mapping caveats in template issues

This commit is contained in:
Paul Huliganga 2026-04-23 09:15:49 -04:00
parent beede0e497
commit 210f273c05
9 changed files with 224 additions and 65 deletions

View File

@ -47,7 +47,7 @@ The pipeline is extended with a FastAPI web layer that wraps all existing src/ m
graph TD
Browser -->|HTTP| FastAPI
FastAPI -->|OAuth| AdobeSign[Adobe Sign API]
FastAPI -->|OAuth/JWT| DocuSign[DocuSign API]
FastAPI -->|OAuth| DocuSign[DocuSign API]
FastAPI -->|calls| Compose[compose_docusign_template.py]
FastAPI -->|calls| Upload[upload_docusign_template.py]
Upload -->|upsert| DocuSign
@ -60,6 +60,19 @@ graph TD
- `web/routers/migrate.py` — triggers pipeline; records history
- `web/static/` — vanilla HTML/JS SPA (no build step)
**Template issue status:**
`GET /api/templates/status` drives the Templates and Issues & Warnings pages.
Its summary status combines pre-migration validation and DocuSign composition
analysis:
- `blockers`: validation failures that stop migration.
- `warnings`: validation warnings that allow migration but need review.
- `field_issues`: field mapping caveats emitted by composition, such as skipped
field types or unsupported conditional logic.
The list-level "Clean" label should only appear when all three collections are
empty, so summary rows match the template detail and migration result views.
**Idempotent Upload (v2):**
`upload_docusign_template.py` now searches for an existing DocuSign template by exact name match and updates the most recently modified one (PUT). Falls back to create (POST) if no match. `--force-create` flag bypasses upsert.

View File

@ -11,8 +11,8 @@ Validates that compose_docusign_template.py produces correctly structured output
"""
import json
import sys
import tempfile
from pathlib import Path
from pprint import pprint
# Support running from src/ or project root
BASE = Path(__file__).parent.parent
@ -21,14 +21,10 @@ sys.path.insert(0, str(Path(__file__).parent))
from compose_docusign_template import compose_template
def test_onboarding_mapping():
output_path = BASE / "validation" / "compose-doc-template-complete.json"
compose_template(
fields_path=str(BASE / "sample-templates" / "onboarding-template-formfields.json"),
template_meta_path=str(BASE / "sample-templates" / "onboarding-template.json"),
pdf_b64_path=str(BASE / "sample-templates" / "onboarding-sample.pdf.b64"),
output_path=str(output_path),
)
def test_onboarding_mapping(tmp_path):
template_dir = BASE / "downloads" / "David Tag Demo Form__CBJCHBCA"
output_path = tmp_path / "compose-doc-template-complete.json"
compose_template(str(template_dir), str(output_path))
template = json.loads(output_path.read_text())
@ -36,10 +32,9 @@ def test_onboarding_mapping():
assert "status" not in template, "Template must not have a top-level 'status' field"
signers = template["recipients"]["signers"]
assert len(signers) == 2, f"Expected 2 signers, got {len(signers)}"
assert len(signers) == 1, f"Expected 1 signer, got {len(signers)}"
signer0_tabs = signers[0]["tabs"]
signer1_tabs = signers[1]["tabs"]
# -- No email/name on role placeholders --
for s in signers:
@ -53,8 +48,6 @@ def test_onboarding_mapping():
assert "checkboxTabs" in signer0_tabs, "Signer 0 missing checkboxTabs"
assert "radioGroupTabs" in signer0_tabs, "Signer 0 missing radioGroupTabs"
assert "signHereTabs" in signer0_tabs, "Signer 0 missing signHereTabs"
assert "textTabs" in signer1_tabs, "Signer 1 missing textTabs"
assert "signHereTabs" in signer1_tabs, "Signer 1 missing signHereTabs"
# -- required / locked are strings --
for tab in signer0_tabs.get("textTabs", []):
@ -73,7 +66,7 @@ def test_onboarding_mapping():
radio_tab = signer0_tabs["radioGroupTabs"][0]
assert "groupName" in radio_tab, "radioGroupTab missing groupName"
assert "radios" in radio_tab, "radioGroupTab missing radios"
assert len(radio_tab["radios"]) == 3, f"Expected 3 radios, got {len(radio_tab['radios'])}"
assert len(radio_tab["radios"]) >= 1, "Expected at least one radio option"
for r in radio_tab["radios"]:
assert "pageNumber" in r, "radio missing pageNumber"
assert "xPosition" in r, "radio missing xPosition"
@ -87,18 +80,11 @@ def test_onboarding_mapping():
+ signer0_tabs.get("signHereTabs", [])
+ signer0_tabs.get("listTabs", [])
+ signer0_tabs.get("checkboxTabs", [])
+ signer1_tabs.get("textTabs", [])
+ signer1_tabs.get("signHereTabs", [])
)
for tab in all_single_tabs:
for field in ("documentId", "pageNumber", "xPosition", "yPosition"):
assert field in tab, f"Tab '{tab.get('tabLabel')}' missing '{field}'"
print("✅ All mapping assertions passed!")
print("\n--- Generated template (recipients section) ---")
pprint(template["recipients"])
if __name__ == "__main__":
test_onboarding_mapping()
with tempfile.TemporaryDirectory() as tmpdir:
test_onboarding_mapping(Path(tmpdir))

View File

@ -159,7 +159,7 @@ def test_status_needs_update():
@respx.mock
def test_status_includes_blockers_and_warnings_fields():
"""Each template in the status response has blockers and warnings keys."""
"""Each template in the status response has issue-analysis keys."""
respx.get(f"{ADOBE_BASE}/libraryDocuments").mock(
return_value=httpx.Response(200, json={
"libraryDocumentList": [
@ -175,13 +175,16 @@ def test_status_includes_blockers_and_warnings_fields():
t = resp.json()["templates"][0]
assert "blockers" in t
assert "warnings" in t
assert "field_issues" in t
assert "analysis_status" in t
assert isinstance(t["blockers"], list)
assert isinstance(t["warnings"], list)
assert isinstance(t["field_issues"], list)
@respx.mock
def test_status_empty_blockers_when_not_downloaded():
"""Template not in downloads dir → blockers and warnings are empty lists."""
"""Template not in downloads dir → no local template analysis issues."""
respx.get(f"{ADOBE_BASE}/libraryDocuments").mock(
return_value=httpx.Response(200, json={
"libraryDocumentList": [
@ -196,6 +199,8 @@ def test_status_empty_blockers_when_not_downloaded():
t = resp.json()["templates"][0]
assert t["blockers"] == []
assert t["warnings"] == []
assert t["field_issues"] == []
assert t["analysis_status"] == "not_downloaded"
@respx.mock
@ -229,3 +234,55 @@ def test_status_blockers_populated_when_template_downloaded(tmp_path, monkeypatc
# blockers and warnings are lists (may be empty if downloads path not resolved in test)
assert isinstance(t["blockers"], list)
assert isinstance(t["warnings"], list)
@respx.mock
def test_status_includes_field_issues_when_template_has_mapping_caveats(tmp_path, monkeypatch):
"""Composition caveats are surfaced in the template summary, not only migration results."""
import json
import web.config as cfg
template_dir = tmp_path / "Contract__adobe-field-warning"
template_dir.mkdir()
(template_dir / "metadata.json").write_text(json.dumps({"name": "Contract", "id": "adobe-field-warning"}))
(template_dir / "documents.json").write_text(json.dumps({"documents": [{"name": "contract.pdf"}]}))
(template_dir / "contract.pdf").write_bytes(b"%PDF-1.4\n% test\n")
(template_dir / "form_fields.json").write_text(json.dumps({
"fields": [
{
"name": "approval_toggle",
"inputType": "CHECKBOX",
"assignee": "recipient0",
"locations": [{"pageNumber": 1, "left": 20, "top": 20, "width": 20, "height": 20}],
},
{
"name": "conditional_notes",
"inputType": "TEXT_FIELD",
"contentType": "DATA",
"assignee": "recipient0",
"locations": [{"pageNumber": 1, "left": 50, "top": 50, "width": 80, "height": 20}],
"conditionalAction": {
"action": "HIDE",
"predicates": [{"fieldName": "approval_toggle", "operator": "EQUALS", "value": "on"}],
},
},
],
}))
monkeypatch.setattr(cfg.settings, "downloads_dir", str(tmp_path))
respx.get(f"{ADOBE_BASE}/libraryDocuments").mock(
return_value=httpx.Response(200, json={
"libraryDocumentList": [
{"id": "adobe-field-warning", "name": "Contract", "modifiedDate": "2026-04-10"},
]
})
)
respx.get(f"{DS_BASE}/v2.1/accounts/{DS_ACCOUNT}/templates").mock(
return_value=httpx.Response(200, json={"envelopeTemplates": []})
)
resp = client.get("/api/templates/status", cookies={_COOKIE_NAME: _adobe_session()})
assert resp.status_code == 200
t = resp.json()["templates"][0]
assert t["analysis_status"] == "analyzed"
assert any(issue["code"] == "HIDE_ACTION" for issue in t["field_issues"])

View File

@ -65,6 +65,7 @@ class Settings:
version: str = "2.0"
build_id: str = _detect_build_id("dev")
asset_version: str = os.getenv("ASSET_VERSION", build_id)
downloads_dir: str = os.getenv("DOWNLOADS_DIR", os.path.abspath(os.path.join(_project_root(), "downloads")))
@property
def admin_emails(self) -> set[str]:

View File

@ -7,6 +7,7 @@ Computes per-template migration status for the side-by-side UI.
from datetime import datetime, timezone
from pathlib import Path
import tempfile
from typing import Optional
import httpx
@ -159,7 +160,7 @@ async def template_status(request: Request):
# needs_update if Adobe was modified after the DS template
status = "needs_update" if adobe_modified > ds_modified else "migrated"
blockers, warnings = _get_validation(t.get("id", ""), name)
analysis = _get_template_analysis(t.get("id", ""), name)
results.append({
"adobe_id": t.get("id"),
@ -168,35 +169,80 @@ async def template_status(request: Request):
"docusign_id": ds_match.get("templateId") if ds_match else None,
"docusign_modified": ds_match.get("lastModified") if ds_match else None,
"status": status,
"blockers": blockers,
"warnings": warnings,
"blockers": analysis["blockers"],
"warnings": analysis["warnings"],
"field_issues": analysis["field_issues"],
"analysis_status": analysis["status"],
})
return {"templates": results}
def _get_validation(template_id: str, template_name: str) -> tuple[list, list]:
"""Return (blockers, warnings) if the template has been downloaded; else ([], [])."""
def _get_template_analysis(template_id: str, template_name: str) -> dict:
"""
Return validation and composition issues for a downloaded template.
Validation blockers/warnings answer "can this migrate at all?"
Field issues answer "what mapping caveats would migration introduce?"
If the template has not been downloaded yet, there is no local field data to analyze.
"""
analysis = {
"blockers": [],
"warnings": [],
"field_issues": [],
"status": "not_downloaded",
}
try:
from src.services.mapping_service import adobe_folder_to_normalized
from src.services.validation_service import validate_template
from src.compose_docusign_template import compose_template
downloads_dir = Path(settings.downloads_dir) if hasattr(settings, "downloads_dir") else Path("downloads")
# Match folder by name__id or name pattern
template_dir = _find_downloaded_template(template_id, template_name)
if not template_dir:
return analysis
normalized, _ = adobe_folder_to_normalized(str(template_dir), include_documents=False)
result = validate_template(normalized)
analysis["blockers"] = result.blockers
analysis["warnings"] = result.warnings
try:
with tempfile.TemporaryDirectory() as tmpdir:
output_path = Path(tmpdir) / "docusign-template.json"
_, _compose_warnings, field_issues = compose_template(str(template_dir), str(output_path))
analysis["field_issues"] = field_issues
except Exception as exc:
analysis["warnings"] = _dedupe([
*analysis["warnings"],
f"Field mapping analysis unavailable: {exc}",
])
analysis["status"] = "analyzed"
return analysis
except Exception as exc:
analysis["warnings"] = [f"Template analysis unavailable: {exc}"]
analysis["status"] = "error"
return analysis
def _find_downloaded_template(template_id: str, template_name: str) -> Path | None:
downloads_dir = Path(settings.downloads_dir)
candidates = list(downloads_dir.glob(f"*__{template_id}"))
if not candidates:
# Try matching by sanitised name prefix
safe = template_name.replace("/", "_").replace("\\", "_")
candidates = list(downloads_dir.glob(f"{safe}*"))
return next((c for c in candidates if c.is_dir()), None)
if not candidates or not candidates[0].is_dir():
return [], []
normalized = adobe_folder_to_normalized(str(candidates[0]))
result = validate_template(normalized)
return result.blockers, result.warnings
except Exception:
return [], []
def _dedupe(items: list[str]) -> list[str]:
seen = set()
result = []
for item in items:
if item in seen:
continue
seen.add(item)
result.append(item)
return result
# asyncio needed for gather — import at top of module

View File

@ -69,7 +69,7 @@ subscribe('issueCount', count => {
subscribe('templates', templates => {
const caveats = (templates || []).filter(t =>
(!t.blockers || t.blockers.length === 0) &&
t.warnings && t.warnings.length > 0
((t.warnings || []).length > 0 || (t.field_issues || []).length > 0)
).length;
const badge = document.getElementById('nav-badge-caveats');
if (badge) {

View File

@ -1,16 +1,16 @@
// Issues & Warnings view — surfaces all validation problems before migration
import { state } from './state.js';
import { escHtml, formatDate } from './utils.js';
import { escHtml, formatDate, renderFieldIssues, bindFieldIssueToggles } from './utils.js';
import { navigate } from './router.js';
export function renderIssues() {
const outlet = document.getElementById('router-outlet');
const templates = state.templates || [];
const blocked = templates.filter(t => t.blockers && t.blockers.length > 0);
const blocked = templates.filter(t => hasBlockers(t));
const warnings = templates.filter(t =>
(!t.blockers || t.blockers.length === 0) && t.warnings && t.warnings.length > 0
!hasBlockers(t) && (hasWarnings(t) || hasFieldIssues(t))
);
if (!state.auth.adobe || !state.auth.docusign) {
@ -32,7 +32,7 @@ export function renderIssues() {
<span class="callout-icon">🎉</span>
<div>
<strong>All templates are ready!</strong>
<div style="margin-top:4px">No blockers or warnings found across ${templates.length} template${templates.length !== 1 ? 's' : ''}.</div>
<div style="margin-top:4px">No validation blockers, warnings, or field mapping caveats found across ${templates.length} template${templates.length !== 1 ? 's' : ''}.</div>
</div>
</div>`;
return;
@ -66,7 +66,7 @@ export function renderIssues() {
${warnings.length ? `
<div>
<div style="font-size:14px;font-weight:700;color:var(--warning);margin-bottom:10px">
Warnings ${warnings.length} template${warnings.length > 1 ? 's' : ''} will migrate with caveats
Caveats ${warnings.length} template${warnings.length > 1 ? 's' : ''} should be reviewed
</div>
<div class="attention-list">
${warnings.map(t => _warningItem(t)).join('')}
@ -85,6 +85,8 @@ export function renderIssues() {
document.querySelectorAll('.btn-view-template').forEach(btn => {
btn.addEventListener('click', () => navigate(`#/templates/${btn.dataset.id}`));
});
bindFieldIssueToggles(outlet);
}
function _blockerItem(t) {
@ -106,6 +108,7 @@ function _blockerItem(t) {
function _warningItem(t) {
const warnings = t.warnings || [];
const fieldIssues = t.field_issues || [];
return `
<div class="attention-item warning">
<span class="attention-icon"></span>
@ -113,6 +116,7 @@ function _warningItem(t) {
<div class="attention-name">${escHtml(t.name)}</div>
${warnings.slice(0, 3).map(w => `<div class="attention-detail">• ${escHtml(w)}</div>`).join('')}
${warnings.length > 3 ? `<div class="attention-detail" style="color:var(--text-muted)">… +${warnings.length - 3} more</div>` : ''}
${fieldIssues.length ? renderFieldIssues(fieldIssues) : ''}
<div style="margin-top:6px;font-size:11px;color:var(--text-muted)">Modified ${formatDate(t.adobe_modified)}</div>
</div>
<div class="attention-action" style="display:flex;flex-direction:column;gap:6px;align-items:flex-end">
@ -122,3 +126,15 @@ function _warningItem(t) {
</div>
`;
}
function hasBlockers(t) {
return (t.blockers || []).length > 0;
}
function hasWarnings(t) {
return (t.warnings || []).length > 0;
}
function hasFieldIssues(t) {
return (t.field_issues || []).length > 0;
}

View File

@ -17,7 +17,7 @@ export const state = {
docusignAccountSelectionRequired: false,
isAdmin: false,
},
templates: [], // [{ adobe_id, name, status, blockers, warnings, ... }]
templates: [], // [{ adobe_id, name, status, blockers, warnings, field_issues, ... }]
templatesError: null, // Visible error state for template loading failures
selectedIds: new Set(),
lastMigrationResults: null, // final batch job results
@ -46,6 +46,10 @@ export function setState(key, value) {
// Recompute derived values after template list updates
export function updateDerivedState() {
const blocked = state.templates.filter(t => t.blockers && t.blockers.length > 0).length;
setState('issueCount', blocked);
const issueCount = state.templates.filter(t =>
(t.blockers || []).length > 0 ||
(t.warnings || []).length > 0 ||
(t.field_issues || []).length > 0
).length;
setState('issueCount', issueCount);
}

View File

@ -12,15 +12,18 @@ function readiness(t) {
if (t.blockers && t.blockers.length > 0) {
return { key: 'blocked', label: 'Blocked', cls: 'badge-blocked' };
}
if (hasFieldIssues(t)) {
return { key: 'field-caveats', label: 'Caveats', cls: 'badge-caveats' };
}
if (t.status === 'migrated') {
return t.warnings && t.warnings.length > 0
return hasWarnings(t)
? { key: 'migrated-warn', label: 'Migrated', cls: 'badge-migrated' }
: { key: 'migrated', label: 'Migrated', cls: 'badge-migrated' };
}
if (t.status === 'needs_update') {
return { key: 'needs-update', label: 'Needs Update', cls: 'badge-needs-update' };
}
if (t.warnings && t.warnings.length > 0) {
if (hasWarnings(t)) {
return { key: 'caveats', label: 'Caveats', cls: 'badge-caveats' };
}
return { key: 'ready', label: 'Ready', cls: 'badge-ready' };
@ -180,10 +183,13 @@ function _templateRow(t) {
const selected = state.selectedIds.has(t.adobe_id);
const warnCount = (t.warnings || []).length;
const blockCount = (t.blockers || []).length;
const issueClass = blockCount > 0 ? 'blocked' : (warnCount > 0 ? 'has-issues' : 'no-issues');
const fieldIssueCount = (t.field_issues || []).length;
const issueClass = blockCount > 0 ? 'blocked' : (warnCount > 0 || fieldIssueCount > 0 ? 'has-issues' : 'no-issues');
const issueLabel = blockCount > 0
? `🚫 ${blockCount} blocker${blockCount > 1 ? 's' : ''}`
: (warnCount > 0 ? `${warnCount} warning${warnCount > 1 ? 's' : ''}` : '✓ Clean');
: (warnCount > 0 || fieldIssueCount > 0
? `${warnCount + fieldIssueCount} caveat${warnCount + fieldIssueCount > 1 ? 's' : ''}`
: '✓ Clean');
return `
<tr class="${selected ? 'row-selected' : ''}" data-id="${escHtml(t.adobe_id)}">
@ -217,7 +223,7 @@ function _statusCounts(templates) {
migrated: templates.filter(t => t.status === 'migrated').length,
needs_update: templates.filter(t => t.status === 'needs_update').length,
blocked: templates.filter(t => t.blockers && t.blockers.length > 0).length,
caveats: templates.filter(t => (!t.blockers || !t.blockers.length) && t.warnings && t.warnings.length > 0).length,
caveats: templates.filter(t => !hasBlockers(t) && (hasWarnings(t) || hasFieldIssues(t))).length,
};
}
@ -233,9 +239,9 @@ function _applyFilter(templates) {
// Status / readiness filter
if (_filter.status !== 'all') {
if (_filter.status === 'blocked') {
list = list.filter(t => t.blockers && t.blockers.length > 0);
list = list.filter(t => hasBlockers(t));
} else if (_filter.status === 'caveats') {
list = list.filter(t => (!t.blockers || !t.blockers.length) && t.warnings && t.warnings.length > 0);
list = list.filter(t => !hasBlockers(t) && (hasWarnings(t) || hasFieldIssues(t)));
} else {
list = list.filter(t => t.status === _filter.status);
}
@ -246,7 +252,7 @@ function _applyFilter(templates) {
let va = a[_sort.col] || '';
let vb = b[_sort.col] || '';
if (_sort.col === 'readiness') { va = readiness(a).key; vb = readiness(b).key; }
if (_sort.col === 'warnings') { va = (a.blockers||[]).length + (a.warnings||[]).length; vb = (b.blockers||[]).length + (b.warnings||[]).length; }
if (_sort.col === 'warnings') { va = totalIssueCount(a); vb = totalIssueCount(b); }
if (typeof va === 'number') return _sort.dir === 'asc' ? va - vb : vb - va;
return _sort.dir === 'asc' ? String(va).localeCompare(String(vb)) : String(vb).localeCompare(String(va));
});
@ -356,6 +362,7 @@ export async function renderTemplateDetail(adobeId) {
}
const r = readiness(t);
const issueCount = totalIssueCount(t);
outlet.innerHTML = `
<div class="page-header">
<div>
@ -371,8 +378,8 @@ export async function renderTemplateDetail(adobeId) {
<div class="tabs" id="detail-tabs">
<div class="tab active" data-tab="overview">Overview</div>
<div class="tab" data-tab="issues">Issues ${(t.blockers||[]).length + (t.warnings||[]).length > 0
? `<span class="nav-badge" style="position:static;display:inline">${(t.blockers||[]).length + (t.warnings||[]).length}</span>` : ''}</div>
<div class="tab" data-tab="issues">Issues ${issueCount > 0
? `<span class="nav-badge" style="position:static;display:inline">${issueCount}</span>` : ''}</div>
<div class="tab" data-tab="history">Migration History</div>
</div>
@ -426,10 +433,15 @@ function _renderDetailTab(t, tabKey) {
} else if (tabKey === 'issues') {
const blockers = t.blockers || [];
const warnings = t.warnings || [];
if (!blockers.length && !warnings.length) {
const fieldIssues = t.field_issues || [];
if (!blockers.length && !warnings.length && !fieldIssues.length) {
content.innerHTML = `<div class="callout success"><span class="callout-icon">✓</span>No issues found. This template is ready to migrate.</div>`;
} else {
content.innerHTML = `
<div class="callout info">
<span class="callout-icon"></span>
This view combines pre-migration validation with field mapping caveats. Field caveats are the same kinds of issues shown after migration.
</div>
${blockers.length ? `
<div class="card">
<div class="card-header"><span class="card-title" style="color:var(--error)">🚫 Blockers (${blockers.length})</span></div>
@ -441,6 +453,13 @@ function _renderDetailTab(t, tabKey) {
</div>`).join('')}
</div>
</div>` : ''}
${fieldIssues.length ? `
<div class="card">
<div class="card-header"><span class="card-title" style="color:var(--warning)"> Field Mapping Caveats (${fieldIssues.length})</span></div>
<div class="card-body">
${renderFieldIssues(fieldIssues)}
</div>
</div>` : ''}
${warnings.length ? `
<div class="card">
<div class="card-header"><span class="card-title" style="color:var(--warning)"> Warnings (${warnings.length})</span></div>
@ -452,6 +471,7 @@ function _renderDetailTab(t, tabKey) {
</div>`).join('')}
</div>
</div>` : ''}`;
bindFieldIssueToggles(content);
}
} else if (tabKey === 'history') {
api.migrate.history().then(data => {
@ -517,3 +537,19 @@ function _renderDetailTab(t, tabKey) {
});
}
}
function hasBlockers(t) {
return (t.blockers || []).length > 0;
}
function hasWarnings(t) {
return (t.warnings || []).length > 0;
}
function hasFieldIssues(t) {
return (t.field_issues || []).length > 0;
}
function totalIssueCount(t) {
return (t.blockers || []).length + (t.warnings || []).length + (t.field_issues || []).length;
}