Compare commits
3 Commits
210f273c05
...
447a89923a
| Author | SHA1 | Date |
|---|---|---|
|
|
447a89923a | |
|
|
2b3413670f | |
|
|
c5b7b9f5b8 |
|
|
@ -20,13 +20,13 @@ Develop an agent/toolkit that can programmatically extract template data and fie
|
|||
|
||||
#### Components
|
||||
- **Adobe Sign Client** (`src/adobe_api.py`) — authenticated API calls, template listing/download
|
||||
- **DocuSign Client** (`src/upload_docusign_template.py`, `src/docusign_auth.py`) — JWT auth, template upsert
|
||||
- **DocuSign Client** (`src/upload_docusign_template.py`, `src/docusign_auth.py`) — OAuth auth, template upsert
|
||||
- **Normalized Schema Model** (`src/models/normalized_template.py`) — platform-agnostic intermediate representation
|
||||
- **Mapping Service** (`src/services/mapping_service.py`) — field type, recipient role, coordinate translation
|
||||
- **Validation Service** (`src/services/validation_service.py`) — field count comparison, recipient checks, missing role detection
|
||||
- **Migration Service** (`src/services/migration_service.py`) — orchestrates download → normalize → validate → compose → upload
|
||||
- **Report Builder** (`src/reports/report_builder.py`) — structured success/warning/error output
|
||||
- **Web API** (`web/`) — FastAPI endpoints for browser-based orchestration
|
||||
- **Mapping Service** (`src/services/mapping_service.py`) — field type, recipient role, coordinate translation; produces `NormalizedTemplate`
|
||||
- **Validation Service** (`src/services/validation_service.py`) — blocker and warning checks on the normalized schema
|
||||
- **Compose** (`src/compose_docusign_template.py`) — converts `NormalizedTemplate` → DocuSign `envelopeTemplate` JSON; emits `FieldIssue` objects for partial/dropped features
|
||||
- **Report Builder** (`src/reports/report_builder.py`) — structured success/warning/error output per template
|
||||
- **Web API** (`web/`) — FastAPI endpoints for browser-based orchestration; full pipeline orchestration lives in `web/routers/migrate.py`
|
||||
- **Frontend** (`web/static/`) — side-by-side template browser, migration UI
|
||||
|
||||
#### Service Separation
|
||||
|
|
@ -34,16 +34,22 @@ Develop an agent/toolkit that can programmatically extract template data and fie
|
|||
src/
|
||||
models/
|
||||
normalized_template.py # intermediate schema
|
||||
field_issue.py # structured field-issue model + issue codes
|
||||
services/
|
||||
migration_service.py # pipeline orchestration
|
||||
mapping_service.py # field/role/coord transformations
|
||||
validation_service.py # pre/post migration checks
|
||||
reports/
|
||||
report_builder.py # structured report output
|
||||
utils/
|
||||
pdf_coords.py # coordinate normalization helpers
|
||||
retry.py # exponential backoff retry helpers
|
||||
log_sanitizer.py # secret redaction from logs
|
||||
```
|
||||
|
||||
> Note: pipeline orchestration (download → normalize → validate → compose → upload → report) is
|
||||
> implemented inline in `web/routers/migrate.py` (`_migrate_one()`) for the web layer and in
|
||||
> `src/migrate_template.py` for the CLI. There is no shared `migration_service.py` orchestration
|
||||
> layer — this is a known divergence from the original spec that is acceptable for the current scope.
|
||||
|
||||
---
|
||||
|
||||
### High-Level Migration Flow
|
||||
|
|
|
|||
73
README.md
73
README.md
|
|
@ -177,6 +177,79 @@ Create one project per customer to keep history and settings separate.
|
|||
|
||||
---
|
||||
|
||||
## Production deployment
|
||||
|
||||
The web UI is designed for local or private-network use during a migration engagement. If you do expose it more broadly, follow these steps:
|
||||
|
||||
### Run behind a reverse proxy (HTTPS required for OAuth)
|
||||
|
||||
OAuth callbacks from both Adobe Sign and DocuSign require HTTPS. Use nginx, Caddy, or a cloud load balancer to terminate TLS and proxy to uvicorn:
|
||||
|
||||
```
|
||||
# nginx example
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
}
|
||||
```
|
||||
|
||||
Start uvicorn without `--reload` in production:
|
||||
```bash
|
||||
uvicorn web.app:app --host 127.0.0.1 --port 8000 --workers 1
|
||||
```
|
||||
|
||||
> Use `--workers 1` — batch job state is in-memory and not safe to share across workers.
|
||||
|
||||
### Required environment variables
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `SESSION_SECRET_KEY` | Random secret for signing session cookies. Generate one with `python3 -c "import secrets; print(secrets.token_hex(32))"` |
|
||||
| `SESSION_STORE_DIR` | Absolute path for server-side session files (default: `.session-store/` in project root) |
|
||||
| `AUDIT_LOG_FILE` | Absolute path for the JSONL audit log (default: `.audit-log.jsonl` in project root) |
|
||||
| `ADOBE_REDIRECT_URI` | Must match the callback URL registered in your Adobe Sign app (e.g. `https://migrator.example.com/api/auth/adobe/callback`) |
|
||||
| `DOCUSIGN_REDIRECT_URI` | Must match the callback URL registered in your DocuSign app (e.g. `https://migrator.example.com/api/auth/docusign/callback`) |
|
||||
|
||||
### Rotating SESSION_SECRET_KEY
|
||||
|
||||
Changing `SESSION_SECRET_KEY` invalidates all existing browser sessions — every user will be logged out and must reconnect their Adobe Sign and DocuSign accounts. There is no migration path for existing session files. To rotate:
|
||||
|
||||
1. Update `SESSION_SECRET_KEY` in `.env`
|
||||
2. Delete all files in `SESSION_STORE_DIR`
|
||||
3. Restart the server
|
||||
|
||||
### Shard configuration
|
||||
|
||||
By default the app targets the Adobe Sign **EU2** shard. To target a different shard, set `ADOBE_SIGN_BASE_URL` in `.env`:
|
||||
|
||||
```
|
||||
# NA1 shard
|
||||
ADOBE_SIGN_BASE_URL=https://api.na1.adobesign.com/api/rest/v6
|
||||
|
||||
# EU2 shard (default)
|
||||
ADOBE_SIGN_BASE_URL=https://api.eu2.adobesign.com/api/rest/v6
|
||||
```
|
||||
|
||||
Also update `ADOBE_REDIRECT_URI` and the OAuth app registration to match your shard's auth server if it differs.
|
||||
|
||||
For DocuSign, switch from sandbox to production by updating:
|
||||
```
|
||||
DOCUSIGN_AUTH_SERVER=account.docusign.com
|
||||
DOCUSIGN_BASE_URL=https://na3.docusign.net/restapi # your account's base URL
|
||||
```
|
||||
|
||||
### Session store maintenance
|
||||
|
||||
Session files accumulate in `SESSION_STORE_DIR` — one file per browser session. Delete stale files periodically:
|
||||
|
||||
```bash
|
||||
# Delete session files older than 7 days
|
||||
find .session-store/ -name "*.json" -mtime +7 -delete
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Running tests
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -13,12 +13,15 @@ DocuSign API reference:
|
|||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import csv
|
||||
import json
|
||||
import argparse
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
from auth_helper import get_access_token # reuses existing JWT auth
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
||||
from docusign_auth import get_access_token
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,81 +1,263 @@
|
|||
# Architecture & Design Overview
|
||||
# Architecture & Design — Adobe Sign → DocuSign Migrator
|
||||
|
||||
## System Components
|
||||
- **Extraction Layer**: Handles authentication, API calls, and raw data retrieval from Adobe Sign. Input: .env credentials. Output: JSON metadata + field data.
|
||||
- **Mapping/Transform Layer**: Pure logic between raw Adobe template objects and canonical DocuSign template model. Handles all 1:1, many:1, and lossy mappings. Logging of ambiguities.
|
||||
- **DocuSign Ingest Layer**: Authenticates, creates/updates templates in DocuSign using mapped objects. Handles feedback, errors, and reporting.
|
||||
- **Validation/QA Layer**: Compares final artifacts, runs coverage and correctness checks, supports dry-run/test modes.
|
||||
- **Testing/Scenario Folder**: Sample templates and responses (see `/sample-templates/`) and mapping/transform test cases.
|
||||
|
||||
## Data Flow
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[Adobe Sign API] -->|Extract| B[Raw JSON]
|
||||
B -->|Transform/Map| C[Canonical Model]
|
||||
C -->|Ingest| D[DocuSign API]
|
||||
D -->|Validate| E[QA/Reporting]
|
||||
E -->|Feedback| B
|
||||
```
|
||||
|
||||
1. Extract Adobe template (metadata, fields, roles, workflows)
|
||||
2. Pass to transform/mapping functions (per field/role/conditional)
|
||||
3. Generate canonical model; attempt creation in DocuSign
|
||||
4. Log result; pull DocuSign result and validate against input
|
||||
5. Drop all validated or problematic test scenarios in `/sample-templates/` or a new `tests/` folder for regression & future QA
|
||||
|
||||
## Key Design Decisions & Logger
|
||||
- Focus on batch/parallelization via pipelined scripts/modules
|
||||
- Use local cache of all raw API payloads for traceability
|
||||
- Mapping module must be testable with static samples (no account needed at first)
|
||||
- Agent harness structure for project traceability, autonomous improvement
|
||||
- **Decision Log** (expand as project runs):
|
||||
- [2026-04-14] Start with static JSON tests and pure transforms before integrating live API. Document all lossy mappings inline in mapping functions & doc.
|
||||
- [2026-04-14] Capture all feature-mapping challenges (fields, roles) as they appear in real-world test cases and update this doc.
|
||||
|
||||
## Extensibility
|
||||
- Designed for: new field types, more templates, transform plugins
|
||||
- Support “mapping hints” or forced overrides for ambiguous/complex field cases
|
||||
*Last updated: 2026-04-23*
|
||||
|
||||
---
|
||||
|
||||
## v2 Architecture — Web UI (2026-04-17)
|
||||
## System Overview
|
||||
|
||||
The pipeline is extended with a FastAPI web layer that wraps all existing src/ modules.
|
||||
The migrator is a Python toolkit with two interfaces that share the same core pipeline:
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
Browser -->|HTTP| FastAPI
|
||||
FastAPI -->|OAuth| AdobeSign[Adobe Sign API]
|
||||
FastAPI -->|OAuth| DocuSign[DocuSign API]
|
||||
FastAPI -->|calls| Compose[compose_docusign_template.py]
|
||||
FastAPI -->|calls| Upload[upload_docusign_template.py]
|
||||
Upload -->|upsert| DocuSign
|
||||
FastAPI -->|reads/writes| History[migration-output/.history.json]
|
||||
```
|
||||
- **CLI** (`src/`) — shell scripts for one-off or scripted migrations
|
||||
- **Web UI** (`web/`) — FastAPI + vanilla JS SPA for browser-based, multi-user migrations
|
||||
|
||||
**New layers:**
|
||||
- `web/routers/auth.py` — browser-initiated OAuth for Adobe Sign and DocuSign
|
||||
- `web/routers/templates.py` — template listing + migration status computation
|
||||
- `web/routers/migrate.py` — triggers pipeline; records history
|
||||
- `web/static/` — vanilla HTML/JS SPA (no build step)
|
||||
|
||||
**Template issue status:**
|
||||
`GET /api/templates/status` drives the Templates and Issues & Warnings pages.
|
||||
Its summary status combines pre-migration validation and DocuSign composition
|
||||
analysis:
|
||||
|
||||
- `blockers`: validation failures that stop migration.
|
||||
- `warnings`: validation warnings that allow migration but need review.
|
||||
- `field_issues`: field mapping caveats emitted by composition, such as skipped
|
||||
field types or unsupported conditional logic.
|
||||
|
||||
The list-level "Clean" label should only appear when all three collections are
|
||||
empty, so summary rows match the template detail and migration result views.
|
||||
|
||||
**Idempotent Upload (v2):**
|
||||
`upload_docusign_template.py` now searches for an existing DocuSign template by exact name match and updates the most recently modified one (PUT). Falls back to create (POST) if no match. `--force-create` flag bypasses upsert.
|
||||
Both interfaces execute the same sequence: authenticate → download → normalize → validate → compose → upload → report.
|
||||
|
||||
---
|
||||
|
||||
*Update as architecture/requirements change. Generated by Cleo (2026-04-14). Updated 2026-04-17.*
|
||||
## Component Map
|
||||
|
||||
```
|
||||
Browser / CLI
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────┐
|
||||
│ web/app.py (FastAPI) OR src/migrate_*.py │
|
||||
│ – session management (web only) │
|
||||
│ – OAuth orchestration (web only) │
|
||||
│ – batch job queue (in-memory dict, web only) │
|
||||
└──────────────┬──────────────────────────────────┘
|
||||
│ calls
|
||||
┌──────────┴──────────┐
|
||||
▼ ▼
|
||||
src/adobe_api.py src/upload_docusign_template.py
|
||||
(Adobe Sign REST) (DocuSign REST — upsert)
|
||||
│ ▲
|
||||
│ raw JSON │ DocuSign JSON
|
||||
▼ │
|
||||
src/services/mapping_service.py
|
||||
└─► src/models/normalized_template.py
|
||||
│ NormalizedTemplate
|
||||
▼
|
||||
src/services/validation_service.py
|
||||
│ blockers / warnings
|
||||
▼
|
||||
src/compose_docusign_template.py
|
||||
└─► src/models/field_issue.py
|
||||
│ (template_dict, warnings, field_issues)
|
||||
│
|
||||
▼
|
||||
src/reports/report_builder.py
|
||||
└─► MigrationReport written to migration-output/.history.json
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Pipeline Stages
|
||||
|
||||
### 1. Authentication
|
||||
|
||||
| Surface | Adobe Sign | DocuSign |
|
||||
|---------|-----------|---------|
|
||||
| CLI | OAuth Auth Code via `adobe_auth.py`; tokens stored in `.env` | OAuth Auth Code via `docusign_auth.py`; tokens stored in `.env` |
|
||||
| Web | OAuth Auth Code via `/api/auth/adobe/callback`; tokens in server-side session file | OAuth Auth Code via `/api/auth/docusign/callback`; tokens in server-side session file |
|
||||
|
||||
The web UI never stores OAuth tokens in `.env` — each browser session carries its own tokens in a signed server-side session file under `.session-store/`. Sessions are identified by a cookie (`session_id`) signed with `SESSION_SECRET_KEY`.
|
||||
|
||||
### 2. Download (Adobe Sign)
|
||||
|
||||
`src/adobe_api.py` fetches from the Adobe Sign REST v6 API. Shard is configured via `ADOBE_SIGN_BASE_URL` (default: `https://api.eu2.adobesign.com/api/rest/v6`).
|
||||
|
||||
For each template, three artifacts are written to `downloads/<template-name>__<id>/`:
|
||||
|
||||
| File | Content |
|
||||
|------|---------|
|
||||
| `metadata.json` | Template metadata (name, status, creator, dates) |
|
||||
| `form_fields.json` | Full form field list with locations, conditions, validations |
|
||||
| `documents.json` | Document list metadata |
|
||||
| `<name>.pdf` | Binary PDF (base64 decoded) |
|
||||
|
||||
### 3. Normalize (`mapping_service.py`)
|
||||
|
||||
`MappingService.from_folder(path)` reads the three JSON files and produces a `NormalizedTemplate` (Pydantic model). This platform-agnostic intermediate schema decouples Adobe-specific field names from the DocuSign composition step.
|
||||
|
||||
Key transformations at this stage:
|
||||
- Participant sets → typed role list (`SIGN`, `APPROVE`, `CC`)
|
||||
- Field locations expanded into flat list (multi-location fields produce N entries)
|
||||
- Conditional action references converted to normalized `ConditionalRule` objects
|
||||
|
||||
### 4. Validate (`validation_service.py`)
|
||||
|
||||
Runs pre-migration checks and returns `(blockers: list[str], warnings: list[str])`.
|
||||
|
||||
| Check | Result on failure |
|
||||
|-------|-----------------|
|
||||
| No recipients | Blocker |
|
||||
| No documents | Blocker |
|
||||
| No signature fields | Warning |
|
||||
| Unassigned fields | Warning |
|
||||
| Unsupported feature detected | Warning |
|
||||
|
||||
Blockers halt migration. Warnings are stored in the history and surfaced in the UI but do not stop the pipeline.
|
||||
|
||||
### 5. Compose (`compose_docusign_template.py`)
|
||||
|
||||
Converts `NormalizedTemplate` → DocuSign `envelopeTemplate` JSON. Returns a 3-tuple:
|
||||
|
||||
```python
|
||||
(template_dict: dict, warnings: list[str], field_issues: list[dict])
|
||||
```
|
||||
|
||||
`field_issues` are structured `FieldIssue` objects (see `src/models/field_issue.py`) emitted when a field migrates successfully but something was silently dropped or approximated. Each issue has a machine-readable `code` (e.g. `CROSS_RECIPIENT_CONDITIONAL`, `HIDE_ACTION`, `FIELD_TYPE_SKIPPED`). See [field-mapping.md](../field-mapping.md) for the full list.
|
||||
|
||||
### 6. Upload (`upload_docusign_template.py`)
|
||||
|
||||
Upsert pattern:
|
||||
1. Search DocuSign for an existing template with the same name
|
||||
2. If found: `PUT /templates/{id}` (update the most recently modified match)
|
||||
3. If not found: `POST /templates` (create new)
|
||||
4. `--force-create` flag bypasses the search and always creates
|
||||
|
||||
### 7. Report (`report_builder.py`)
|
||||
|
||||
A `MigrationReport` is built per template and appended to `migration-output/.history.json`. Each record contains:
|
||||
- template name, Adobe ID, DocuSign ID
|
||||
- status (`success`, `dry_run`, `skipped`, `error`)
|
||||
- blockers, warnings, field_issues
|
||||
- PDF checksum (SHA-256)
|
||||
- timestamp
|
||||
|
||||
---
|
||||
|
||||
## Web Layer
|
||||
|
||||
### FastAPI App (`web/app.py`)
|
||||
|
||||
- Mounts all routers under `/api/`
|
||||
- Serves the SPA shell from `web/static/index.html`
|
||||
- Installs `SanitizingFilter` on the root logger at startup (redacts tokens and secrets from all log output)
|
||||
- Logs a warning at startup if `SESSION_SECRET_KEY` is the default development value
|
||||
|
||||
### Routers
|
||||
|
||||
| Router | Prefix | Responsibility |
|
||||
|--------|--------|---------------|
|
||||
| `auth.py` | `/api/auth` | Adobe Sign + DocuSign OAuth flows, session status |
|
||||
| `templates.py` | `/api/templates` | Adobe template listing; migration status per template |
|
||||
| `migrate.py` | `/api/migrate` | Single and batch migration; history; job polling |
|
||||
| `verify.py` | `/api/verify` | Send test envelopes; poll status; void |
|
||||
| `audit.py` | `/api/audit` | Audit log access + CSV export |
|
||||
| `admin.py` | `/api/admin` | Admin-only operations (admin_emails gating) |
|
||||
|
||||
### Session Lifecycle
|
||||
|
||||
```
|
||||
Browser makes first request
|
||||
→ middleware generates UUID session_id
|
||||
→ signed cookie set (itsdangerous, SESSION_SECRET_KEY)
|
||||
→ session file created at .session-store/<session_id>.json
|
||||
|
||||
User connects Adobe Sign / DocuSign
|
||||
→ OAuth tokens written to session file (never to .env)
|
||||
→ session file updated on every token refresh
|
||||
|
||||
User disconnects or session file deleted
|
||||
→ next request gets a fresh session_id and new file
|
||||
→ old file can be deleted manually to force re-auth
|
||||
```
|
||||
|
||||
Session files are plain JSON. Delete all files in `.session-store/` to reset all user sessions. Set `SESSION_STORE_DIR` in `.env` to change the location.
|
||||
|
||||
### Multi-Account DocuSign Support
|
||||
|
||||
When a DocuSign user belongs to multiple accounts, the web UI:
|
||||
1. Fetches `/oauth/userinfo` after the OAuth callback
|
||||
2. Sorts available accounts alphabetically
|
||||
3. Prompts the user to pick one account for the session
|
||||
4. Stores `docusign_account_id` in the session alongside the tokens
|
||||
|
||||
### Batch Job State
|
||||
|
||||
Batch migrations are tracked in an in-memory dict (`_batch_jobs`) in `web/routers/migrate.py`. Job state is lost on server restart — any in-flight batch becomes unrecoverable. This is a known limitation appropriate for single-operator deployments. Production deployments requiring durability should persist job state to a database or file store.
|
||||
|
||||
### Audit Log
|
||||
|
||||
`web/audit.py` writes one JSONL record per migration event to `AUDIT_LOG_FILE` (default: `.audit-log.jsonl`). Each record:
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": "2026-04-23T12:00:00Z",
|
||||
"session_id": "abc123",
|
||||
"user_email": "user@example.com",
|
||||
"action": "migrate",
|
||||
"template_name": "Sales Agreement",
|
||||
"adobe_template_id": "3AAA...",
|
||||
"docusign_template_id": "uuid",
|
||||
"status": "success",
|
||||
"field_issues_count": 2,
|
||||
"pdf_checksum": "sha256:abcdef..."
|
||||
}
|
||||
```
|
||||
|
||||
The `/api/audit` endpoints expose this log with filtering and CSV export. Sensitive fields (tokens, secrets) are never written — the `SanitizingFilter` on the root logger ensures they are redacted before hitting any output.
|
||||
|
||||
---
|
||||
|
||||
## Frontend SPA
|
||||
|
||||
Single-page app in `web/static/`. No build step — plain HTML + ES modules.
|
||||
|
||||
| File | Responsibility |
|
||||
|------|---------------|
|
||||
| `index.html` | Shell, left nav, top bar, router outlet |
|
||||
| `js/router.js` | Hash-based routing (`#/templates`, `#/results`, etc.) |
|
||||
| `js/state.js` | Global pub/sub state store |
|
||||
| `js/api.js` | Typed fetch wrappers for all backend endpoints |
|
||||
| `js/auth.js` | Auth chip UI, OAuth flow, toast notifications |
|
||||
| `js/templates.js` | Templates view + detail tabs (overview / issues / history) |
|
||||
| `js/migration.js` | Migration modal, progress polling, results view |
|
||||
| `js/issues.js` | Issues & Warnings view |
|
||||
| `js/verification.js` | Verification view (send / poll / void envelopes) |
|
||||
| `js/history.js` | History & Audit view |
|
||||
| `js/settings.js` | Settings view |
|
||||
| `js/project.js` | Per-customer project context (localStorage) |
|
||||
| `js/utils.js` | `escHtml`, `formatDate`, `renderFieldIssues`, etc. |
|
||||
|
||||
CSS uses DocuSign 2024 brand design tokens defined in `css/tokens.css`.
|
||||
|
||||
---
|
||||
|
||||
## Security Design
|
||||
|
||||
| Concern | Mechanism |
|
||||
|---------|----------|
|
||||
| Token leakage in logs | `SanitizingFilter` installed on root logger at startup; redacts Bearer tokens, JWTs, long base64 strings, and key=value assignments for known secret fields |
|
||||
| Session integrity | Sessions signed with `SESSION_SECRET_KEY` via `itsdangerous`; secret must be set in `.env` |
|
||||
| Secret exposure at startup | Warning logged if `SESSION_SECRET_KEY` is the default value |
|
||||
| PDF integrity | SHA-256 checksum computed before upload and stored in history |
|
||||
| Credential storage | OAuth tokens stored in server-side session files, never in browser localStorage or logs |
|
||||
|
||||
---
|
||||
|
||||
## Utilities
|
||||
|
||||
### `src/utils/retry.py`
|
||||
|
||||
`retry_with_backoff` and `async_retry_with_backoff` decorators implement exponential backoff (configurable max retries, base delay, max delay). They target HTTP 429 / 5xx transient errors. These decorators are defined and tested but are not yet applied to API call sites — adding `@retry_with_backoff()` to functions in `adobe_api.py` and `upload_docusign_template.py` is the recommended next step for production hardening.
|
||||
|
||||
### `src/utils/log_sanitizer.py`
|
||||
|
||||
`install_sanitizing_filter()` attaches a `logging.Filter` to the root logger. The filter runs `redact()` on every log record's message and args, replacing Bearer tokens, JWTs, long base64 strings, and key=value secret assignments with `[REDACTED]`.
|
||||
|
||||
---
|
||||
|
||||
## Known Limitations
|
||||
|
||||
| Limitation | Impact | Mitigation |
|
||||
|-----------|--------|-----------|
|
||||
| Batch job state is in-memory | Lost on restart | Acceptable for CLI/single-operator; add DB persistence for multi-operator prod |
|
||||
| Adobe shard configured via full base URL only | Changing shard requires `.env` update | Set `ADOBE_SIGN_BASE_URL` in `.env` |
|
||||
| Retry decorators not applied to API calls | 429/5xx errors propagate immediately | Apply `@retry_with_backoff()` to `adobe_api.py` + `upload_docusign_template.py` |
|
||||
| Regression tests require real fixture data | CI cannot run regression tests without downloaded templates | Check in anonymised fixtures or generate synthetic ones |
|
||||
|
||||
*Updated 2026-04-23 — reflects v2 web UI, session lifecycle, audit log schema, multi-account support, batch job state, security design.*
|
||||
|
|
|
|||
|
|
@ -80,6 +80,32 @@ Tab types that do not merge (only first location used or handled specially):
|
|||
`radioGroupTabs` — each location is one radio button within the group
|
||||
`signerAttachmentTabs` — each location is an independent attachment request
|
||||
|
||||
## Multi-Document Templates
|
||||
|
||||
Adobe Sign library documents can contain multiple documents (PDFs) stacked into one template. DocuSign templates also support multiple documents — each document gets a unique `documentId` starting from 1.
|
||||
|
||||
### How it works
|
||||
|
||||
The compose pipeline assigns a `documentId` to each document in the order returned by the Adobe Sign `documents.json` list. All form fields reference their page position within the document they belong to (`pageNumber` is 1-based within the document's own page sequence, not the overall template page count).
|
||||
|
||||
```
|
||||
Adobe Sign template with 2 docs:
|
||||
doc[0]: "Contract.pdf" (3 pages) → documentId: 1
|
||||
doc[1]: "Exhibit-A.pdf" (2 pages) → documentId: 2
|
||||
|
||||
A field on page 2 of Exhibit-A.pdf:
|
||||
adobe_location.pageNumber = 2 (within the exhibit)
|
||||
compose emits: documentId=2, pageNumber=2
|
||||
```
|
||||
|
||||
DocuSign uses `(documentId, pageNumber)` together to locate every tab. If only one document exists, `documentId` is always `1`.
|
||||
|
||||
### Known limitation
|
||||
|
||||
Adobe Sign form fields store `pageNumber` as a sequential page number across the **entire** template (all documents concatenated). If a template has two 3-page documents, fields on document 2 have `pageNumber` 4–6. The compose pipeline does not currently rebase page numbers per document — it passes Adobe's page numbers through as-is and sets `documentId` based on field assignment.
|
||||
|
||||
**Impact**: For single-document templates this is correct. For multi-document templates, verify field placement visually in DocuSign after migration if the template spans more than one PDF.
|
||||
|
||||
## Conditional Logic Mapping
|
||||
|
||||
Adobe Sign `conditionalAction` → DocuSign `conditionalParentLabel` + `conditionalParentValue` on the dependent tab.
|
||||
|
|
|
|||
|
|
@ -10,6 +10,9 @@ uvicorn[standard]
|
|||
itsdangerous
|
||||
httpx
|
||||
|
||||
# PDF generation (sample template tooling)
|
||||
reportlab
|
||||
|
||||
# Testing
|
||||
responses
|
||||
respx
|
||||
|
|
|
|||
|
|
@ -1,9 +1,15 @@
|
|||
import os
|
||||
import sys
|
||||
import requests
|
||||
from dotenv import load_dotenv, set_key
|
||||
|
||||
load_dotenv()
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from utils.retry import RetryableHTTPError, raise_for_retryable_status, retry_with_backoff
|
||||
|
||||
_RETRY = dict(max_retries=3, base_delay=1.0, max_delay=16.0, retryable_exceptions=(RetryableHTTPError,))
|
||||
|
||||
SHARD = "eu2"
|
||||
TOKEN_URL = f"https://api.{SHARD}.adobesign.com/oauth/v2/token" # initial auth code exchange
|
||||
REFRESH_URL = f"https://api.{SHARD}.adobesign.com/oauth/v2/refresh" # token refresh (non-standard separate endpoint)
|
||||
|
|
@ -36,6 +42,7 @@ def _refresh_access_token():
|
|||
return new_token
|
||||
|
||||
|
||||
@retry_with_backoff(**_RETRY)
|
||||
def adobe_api_post_multipart(endpoint, files, data=None):
|
||||
"""Upload a file via multipart/form-data (e.g. transient documents)."""
|
||||
token = os.getenv("ADOBE_ACCESS_TOKEN")
|
||||
|
|
@ -47,10 +54,11 @@ def adobe_api_post_multipart(endpoint, files, data=None):
|
|||
token = _refresh_access_token()
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
resp = requests.post(url, headers=headers, files=files, data=data or {})
|
||||
resp.raise_for_status()
|
||||
raise_for_retryable_status(resp)
|
||||
return resp.json()
|
||||
|
||||
|
||||
@retry_with_backoff(**_RETRY)
|
||||
def adobe_api_post_json(endpoint, body):
|
||||
"""POST JSON body to an Adobe Sign endpoint."""
|
||||
token = os.getenv("ADOBE_ACCESS_TOKEN")
|
||||
|
|
@ -66,10 +74,11 @@ def adobe_api_post_json(endpoint, body):
|
|||
token = _refresh_access_token()
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
resp = requests.post(url, headers=headers, json=body)
|
||||
resp.raise_for_status()
|
||||
raise_for_retryable_status(resp)
|
||||
return resp.json()
|
||||
|
||||
|
||||
@retry_with_backoff(**_RETRY)
|
||||
def adobe_api_put_json(endpoint, body):
|
||||
"""PUT JSON body to an Adobe Sign endpoint."""
|
||||
token = os.getenv("ADOBE_ACCESS_TOKEN")
|
||||
|
|
@ -85,10 +94,11 @@ def adobe_api_put_json(endpoint, body):
|
|||
token = _refresh_access_token()
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
resp = requests.put(url, headers=headers, json=body)
|
||||
resp.raise_for_status()
|
||||
raise_for_retryable_status(resp)
|
||||
return resp.json()
|
||||
|
||||
|
||||
@retry_with_backoff(**_RETRY)
|
||||
def adobe_api_get_bytes(endpoint):
|
||||
"""Download binary content (e.g. PDF files) from the Adobe Sign API."""
|
||||
token = os.getenv("ADOBE_ACCESS_TOKEN")
|
||||
|
|
@ -103,10 +113,11 @@ def adobe_api_get_bytes(endpoint):
|
|||
headers["Authorization"] = f"Bearer {token}"
|
||||
resp = requests.get(url, headers=headers)
|
||||
|
||||
resp.raise_for_status()
|
||||
raise_for_retryable_status(resp)
|
||||
return resp.content
|
||||
|
||||
|
||||
@retry_with_backoff(**_RETRY)
|
||||
def adobe_api_get(endpoint, params=None):
|
||||
token = os.getenv("ADOBE_ACCESS_TOKEN")
|
||||
base_url = os.getenv("ADOBE_SIGN_BASE_URL", f"https://api.{SHARD}.adobesign.com/api/rest/v6")
|
||||
|
|
@ -125,7 +136,7 @@ def adobe_api_get(endpoint, params=None):
|
|||
headers["Authorization"] = f"Bearer {token}"
|
||||
resp = requests.get(url, headers=headers, params=params)
|
||||
|
||||
resp.raise_for_status()
|
||||
raise_for_retryable_status(resp)
|
||||
return resp.json()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@ Generates realistic sample PDFs for adobe-to-docusign migration testing.
|
|||
Each PDF mirrors the form fields described in the matching *-formfields.json
|
||||
so that tab positions map to visible labels on the document.
|
||||
|
||||
Adobe rect coordinates are top-left origin; DocuSign yPosition is bottom-left.
|
||||
Formula: docusign_y = PAGE_HEIGHT - adobe_top - adobe_height
|
||||
To place a *label* just above a field: label_y = PAGE_HEIGHT - adobe_top + 2
|
||||
Both Adobe Sign and DocuSign use top-left origin with y increasing downward — no
|
||||
coordinate inversion is needed. DocuSign xPosition = adobe left, yPosition = adobe top.
|
||||
To place a *label* just above a field in this PDF: label_y = page_height - adobe_top + 2
|
||||
"""
|
||||
|
||||
import base64
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ def run_migration(template_dir: Path) -> Path:
|
|||
|
||||
output_path = MIGRATION_OUTPUT_DIR / template_dir.name / "docusign-template.json"
|
||||
print(f"\nRunning migration: {template_dir.name}")
|
||||
template_dict, warnings = compose_template(str(template_dir), str(output_path))
|
||||
template_dict, warnings, field_issues = compose_template(str(template_dir), str(output_path))
|
||||
|
||||
print(f" Written: {output_path}")
|
||||
if warnings:
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ def download_template(template) -> Path:
|
|||
def convert_template(template_dir: Path) -> Path:
|
||||
output_path = OUTPUT_DIR / template_dir.name / "docusign-template.json"
|
||||
print(f"\nConverting to DocuSign format...")
|
||||
_, warnings = compose_template(str(template_dir), str(output_path))
|
||||
_, warnings, _ = compose_template(str(template_dir), str(output_path))
|
||||
print(f" Written: {output_path}")
|
||||
for w in warnings:
|
||||
print(f" WARNING: {w}")
|
||||
|
|
|
|||
|
|
@ -34,6 +34,9 @@ load_dotenv()
|
|||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
from docusign_auth import get_access_token
|
||||
from utils.retry import RetryableHTTPError, raise_for_retryable_status, retry_with_backoff
|
||||
|
||||
_RETRY = dict(max_retries=3, base_delay=1.0, max_delay=16.0, retryable_exceptions=(RetryableHTTPError,))
|
||||
|
||||
|
||||
def _make_headers(token: str) -> dict:
|
||||
|
|
@ -68,6 +71,10 @@ def find_existing_template(
|
|||
headers.update(_refresh_token_once(headers))
|
||||
resp = requests.get(url, headers=headers, params={"search_text": name, "count": 100})
|
||||
|
||||
# Raise on 429/5xx so the enclosing upload_template retry decorator can handle it.
|
||||
# For other non-2xx errors, treat as "no match found" rather than a fatal error.
|
||||
if resp.status_code in {429, 500, 502, 503, 504}:
|
||||
raise_for_retryable_status(resp)
|
||||
if not resp.ok:
|
||||
return None
|
||||
|
||||
|
|
@ -84,6 +91,7 @@ def find_existing_template(
|
|||
return exact[0]["templateId"]
|
||||
|
||||
|
||||
@retry_with_backoff(**_RETRY)
|
||||
def upload_template(file_path: str, force_create: bool = False) -> str:
|
||||
"""
|
||||
Upsert a template JSON file to DocuSign.
|
||||
|
|
@ -123,10 +131,7 @@ def upload_template(file_path: str, force_create: bool = False) -> str:
|
|||
headers = _refresh_token_once(headers)
|
||||
resp = requests.put(url, headers=headers, json=template)
|
||||
|
||||
if not resp.ok:
|
||||
print(f"ERROR: Update failed ({resp.status_code})")
|
||||
print(resp.text)
|
||||
sys.exit(1)
|
||||
raise_for_retryable_status(resp)
|
||||
|
||||
print(f"Template updated: {existing_id}")
|
||||
return existing_id
|
||||
|
|
@ -139,10 +144,7 @@ def upload_template(file_path: str, force_create: bool = False) -> str:
|
|||
headers = _refresh_token_once(headers)
|
||||
resp = requests.post(url, headers=headers, json=template)
|
||||
|
||||
if not resp.ok:
|
||||
print(f"ERROR: Upload failed ({resp.status_code})")
|
||||
print(resp.text)
|
||||
sys.exit(1)
|
||||
raise_for_retryable_status(resp)
|
||||
|
||||
result = resp.json()
|
||||
template_id = result.get("templateId")
|
||||
|
|
|
|||
|
|
@ -21,6 +21,21 @@ T = TypeVar("T")
|
|||
_RETRYABLE_STATUS = {429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
class RetryableHTTPError(Exception):
|
||||
"""Raised for HTTP status codes that warrant a retry (429, 500, 502, 503, 504)."""
|
||||
|
||||
|
||||
def raise_for_retryable_status(resp) -> None:
|
||||
"""
|
||||
Raise RetryableHTTPError for retryable status codes; call raise_for_status() for
|
||||
all others. Use this instead of resp.raise_for_status() in functions decorated with
|
||||
@retry_with_backoff(retryable_exceptions=(RetryableHTTPError,)).
|
||||
"""
|
||||
if resp.status_code in _RETRYABLE_STATUS:
|
||||
raise RetryableHTTPError(f"HTTP {resp.status_code} from {resp.url} — will retry")
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
def retry_with_backoff(
|
||||
max_retries: int = 3,
|
||||
base_delay: float = 1.0,
|
||||
|
|
|
|||
15
web/app.py
15
web/app.py
|
|
@ -12,11 +12,26 @@ From the project root.
|
|||
from fastapi import FastAPI
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse, HTMLResponse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from web.config import settings
|
||||
from web.routers import auth, templates, migrate, verify, audit, admin
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
from src.utils.log_sanitizer import install_sanitizing_filter
|
||||
|
||||
install_sanitizing_filter()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_SECRET = "dev-secret-change-in-production"
|
||||
if settings.session_secret_key == _DEFAULT_SECRET:
|
||||
logger.warning(
|
||||
"SESSION_SECRET_KEY is using the default dev value — set a random secret in .env before exposing this app"
|
||||
)
|
||||
|
||||
app = FastAPI(
|
||||
title="Adobe Sign → DocuSign Migrator",
|
||||
version=settings.version,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Template listing endpoints for Adobe Sign and DocuSign.
|
|||
Computes per-template migration status for the side-by-side UI.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
|
@ -243,7 +244,3 @@ def _dedupe(items: list[str]) -> list[str]:
|
|||
seen.add(item)
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
# asyncio needed for gather — import at top of module
|
||||
import asyncio
|
||||
|
|
|
|||
Loading…
Reference in New Issue