diff --git a/demos/reasoning-notebook/.env.example b/demos/reasoning-notebook/.env.example new file mode 100644 index 00000000..4fbe82e9 --- /dev/null +++ b/demos/reasoning-notebook/.env.example @@ -0,0 +1,18 @@ +# InputLayer engine +INPUTLAYER_URL=ws://localhost:8080/ws +INPUTLAYER_USER=admin +# INPUTLAYER_PASSWORD is auto-read from .inputlayer-credentials.toml + +# Knowledge graph name +KG_NAME=reasoning_notebook + +# LLM (LM Studio default, or set OPENAI_API_KEY for OpenAI) +LLM_BASE_URL=http://localhost:1234/v1 +LLM_MODEL=gpt-4o-mini +# OPENAI_API_KEY=sk-... + +# Extraction +EXTRACTION_MAX_CHARS=4000 + +# Frontend dev server +FRONTEND_ORIGIN=http://localhost:5173 diff --git a/demos/reasoning-notebook/.gitignore b/demos/reasoning-notebook/.gitignore new file mode 100644 index 00000000..0a320062 --- /dev/null +++ b/demos/reasoning-notebook/.gitignore @@ -0,0 +1,17 @@ +node_modules/ +dist/ +.venv/ +__pycache__/ +*.pyc +data/ +uploads/ +benchmarks/results/ +benchmarks/inputs/*.jpg +benchmarks/inputs/*.jpeg +benchmarks/inputs/*.png +.ipynb_checkpoints/ +.inputlayer-credentials.toml +*.tsbuildinfo +bun.lock +uv.lock +.env diff --git a/demos/reasoning-notebook/README.md b/demos/reasoning-notebook/README.md new file mode 100644 index 00000000..e9e71a1f --- /dev/null +++ b/demos/reasoning-notebook/README.md @@ -0,0 +1,293 @@ +# Reasoning Notebook + +A self-contained demo: Obsidian-like note editor backed by InputLayer for reasoning, LangChain for extraction, and LangGraph for ontology consolidation. + +Write notes, drop images, watch the knowledge graph grow, ask questions across everything you've written. + +## Architecture + +``` +Frontend (Vite + React) --> Backend (FastAPI) --> InputLayer (Rust) + :5173 REST :8000 WebSocket :8080 + | + LangChain (extraction) + LangGraph (ontology agent) + ChatOpenAI (Q&A) + Vision LLM (image analysis) +``` + +Three processes: +- **InputLayer server** (Rust) — reasoning engine, stores facts and rules, incremental maintenance, provenance +- **FastAPI backend** (Python) — note CRUD, LLM extraction pipeline, chat, ontology consolidation, image analysis +- **Vite frontend** (React) — markdown editor, force-directed graph, chat panel, provenance viewer + +## Prerequisites + +- **Rust toolchain** — to build the InputLayer server +- **Python 3.10+** and **uv** — for the backend +- **Bun** — for the frontend +- **LLM** — one of: + - LM Studio running locally (free, default) + - OpenAI API key + - Anthropic API key + +## Quick start + +```bash +# From the repository root: + +# 1. Build the engine (first time only) +cargo build --release --bin inputlayer-server + +# 2. Install backend dependencies (first time only) +cd demos/reasoning-notebook/backend +uv sync + +# 3. Install frontend dependencies (first time only) +cd ../frontend +bun install + +# 4. Start everything (from the repository root) +cd ../.. +./demos/reasoning-notebook/start.sh +``` + +Open http://localhost:5173 + +## Manual start (three terminals) + +If `start.sh` doesn't work or you want more control, start each process separately. + +**Terminal 1 — InputLayer server** (from repository root): +```bash +./target/release/inputlayer-server +``` +Wait for "INITIAL ADMIN CREDENTIALS CREATED" message. + +**Terminal 2 — Backend** (from repository root): +```bash +cd demos/reasoning-notebook/backend +uv run uvicorn main:app --host 0.0.0.0 --port 8000 --reload +``` +Wait for "Schema and rules deployed" message. + +**Terminal 3 — Frontend** (from repository root): +```bash +cd demos/reasoning-notebook/frontend +bun run dev +``` + +Open http://localhost:5173 + +## LLM setup + +The extraction pipeline, chat, and image analysis need an LLM. + +### LM Studio (default, free) + +1. Open LM Studio +2. Load a model (recommended: `mistralai/ministral-3-3b` for multimodal, or any model that supports structured output) +3. Go to the "Developer" tab and start the local server + +The backend auto-connects to `localhost:1234`. No configuration needed. + +### OpenAI + +Set the API key before starting the backend: +```bash +OPENAI_API_KEY=sk-... uv run uvicorn main:app --host 0.0.0.0 --port 8000 --reload +``` + +### Anthropic + +```bash +ANTHROPIC_API_KEY=sk-ant-... uv run uvicorn main:app --host 0.0.0.0 --port 8000 --reload +``` + +## Features + +### Editor (Cmd+E) +- Create, edit, delete markdown notes +- Auto-save with debounce or Cmd+S +- Markdown preview toggle +- Drag-and-drop or paste images into notes +- Notes persisted in InputLayer KG + +### Extraction +- Manual "Extract" button for text entity extraction +- Image upload triggers multimodal analysis (scene, objects, people, emotion, aesthetic, caption) +- Entity tags and relationship list shown in collapsible panel below editor +- Rich image scene analysis displayed when expanded + +### Graph (Cmd+G) +- Force-directed visualization of all entities and relationships +- Nodes colored by kind (person, organization, technology, concept, scene, object, emotion, etc.) +- Image scene data shown as grouped nodes (scene hub with objects, emotion, event, cultural context) +- Click a node to see detail panel: description, source notes, relationships, "Why?" button +- Edge labels shown only on selected node for readability +- "Consolidate Ontology" normalizes synonymous predicates and entity names +- "Resolve Entities" merges near-duplicate entities via HNSW vector similarity + +### Chat (Cmd+K) +- Ask questions across all your notes +- LLM uses the full knowledge graph (notes, entities, relationships) as context +- Conversation history within the session + +### Provenance +- Click "Why?" on any entity or relationship in the graph +- Shows the InputLayer proof tree explaining how the fact was derived +- For base facts: shows the source (edb = extensional database) +- For derived facts: shows the rule chain back to base facts + +## Keyboard shortcuts + +| Shortcut | Action | +|----------|--------| +| Cmd+N | New note | +| Cmd+S | Save note | +| Cmd+E | Switch to Editor | +| Cmd+G | Switch to Graph | +| Cmd+K | Switch to Chat | +| Enter | Send chat message | + +## Jupyter Notebook + +A query patterns notebook demonstrates InputLayer's retrieval capabilities: + +```bash +cd demos/reasoning-notebook/backend +uv run jupyter notebook ../notebooks/query_patterns.ipynb +``` + +The notebook covers: +1. **Semantic retrieval** — HNSW vector search over entity embeddings +2. **Structured retrieval** — multi-hop IQL rules (connected, two_hop, same_note) +3. **Hybrid queries** — vector similarity seeding into rule-based traversal +4. **Multimodal queries** — querying across text and image-extracted entities + +Requires the demo to be running (InputLayer server + notes with extracted entities). + +## Benchmarks + +Compare extraction quality and speed across multiple LLMs (local and cloud). + +### Setup + +Place test inputs in `benchmarks/inputs/`: +- Text files: `.txt` (two samples included) +- Images: `.jpg`, `.jpeg`, `.png` (add your own) + +Configure models in `benchmarks/config.py`. + +### Running benchmarks + +All commands run from the `backend/` directory: + +```bash +cd demos/reasoning-notebook/backend + +# All enabled models, all inputs +uv run python ../benchmarks/run_benchmark.py + +# Local models only (requires LM Studio running with model loaded) +uv run python ../benchmarks/run_benchmark.py --models local + +# Cloud models only +OPENAI_API_KEY=sk-... ANTHROPIC_API_KEY=sk-ant-... \ + uv run python ../benchmarks/run_benchmark.py --models cloud + +# Text inputs only +uv run python ../benchmarks/run_benchmark.py --text-only + +# Image inputs only +uv run python ../benchmarks/run_benchmark.py --image-only +``` + +### Viewing results + +```bash +# Terminal summary (latest results) +uv run python ../benchmarks/compare.py + +# HTML report with charts (opens in browser) +uv run python ../benchmarks/compare.py --html + +# Specific results file +uv run python ../benchmarks/compare.py ../benchmarks/results/benchmark_20260430.json --html +``` + +The HTML report includes: +- Per-input comparison tables +- Entity extraction comparison (tags per model) +- Image scene analysis comparison +- Chart.js visualizations: entities per input, extraction time, entities vs relationships, speed vs quality scatter + +### Notes + +- LM Studio only keeps one model in memory at a time. When benchmarking multiple local models, only the currently loaded model will succeed. +- Cloud models require API keys set as environment variables. +- Results are saved as JSON in `benchmarks/results/` for later comparison. + +## Configuration + +Copy `.env.example` and adjust as needed. All settings have sensible defaults for local development. + +| Variable | Default | Description | +|----------|---------|-------------| +| `INPUTLAYER_URL` | `ws://localhost:8080/ws` | Engine WebSocket URL | +| `INPUTLAYER_USER` | `admin` | Engine username | +| `KG_NAME` | `reasoning_notebook` | Knowledge graph name | +| `LLM_BASE_URL` | `http://localhost:1234/v1` | LLM API endpoint | +| `LLM_MODEL` | `gpt-4o-mini` | Model name (LM Studio ignores this) | +| `OPENAI_API_KEY` | — | OpenAI API key | +| `EXTRACTION_MAX_CHARS` | `4000` | Max content chars sent to LLM | +| `FRONTEND_ORIGIN` | `http://localhost:5173` | CORS origin | + +## Resetting data + +To start fresh, delete the engine's data directory and credentials: + +```bash +# From the repository root +rm -rf data/ .inputlayer-credentials.toml +``` + +Then restart the demo. New credentials will be auto-generated. + +## Project structure + +``` +demos/reasoning-notebook/ + backend/ + main.py FastAPI app, CRUD, extraction, chat, provenance, image endpoints + config.py Environment configuration + schemas.py Pydantic request/response models + extraction.py LangChain entity/relationship extraction pipeline + ontology.py LangGraph ontology consolidation agent + resolution.py Entity resolution via HNSW vector similarity + chat.py Chat agent using KG context + images.py Image upload, storage, and multimodal extraction + frontend/ + src/ + App.tsx Main layout, routing, keyboard shortcuts + api.ts Typed fetch wrapper for all backend endpoints + types.ts TypeScript interfaces + components/ + Sidebar.tsx Note list with create/delete + Editor.tsx Markdown editor with preview and image drop + ExtractionPanel.tsx Entity/relationship display + image scene analysis + GraphView.tsx Force-directed graph + detail panel + provenance + ChatPanel.tsx Chat interface + ProvenanceTree.tsx Proof tree viewer modal + notebooks/ + query_patterns.ipynb Jupyter notebook: semantic, structured, hybrid, multimodal queries + benchmarks/ + config.py Model definitions (local + cloud) + run_benchmark.py Run extraction across models, save JSON results + compare.py Generate terminal + HTML comparison reports + inputs/ Test text and images + results/ Benchmark output (JSON + HTML) + start.sh Launch all three processes + docker-compose.yml Containerized deployment + .env.example Configuration template +``` diff --git a/demos/reasoning-notebook/backend/chat.py b/demos/reasoning-notebook/backend/chat.py new file mode 100644 index 00000000..87706640 --- /dev/null +++ b/demos/reasoning-notebook/backend/chat.py @@ -0,0 +1,105 @@ +"""Chat agent: answer questions across notes using the knowledge graph.""" + +from __future__ import annotations + +import logging +from typing import Any + +from langchain_core.output_parsers import StrOutputParser +from langchain_core.prompts import ChatPromptTemplate +from langchain_openai import ChatOpenAI + +from inputlayer.integrations.langchain.params import iql_literal + +from config import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL + +logger = logging.getLogger("reasoning_notebook.chat") + + +def _get_llm() -> ChatOpenAI: + return ChatOpenAI( + base_url=LLM_BASE_URL, + api_key=LLM_API_KEY, + model=LLM_MODEL, + temperature=0.3, + ) + + +SYSTEM_PROMPT = ( + "You are a helpful assistant that answers questions using a personal knowledge graph.\n" + "You have access to the user's notes, extracted entities, and their relationships.\n\n" + "When answering:\n" + "- Cite specific notes by title when referencing information\n" + "- If the knowledge graph contains relevant relationships, explain the chain of reasoning\n" + "- If you don't have enough information, say so honestly\n" + "- Keep answers concise and direct\n" +) + +QA_TEMPLATE = ( + "{system}\n\n" + "=== Notes in the knowledge graph ===\n{notes_context}\n\n" + "=== Entities ===\n{entities_context}\n\n" + "=== Relationships ===\n{relationships_context}\n\n" + "User question: {question}" +) + + +async def _gather_context(kg: Any, question: str) -> dict[str, str]: + """Query the KG for relevant notes, entities, and relationships.""" + + # Get all notes (titles + content snippets) + notes_result = await kg.execute("?note(Id, Title, Content, CreatedAt, UpdatedAt)") + notes_lines = [] + if notes_result.rows and notes_result.columns != ["error"]: + for row in notes_result.rows: + data = dict(zip(notes_result.columns, row, strict=True)) + snippet = str(data["content"])[:300] + notes_lines.append(f'- "{data["title"]}": {snippet}') + + # Get all entities + ent_result = await kg.execute("?entity(Id, Name, Kind, Description, SourceNoteId)") + ent_lines = [] + if ent_result.rows and ent_result.columns != ["error"]: + for row in ent_result.rows: + data = dict(zip(ent_result.columns, row, strict=True)) + ent_lines.append(f'- {data["name"]} ({data["kind"]}): {data["description"]}') + + # Get all relationships + rel_result = await kg.execute("?relationship(Id, Subject, Predicate, Object, SourceNoteId)") + rel_lines = [] + if rel_result.rows and rel_result.columns != ["error"]: + for row in rel_result.rows: + data = dict(zip(rel_result.columns, row, strict=True)) + rel_lines.append(f'- {data["subject"]} --{data["predicate"]}--> {data["object"]}') + + return { + "notes_context": "\n".join(notes_lines) if notes_lines else "(no notes yet)", + "entities_context": "\n".join(ent_lines) if ent_lines else "(no entities extracted yet)", + "relationships_context": "\n".join(rel_lines) if rel_lines else "(no relationships yet)", + } + + +async def chat(kg: Any, question: str, history: list[dict[str, str]]) -> str: + """Answer a question using the knowledge graph as context. + + Args: + kg: KnowledgeGraph handle. + question: The user's question. + history: List of {"role": "user"|"assistant", "content": "..."} messages. + + Returns: + The assistant's response text. + """ + context = await _gather_context(kg, question) + + llm = _get_llm() + prompt = ChatPromptTemplate.from_template(QA_TEMPLATE) + chain = prompt | llm | StrOutputParser() + + answer = await chain.ainvoke({ + "system": SYSTEM_PROMPT, + "question": question, + **context, + }) + + return answer.strip() diff --git a/demos/reasoning-notebook/backend/config.py b/demos/reasoning-notebook/backend/config.py new file mode 100644 index 00000000..fb56e3e9 --- /dev/null +++ b/demos/reasoning-notebook/backend/config.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import os +from pathlib import Path + + +def _read_credentials_file() -> dict[str, str]: + """Read auto-generated credentials from the InputLayer server.""" + candidates = [ + Path(os.environ.get("INPUTLAYER_CREDENTIALS", "")), + Path(__file__).resolve().parent.parent.parent.parent / ".inputlayer-credentials.toml", + Path.cwd() / ".inputlayer-credentials.toml", + ] + for path in candidates: + if path.is_file(): + creds: dict[str, str] = {} + for line in path.read_text().splitlines(): + if "=" in line: + k, v = line.split("=", 1) + creds[k.strip()] = v.strip().strip('"') + return creds + return {} + + +_creds = _read_credentials_file() + +INPUTLAYER_URL = os.environ.get("INPUTLAYER_URL", "ws://localhost:8080/ws") +INPUTLAYER_USER = os.environ.get("INPUTLAYER_USER", "admin") +INPUTLAYER_PASSWORD = os.environ.get("INPUTLAYER_PASSWORD", _creds.get("admin_password", "admin")) +KG_NAME = os.environ.get("KG_NAME", "reasoning_notebook") +FRONTEND_ORIGIN = os.environ.get("FRONTEND_ORIGIN", "http://localhost:5173") + +LLM_BASE_URL = os.environ.get("LLM_BASE_URL", "http://localhost:1234/v1") +LLM_MODEL = os.environ.get("LLM_MODEL", "gpt-4o-mini") +LLM_API_KEY = os.environ.get("OPENAI_API_KEY", "lm-studio") diff --git a/demos/reasoning-notebook/backend/extraction.py b/demos/reasoning-notebook/backend/extraction.py new file mode 100644 index 00000000..c4a4391b --- /dev/null +++ b/demos/reasoning-notebook/backend/extraction.py @@ -0,0 +1,174 @@ +"""LangChain extraction pipeline: note -> entities + relationships.""" + +from __future__ import annotations + +import logging +import os +from typing import Any + +from langchain_openai import ChatOpenAI +from pydantic import BaseModel, Field + +from inputlayer import Relation +from inputlayer.integrations.langchain.params import iql_literal + +from config import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL + +logger = logging.getLogger("reasoning_notebook.extraction") + + +# ── KG Schema ────────────────────────────────────────────────────── + + +class Entity(Relation): + """An entity extracted from a note.""" + + id: str + name: str + kind: str + description: str + source_note_id: str + + +class Relationship(Relation): + """A relationship between two entities extracted from a note.""" + + id: str + subject: str + predicate: str + object: str + source_note_id: str + + +# ── Extraction models (Pydantic for structured output) ───────────── + + +class ExtractedEntity(BaseModel): + name: str = Field(description="Entity name, normalized to lowercase") + kind: str = Field( + description="Type: person, organization, technology, concept, place, event, role" + ) + description: str = Field(description="One-sentence description of the entity") + + +class ExtractedRelationship(BaseModel): + subject: str = Field(description="Source entity name (must match an entity name)") + predicate: str = Field( + description="Relationship type, e.g. works_at, uses, created_by, part_of, collaborates_with" + ) + object: str = Field(description="Target entity name (must match an entity name)") + + +class Extraction(BaseModel): + entities: list[ExtractedEntity] = Field(description="Entities found in the text") + relationships: list[ExtractedRelationship] = Field( + description="Relationships between extracted entities" + ) + + +# ── Pipeline ─────────────────────────────────────────────────────── + + +def _get_llm() -> ChatOpenAI: + return ChatOpenAI( + base_url=LLM_BASE_URL, + api_key=LLM_API_KEY, + model=LLM_MODEL, + temperature=0, + ) + + +EXTRACTION_PROMPT = ( + "Extract all notable entities and their relationships from the following note.\n\n" + "Entity types to look for:\n" + "- People (named or described, e.g. 'the keeper', 'a sailor')\n" + "- Places (cities, landmarks, buildings, natural features)\n" + "- Objects (vehicles, tools, notable physical things)\n" + "- Organizations, technologies, concepts, events, roles\n\n" + "Rules:\n" + "- Normalize all entity names to lowercase\n" + "- For unnamed characters, use descriptive names (e.g. 'the keeper', 'narrator')\n" + "- Use short predicate names (located_at, part_of, contains, guides, observed_by, " + "works_at, uses, manages, created_by, reports_to, near)\n" + "- Every relationship's subject and object must match an entity name exactly\n" + "- Extract at least the key subjects and locations mentioned\n\n" + "Note title: {title}\n\n" + "Note content:\n{content}" +) + + +async def extract_from_note(kg: Any, note_id: str, title: str, content: str) -> dict[str, Any]: + """Extract entities and relationships from a note and store in the KG. + + Returns counts: {"entities": N, "relationships": M}. + """ + if not content.strip(): + return {"entities": 0, "relationships": 0} + + # Truncate very long content to avoid exceeding model context limits + max_chars = int(os.environ.get("EXTRACTION_MAX_CHARS", "4000")) + truncated_content = content[:max_chars] + + llm = _get_llm() + extractor = llm.with_structured_output(Extraction) + + prompt = EXTRACTION_PROMPT.format(title=title, content=truncated_content) + + try: + result = await extractor.ainvoke(prompt) + except Exception as exc: + logger.exception("Extraction failed for note %s", note_id) + return {"entities": 0, "relationships": 0, "error": str(exc)} + + # Retract ALL old extractions for this note, then re-insert + # (both text and image entities share the same source_note_id) + await kg.execute( + f'-entity(Id, N, K, D, Src) <- entity(Id, N, K, D, Src), Src = "{note_id}"' + ) + await kg.execute( + f'-relationship(Id, S, P, O, Src) <- relationship(Id, S, P, O, Src), Src = "{note_id}"' + ) + + # Insert entities + entity_names = set() + for i, e in enumerate(result.entities): + eid = f"t_{note_id}_e{i}" + entity_names.add(e.name) + await kg.execute( + f"+entity({iql_literal(eid)}, {iql_literal(e.name)}, " + f"{iql_literal(e.kind)}, {iql_literal(e.description)}, " + f"{iql_literal(note_id)})" + ) + + # Auto-create entities referenced in relationships but not in the entity list + extra_idx = len(result.entities) + for r in result.relationships: + for name in [r.subject, r.object]: + if name not in entity_names: + eid = f"t_{note_id}_e{extra_idx}" + extra_idx += 1 + entity_names.add(name) + await kg.execute( + f"+entity({iql_literal(eid)}, {iql_literal(name)}, " + f'"concept", "auto-created from relationship", ' + f"{iql_literal(note_id)})" + ) + + # Insert relationships (only where both sides exist as entities) + for i, r in enumerate(result.relationships): + if r.subject not in entity_names or r.object not in entity_names: + continue + rid = f"t_{note_id}_r{i}" + await kg.execute( + f"+relationship({iql_literal(rid)}, {iql_literal(r.subject)}, " + f"{iql_literal(r.predicate)}, {iql_literal(r.object)}, " + f"{iql_literal(note_id)})" + ) + + logger.info( + "Extracted %d entities, %d relationships from note %s", + len(result.entities), + len(result.relationships), + note_id, + ) + return {"entities": len(result.entities), "relationships": len(result.relationships)} diff --git a/demos/reasoning-notebook/backend/images.py b/demos/reasoning-notebook/backend/images.py new file mode 100644 index 00000000..04401a83 --- /dev/null +++ b/demos/reasoning-notebook/backend/images.py @@ -0,0 +1,240 @@ +"""Image upload, storage, and multimodal extraction.""" + +from __future__ import annotations + +import base64 +import logging +import os +import uuid +from pathlib import Path +from typing import Any + +from langchain_openai import ChatOpenAI +from pydantic import BaseModel, Field + +from inputlayer import Relation +from inputlayer.integrations.langchain.params import iql_literal + +from config import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL + +logger = logging.getLogger("reasoning_notebook.images") + +UPLOAD_DIR = Path(__file__).parent / "uploads" +UPLOAD_DIR.mkdir(exist_ok=True) + + +# ── KG Schema ────────────────────────────────────────────────────── + + +class Image(Relation): + """An image attached to a note.""" + + id: str + note_id: str + filename: str + description: str + + +# ── Structured extraction model ──────────────────────────────────── + + +class ImageEntity(BaseModel): + name: str = Field(description="Entity name, lowercase") + kind: str = Field(description="Type: person, place, object, building, artwork, animal, concept") + description: str = Field(description="One-sentence description") + + +class ImageRelationship(BaseModel): + subject: str = Field(description="Source entity name") + predicate: str = Field(description="Relationship type") + object: str = Field(description="Target entity name") + + +class ImageAnalysis(BaseModel): + scene: str = Field(description="Brief scene description, e.g. 'birthday party, indoors'") + objects: list[str] = Field(description="List of objects visible, e.g. ['cake', 'candles', 'balloons']") + people: str = Field(default="none", description="People count and description, e.g. '5 (2 children, 3 adults)' or 'none'") + emotion: str = Field(default="neutral", description="Emotional quality, e.g. 'joyful, celebratory'") + event_type: str = Field(default="", description="Type of event if applicable, e.g. 'birthday', 'ceremony', 'travel'") + aesthetic: str = Field(default="", description="Visual style, e.g. 'warm lighting, candid' or 'dramatic, high contrast'") + caption_seed: str = Field(default="", description="A short phrase that could caption this image, e.g. 'blowing out the candles'") + cultural_context: str = Field(default="", description="Cultural or historical context if applicable") + visible_text: str = Field(default="", description="Any text visible in the image") + entities: list[ImageEntity] = Field(default_factory=list, description="Notable entities found") + relationships: list[ImageRelationship] = Field(default_factory=list, description="Relationships between entities") + + +# ── KG Schema for image analysis ─────────────────────────────────── + + +class ImageScene(Relation): + """Scene-level analysis of an image.""" + + image_id: str + note_id: str + scene: str + objects: str + people: str + emotion: str + event_type: str + aesthetic: str + caption_seed: str + cultural_context: str + visible_text: str + + +# ── Storage ──────────────────────────────────────────────────────── + + +def save_image(data: bytes, original_filename: str) -> tuple[str, str]: + """Save image to disk. Returns (image_id, filepath).""" + image_id = uuid.uuid4().hex[:12] + ext = Path(original_filename).suffix or ".jpg" + filename = f"{image_id}{ext}" + filepath = UPLOAD_DIR / filename + filepath.write_bytes(data) + return image_id, filename + + +def get_image_path(filename: str) -> Path | None: + """Get the full path to an uploaded image.""" + path = UPLOAD_DIR / filename + return path if path.is_file() else None + + +# ── Multimodal extraction ────────────────────────────────────────── + + +VISION_PROMPT = ( + "Analyze this image and extract structured information.\n\n" + "Provide:\n" + "- scene: brief description (e.g. 'birthday party, indoors')\n" + "- objects: list of visible objects (e.g. ['cake', 'candles'])\n" + "- people: count and description (e.g. '5 (2 children, 3 adults)') or 'none'\n" + "- emotion: emotional quality (e.g. 'joyful, celebratory')\n" + "- event_type: type of event (e.g. 'birthday', 'ceremony', 'travel')\n" + "- aesthetic: visual style (e.g. 'warm lighting, candid')\n" + "- caption_seed: a short caption phrase (e.g. 'blowing out the candles')\n" + "- cultural_context: cultural or historical context if any\n" + "- visible_text: any text visible in the image\n" + "- entities: notable entities with name, kind, description\n" + "- relationships: how entities relate to each other" +) + + +async def extract_from_image( + kg: Any, image_id: str, note_id: str, filename: str +) -> dict[str, Any]: + """Send image to vision LLM and extract entities into the KG.""" + filepath = UPLOAD_DIR / filename + if not filepath.is_file(): + return {"error": "Image file not found"} + + img_bytes = filepath.read_bytes() + img_b64 = base64.b64encode(img_bytes).decode() + + # Detect mime type + ext = filepath.suffix.lower() + mime = {"jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", + ".gif": "image/gif", ".webp": "image/webp"}.get(ext, "image/jpeg") + + llm = ChatOpenAI( + base_url=LLM_BASE_URL, + api_key=LLM_API_KEY, + model=LLM_MODEL, + temperature=0, + max_tokens=800, + ) + + # Step 1: Get a text description (plain completion, more reliable) + try: + from langchain_core.messages import HumanMessage + + description_resp = await llm.ainvoke([ + HumanMessage(content=[ + {"type": "text", "text": "Describe this image in 2-3 sentences. Be specific about what you see."}, + {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{img_b64}"}}, + ]) + ]) + description = description_resp.content.strip() + except Exception: + logger.exception("Vision description failed for image %s", image_id) + description = "" + + # Step 2: Structured extraction with rich schema + entities_count = 0 + relationships_count = 0 + analysis = None + try: + extractor = llm.with_structured_output(ImageAnalysis) + analysis = await extractor.ainvoke([ + HumanMessage(content=[ + {"type": "text", "text": VISION_PROMPT}, + {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{img_b64}"}}, + ]) + ]) + + # Store image record with description + await kg.execute( + f"+image({iql_literal(image_id)}, {iql_literal(note_id)}, " + f"{iql_literal(filename)}, {iql_literal(description)})" + ) + + # Store scene-level analysis + await kg.execute( + f"+image_scene({iql_literal(image_id)}, {iql_literal(note_id)}, " + f"{iql_literal(analysis.scene)}, {iql_literal(', '.join(analysis.objects))}, " + f"{iql_literal(analysis.people)}, {iql_literal(analysis.emotion)}, " + f"{iql_literal(analysis.event_type)}, {iql_literal(analysis.aesthetic)}, " + f"{iql_literal(analysis.caption_seed)}, {iql_literal(analysis.cultural_context)}, " + f"{iql_literal(analysis.visible_text)})" + ) + + # Store extracted entities with img: source prefix + img_source = f"img:{note_id}" + for i, e in enumerate(analysis.entities): + eid = f"i_{image_id}_e{i}" + await kg.execute( + f"+entity({iql_literal(eid)}, {iql_literal(e.name)}, " + f"{iql_literal(e.kind)}, {iql_literal(e.description)}, " + f"{iql_literal(img_source)})" + ) + entities_count += 1 + + # Store extracted relationships with img: source prefix + for i, r in enumerate(analysis.relationships): + rid = f"i_{image_id}_r{i}" + await kg.execute( + f"+relationship({iql_literal(rid)}, {iql_literal(r.subject)}, " + f"{iql_literal(r.predicate)}, {iql_literal(r.object)}, " + f"{iql_literal(img_source)})" + ) + relationships_count += 1 + + except Exception: + logger.exception("Vision structured extraction failed for image %s", image_id) + + logger.info( + "Image %s: description=%d chars, entities=%d, relationships=%d", + image_id, len(description), entities_count, relationships_count, + ) + + result: dict[str, Any] = { + "image_id": image_id, + "description": description, + "entities": entities_count, + "relationships": relationships_count, + } + if analysis: + result["analysis"] = { + "scene": analysis.scene, + "objects": analysis.objects, + "people": analysis.people, + "emotion": analysis.emotion, + "event_type": analysis.event_type, + "aesthetic": analysis.aesthetic, + "caption_seed": analysis.caption_seed, + "cultural_context": analysis.cultural_context, + "visible_text": analysis.visible_text, + } + return result diff --git a/demos/reasoning-notebook/backend/main.py b/demos/reasoning-notebook/backend/main.py new file mode 100644 index 00000000..01780ba3 --- /dev/null +++ b/demos/reasoning-notebook/backend/main.py @@ -0,0 +1,683 @@ +from __future__ import annotations + +import logging +import time +import uuid +from contextlib import asynccontextmanager +from typing import Any + +from fastapi import FastAPI, HTTPException, Request +from fastapi.middleware.cors import CORSMiddleware + +from fastapi import BackgroundTasks +from inputlayer import InputLayer, Relation + +from config import ( + FRONTEND_ORIGIN, + INPUTLAYER_PASSWORD, + INPUTLAYER_URL, + INPUTLAYER_USER, + KG_NAME, +) +from chat import chat as chat_fn +from extraction import Entity, Relationship, extract_from_note +from images import Image, ImageScene, extract_from_image, get_image_path, save_image +from ontology import consolidate_ontology +from resolution import EntityEmbedding, resolve_entities +from schemas import ChatRequest, ChatResponse, NoteCreate, NoteResponse, NoteUpdate + +logger = logging.getLogger("reasoning_notebook") +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s") + + +# ── KG Schema ────────────────────────────────────────────────────── + + +class Note(Relation): + """A markdown note stored in the knowledge graph.""" + + id: str + title: str + content: str + created_at: int + updated_at: int + + +# ── App setup ────────────────────────────────────────────────────── + + +async def _connect(app: FastAPI) -> None: + """Connect to InputLayer and deploy schema. Used at startup and for reconnect.""" + il = InputLayer(INPUTLAYER_URL, username=INPUTLAYER_USER, password=INPUTLAYER_PASSWORD) + await il.connect() + logger.info("Connected to InputLayer at %s", INPUTLAYER_URL) + + kg = il.knowledge_graph(KG_NAME) + await kg.define(Note, Entity, Relationship, EntityEmbedding, Image, ImageScene) + + # Derived rules — these create inferred facts from extracted data + rules = [ + # Two people mentioned in the same note are colleagues + '+colleague(A, B) <- entity(_, A, "person", _, S), entity(_, B, "person", _, S), A != B', + # Direct connection via any relationship (bidirectional) + "+connected(A, B) <- relationship(_, A, _, B, _)", + "+connected(A, B) <- relationship(_, B, _, A, _)", + ] + for rule in rules: + try: + await kg.execute(rule) + except Exception: + pass # rule may already exist + logger.info("Schema and rules deployed") + + app.state.il = il + app.state.kg = kg + + +@asynccontextmanager +async def lifespan(app: FastAPI): + await _connect(app) + yield + await app.state.il.close() + logger.info("Disconnected from InputLayer") + + +app = FastAPI(title="Reasoning Notebook", lifespan=lifespan) + +app.add_middleware( + CORSMiddleware, + allow_origins=[FRONTEND_ORIGIN], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +async def get_kg(request: Request) -> Any: + """Get the KG handle, reconnecting if the WebSocket dropped.""" + try: + il = request.app.state.il + conn = il._conn + ws = conn._ws + if ws is None or not conn._connected or (hasattr(ws, "close_code") and ws.close_code is not None): + raise ConnectionError("stale connection") + except (AttributeError, ConnectionError): + logger.warning("WebSocket disconnected, reconnecting...") + await _connect(request.app) + return request.app.state.kg + + +# ── Health ───────────────────────────────────────────────────────── + + +@app.get("/health") +async def health(request: Request): + kg = await get_kg(request) + try: + result = await kg.execute("?__health(1)") + except Exception: + result = None + return { + "status": "ok", + "engine": "connected" if result is not None else "error", + "kg": KG_NAME, + } + + +# ── Note CRUD ────────────────────────────────────────────────────── + + +def _unescape_iql_string(s: Any) -> Any: + """Unescape IQL string literals returned by the engine.""" + if not isinstance(s, str): + return s + return ( + s.replace("\\n", "\n") + .replace("\\r", "\r") + .replace("\\t", "\t") + .replace("\\0", "\x00") + .replace('\\"', '"') + .replace("\\\\", "\\") + ) + + +def _row_to_note(columns: list[str], row: list[Any]) -> NoteResponse: + data = {k: _unescape_iql_string(v) for k, v in zip(columns, row, strict=True)} + return NoteResponse(**data) + + +@app.post("/notes", status_code=201) +async def create_note(body: NoteCreate, request: Request) -> NoteResponse: + kg = await get_kg(request) + now = int(time.time()) + note = Note( + id=uuid.uuid4().hex[:12], + title=body.title, + content=body.content, + created_at=now, + updated_at=now, + ) + await kg.insert(note) + return NoteResponse( + id=note.id, + title=note.title, + content=note.content, + created_at=note.created_at, + updated_at=note.updated_at, + ) + + +@app.get("/notes") +async def list_notes(request: Request) -> list[NoteResponse]: + kg = await get_kg(request) + result = await kg.execute("?note(Id, Title, Content, CreatedAt, UpdatedAt)") + if not result.rows or result.columns == ["error"]: + return [] + return [_row_to_note(result.columns, row) for row in result.rows] + + +@app.get("/notes/{note_id}") +async def get_note(note_id: str, request: Request) -> NoteResponse: + kg = await get_kg(request) + result = await kg.execute( + f'?note("{note_id}", Title, Content, CreatedAt, UpdatedAt)' + ) + if not result.rows or result.columns == ["error"]: + raise HTTPException(status_code=404, detail="Note not found") + return _row_to_note(result.columns, result.rows[0]) + + +@app.put("/notes/{note_id}") +async def update_note( + note_id: str, + body: NoteUpdate, + request: Request, + bg: BackgroundTasks, +) -> NoteResponse: + kg = await get_kg(request) + + existing = await kg.execute( + f'?note("{note_id}", Title, Content, CreatedAt, UpdatedAt)' + ) + if not existing.rows or existing.columns == ["error"]: + raise HTTPException(status_code=404, detail="Note not found") + + old_data = dict(zip(existing.columns, existing.rows[0], strict=True)) + new_title = body.title if body.title is not None else old_data["title"] + new_content = body.content if body.content is not None else old_data["content"] + created_at = old_data["created_at"] + now = int(time.time()) + + await kg.execute( + f'-note(Id, T, C, Ca, Ua) <- note(Id, T, C, Ca, Ua), Id = "{note_id}"' + ) + note = Note( + id=note_id, + title=new_title, + content=new_content, + created_at=created_at, + updated_at=now, + ) + await kg.insert(note) + + bg.add_task(extract_from_note, kg, note_id, new_title, new_content) + + return NoteResponse( + id=note_id, + title=new_title, + content=new_content, + created_at=created_at, + updated_at=now, + ) + + +@app.delete("/notes/{note_id}", status_code=204) +async def delete_note(note_id: str, request: Request): + kg = await get_kg(request) + img_source = f"img:{note_id}" + await kg.execute( + f'-note(Id, T, C, Ca, Ua) <- note(Id, T, C, Ca, Ua), Id = "{note_id}"' + ) + # Delete text-extracted entities + await kg.execute( + f'-entity(Id, N, K, D, Src) <- entity(Id, N, K, D, Src), Src = "{note_id}"' + ) + await kg.execute( + f'-relationship(Id, S, P, O, Src) <- relationship(Id, S, P, O, Src), Src = "{note_id}"' + ) + # Delete image-extracted entities + await kg.execute( + f'-entity(Id, N, K, D, Src) <- entity(Id, N, K, D, Src), Src = "{img_source}"' + ) + await kg.execute( + f'-relationship(Id, S, P, O, Src) <- relationship(Id, S, P, O, Src), Src = "{img_source}"' + ) + # Delete image records and scene analysis + await kg.execute( + f'-image(Id, N, F, D) <- image(Id, N, F, D), N = "{note_id}"' + ) + await kg.execute( + f'-image_scene(Id, N, S, O, P, E, Et, A, C, Cu, T) <- ' + f'image_scene(Id, N, S, O, P, E, Et, A, C, Cu, T), N = "{note_id}"' + ) + + +# ── Extraction ───────────────────────────────────────────────────── + + +@app.post("/notes/{note_id}/extract") +async def trigger_extraction(note_id: str, request: Request): + kg = await get_kg(request) + result = await kg.execute( + f'?note("{note_id}", Title, Content, CreatedAt, UpdatedAt)' + ) + logger.info( + "Extract lookup note_id=%s columns=%s rows=%d", + note_id, result.columns, len(result.rows or []), + ) + if not result.rows or result.columns == ["error"]: + raise HTTPException(status_code=404, detail="Note not found") + data = dict(zip(result.columns, result.rows[0], strict=True)) + counts = await extract_from_note(kg, note_id, data["title"], data["content"]) + return counts + + +@app.get("/notes/{note_id}/entities") +async def get_note_entities(note_id: str, request: Request): + kg = await get_kg(request) + img_source = f"img:{note_id}" + + # Fetch both text-extracted and image-extracted entities + text_ents = await kg.execute(f'?entity(Id, Name, Kind, Desc, "{note_id}")') + img_ents = await kg.execute(f'?entity(Id, Name, Kind, Desc, "{img_source}")') + text_rels = await kg.execute(f'?relationship(Id, Subject, Predicate, Object, "{note_id}")') + img_rels = await kg.execute(f'?relationship(Id, Subject, Predicate, Object, "{img_source}")') + + def collect_rows(result): + if not result.rows or result.columns == ["error"]: + return [] + return [dict(zip(result.columns, row, strict=True)) for row in result.rows] + + return { + "entities": collect_rows(text_ents) + collect_rows(img_ents), + "relationships": collect_rows(text_rels) + collect_rows(img_rels), + } + + +@app.get("/notes/{note_id}/scenes") +async def get_note_scenes(note_id: str, request: Request): + kg = await get_kg(request) + result = await kg.execute( + f'?image_scene(ImageId, "{note_id}", Scene, Objects, People, Emotion, ' + f'EventType, Aesthetic, Caption, Culture, Text)' + ) + if not result.rows or result.columns == ["error"]: + return [] + return [ + {k.lower(): _unescape_iql_string(v) for k, v in zip(result.columns, row, strict=True)} + for row in result.rows + ] + + +@app.get("/graph") +async def get_graph(request: Request): + kg = await get_kg(request) + ent_result = await kg.execute("?entity(Id, Name, Kind, Desc, SourceNoteId)") + rel_result = await kg.execute("?relationship(Id, Subject, Predicate, Object, SourceNoteId)") + + nodes = [] + if ent_result.rows and ent_result.columns != ["error"]: + for row in ent_result.rows: + data = dict(zip(ent_result.columns, row, strict=True)) + nodes.append(data) + + edges = [] + if rel_result.rows and rel_result.columns != ["error"]: + for row in rel_result.rows: + data = dict(zip(rel_result.columns, row, strict=True)) + edges.append(data) + + # Add image_scene data as graph nodes + scene_result = await kg.execute( + "?image_scene(ImageId, NoteId, Scene, Objects, People, Emotion, " + "EventType, Aesthetic, Caption, Culture, Text)" + ) + if scene_result.rows and scene_result.columns != ["error"]: + for row in scene_result.rows: + sd = dict(zip(scene_result.columns, row, strict=True)) + img_id = sd["image_id"] + note_id = sd["note_id"] + scene_node = f"scene:{img_id}" + + # Scene hub node + nodes.append({ + "id": f"scene_{img_id}", + "name": sd["caption_seed"] or sd["scene"][:40], + "kind": "scene", + "description": sd["scene"], + "source_note_id": note_id, + }) + + # Emotion node + if sd.get("emotion") and sd["emotion"] != "neutral": + nodes.append({ + "id": f"emotion_{img_id}", + "name": sd["emotion"].split(",")[0].strip(), + "kind": "emotion", + "description": sd["emotion"], + "source_note_id": note_id, + }) + edges.append({ + "id": f"scene_emotion_{img_id}", + "subject": sd["caption_seed"] or sd["scene"][:40], + "predicate": "evokes", + "object": sd["emotion"].split(",")[0].strip(), + "source_note_id": note_id, + "derived": True, + }) + + # Event type node + if sd.get("event_type"): + nodes.append({ + "id": f"event_{img_id}", + "name": sd["event_type"], + "kind": "event", + "description": f"Event type: {sd['event_type']}", + "source_note_id": note_id, + }) + edges.append({ + "id": f"scene_event_{img_id}", + "subject": sd["caption_seed"] or sd["scene"][:40], + "predicate": "depicts", + "object": sd["event_type"], + "source_note_id": note_id, + "derived": True, + }) + + # Object nodes from the comma-separated list + objects_str = sd.get("objects", "") + if objects_str: + for obj_name in objects_str.split(", "): + obj_clean = obj_name.strip().lower()[:50] + if not obj_clean: + continue + nodes.append({ + "id": f"obj_{img_id}_{obj_clean[:10]}", + "name": obj_clean, + "kind": "object", + "description": f"Object seen in image", + "source_note_id": note_id, + }) + edges.append({ + "id": f"scene_obj_{img_id}_{obj_clean[:10]}", + "subject": sd["caption_seed"] or sd["scene"][:40], + "predicate": "contains", + "object": obj_clean, + "source_note_id": note_id, + "derived": True, + }) + + # Cultural context node + if sd.get("cultural_context"): + nodes.append({ + "id": f"culture_{img_id}", + "name": sd["cultural_context"][:40], + "kind": "concept", + "description": sd["cultural_context"], + "source_note_id": note_id, + }) + edges.append({ + "id": f"scene_culture_{img_id}", + "subject": sd["caption_seed"] or sd["scene"][:40], + "predicate": "cultural_context", + "object": sd["cultural_context"][:40], + "source_note_id": note_id, + "derived": True, + }) + + # Add derived edges from rules + derived_queries = [ + ("colleague", "?colleague(A, B)"), + ] + entity_names = {n["name"] for n in nodes} + seen_edges = {(e["subject"], e["predicate"], e["object"]) for e in edges} + + for predicate, query in derived_queries: + try: + result = await kg.execute(query) + if result.rows and result.columns != ["error"]: + for row in result.rows: + a, b = row[0], row[1] + if a in entity_names and b in entity_names and (a, predicate, b) not in seen_edges: + edges.append({ + "id": f"derived_{predicate}_{a}_{b}", + "subject": a, + "predicate": predicate, + "object": b, + "source_note_id": "derived", + "derived": True, + }) + seen_edges.add((a, predicate, b)) + except Exception: + pass + + return {"nodes": nodes, "edges": edges} + + +# ── Ontology ─────────────────────────────────────────────────────── + + +@app.get("/ontology/predicates") +async def list_predicates(request: Request): + kg = await get_kg(request) + result = await kg.execute("?relationship(_, _, Predicate, _, _)") + predicates = sorted({row[0] for row in (result.rows or [])}) if result.rows else [] + return {"predicates": predicates} + + +@app.post("/ontology/cleanup") +async def cleanup_orphans(request: Request): + """Remove entities and relationships whose source note no longer exists.""" + kg = await get_kg(request) + notes = await kg.execute("?note(Id, Title, Content, Ca, Ua)") + note_ids = {row[0] for row in (notes.rows or [])} + + def _is_orphan(source: str) -> bool: + clean = source.replace("img:", "") + return clean not in note_ids + + removed = 0 + ents = await kg.execute("?entity(Id, Name, Kind, Desc, Source)") + for row in (ents.rows or []): + if _is_orphan(row[4]): + from inputlayer.integrations.langchain.params import iql_literal + await kg.execute( + f"-entity({iql_literal(row[0])}, {iql_literal(row[1])}, " + f"{iql_literal(row[2])}, {iql_literal(row[3])}, {iql_literal(row[4])})" + ) + removed += 1 + + rels = await kg.execute("?relationship(Id, Subject, Predicate, Object, Source)") + for row in (rels.rows or []): + if _is_orphan(row[4]): + from inputlayer.integrations.langchain.params import iql_literal + await kg.execute( + f"-relationship({iql_literal(row[0])}, {iql_literal(row[1])}, " + f"{iql_literal(row[2])}, {iql_literal(row[3])}, {iql_literal(row[4])})" + ) + removed += 1 + + # Clean orphaned image records and scene analyses + imgs = await kg.execute("?image(Id, NoteId, Filename, Desc)") + for row in (imgs.rows or []): + if row[1] not in note_ids: + from inputlayer.integrations.langchain.params import iql_literal + await kg.execute( + f"-image({iql_literal(row[0])}, {iql_literal(row[1])}, " + f"{iql_literal(row[2])}, {iql_literal(row[3])})" + ) + removed += 1 + + scenes = await kg.execute( + "?image_scene(ImageId, NoteId, Scene, Objects, People, Emotion, " + "EventType, Aesthetic, Caption, Culture, Text)" + ) + for row in (scenes.rows or []): + if row[1] not in note_ids: + from inputlayer.integrations.langchain.params import iql_literal + vals = ", ".join(iql_literal(v) for v in row) + await kg.execute(f"-image_scene({vals})") + removed += 1 + + return {"removed": removed} + + +@app.post("/ontology/consolidate") +async def consolidate(request: Request): + kg = await get_kg(request) + return await consolidate_ontology(kg) + + +@app.post("/ontology/resolve") +async def resolve(request: Request): + kg = await get_kg(request) + return await resolve_entities(kg) + + +# ── Chat ─────────────────────────────────────────────────────────── + + +@app.post("/chat") +async def chat_endpoint(body: ChatRequest, request: Request) -> ChatResponse: + kg = await get_kg(request) + reply = await chat_fn(kg, body.message, body.history) + return ChatResponse(reply=reply) + + +# ── Provenance ───────────────────────────────────────────────────── + + +def _proof_tree_to_dict(tree) -> dict[str, Any]: + """Serialize a ProofTree to a JSON-safe dict.""" + nodes = {} + for nid, node in tree.nodes.items(): + nodes[nid] = { + "kind": node.kind, + "conclusion": {"pred": node.conclusion.pred, "args": node.conclusion.args}, + "children": node.children or [], + "source": node.source, + "rule_id": node.rule_id, + "bindings": node.bindings, + } + return {"roots": tree.roots, "nodes": nodes, "query": tree.query} + + +def _raw_tree_to_dict(tree: dict[str, Any]) -> dict[str, Any]: + """Normalize a raw wire-format proof tree dict.""" + nodes = {} + for nid, node in tree.get("nodes", {}).items(): + conc = node.get("conclusion", {}) + nodes[nid] = { + "kind": node.get("kind", "unknown"), + "conclusion": {"pred": conc.get("pred", ""), "args": conc.get("args", [])}, + "children": node.get("children", []), + "source": node.get("source"), + "rule_id": node.get("rule_id"), + "bindings": node.get("bindings"), + } + return {"roots": tree.get("roots", []), "nodes": nodes, "query": tree.get("query")} + + +@app.post("/why") +async def why_endpoint(request: Request): + body = await request.json() + kg = await get_kg(request) + query = body.get("query", "") + if not query: + raise HTTPException(status_code=400, detail="query is required") + + # Use _execute() to get the raw ResultResponse which includes proof_trees + result = await kg._execute(f".why {query}") + raw_trees = getattr(result, "proof_trees", None) or [] + + from inputlayer.knowledge_graph import ProofTree + + trees = [] + for t in raw_trees: + if isinstance(t, dict): + trees.append(_raw_tree_to_dict(t)) + elif isinstance(t, ProofTree): + trees.append(_proof_tree_to_dict(t)) + + return { + "columns": result.columns, + "rows": result.rows or [], + "proof_trees": trees, + } + + +@app.post("/why_not") +async def why_not_endpoint(request: Request): + body = await request.json() + kg = await get_kg(request) + query = body.get("query", "") + if not query: + raise HTTPException(status_code=400, detail="query is required") + + result = await kg._execute(f".why_not {query}") + text = "\n".join(str(row[0]) for row in (result.rows or [])) + raw_trees = getattr(result, "proof_trees", None) or [] + + from inputlayer.knowledge_graph import ProofTree + + tree = None + if raw_trees: + t = raw_trees[0] + if isinstance(t, dict): + tree = t + elif isinstance(t, ProofTree): + tree = _proof_tree_to_dict(t) + + return {"text": text, "proof_tree": tree} + + +# ── Images ───────────────────────────────────────────────────────── + + +@app.post("/notes/{note_id}/images") +async def upload_image(note_id: str, request: Request): + from fastapi import UploadFile, File + + kg = await get_kg(request) + + # Read multipart form data + form = await request.form() + file = form.get("file") + if file is None: + raise HTTPException(status_code=400, detail="No file uploaded") + + data = await file.read() + filename_orig = getattr(file, "filename", "image.jpg") or "image.jpg" + image_id, filename = save_image(data, filename_orig) + + # Extract in foreground so we can return the description + result = await extract_from_image(kg, image_id, note_id, filename) + + return { + "image_id": image_id, + "filename": filename, + "url": f"/images/{filename}", + "description": result.get("description", ""), + "entities": result.get("entities", 0), + "relationships": result.get("relationships", 0), + } + + +@app.get("/images/{filename}") +async def serve_image(filename: str): + from fastapi.responses import FileResponse + + path = get_image_path(filename) + if path is None: + raise HTTPException(status_code=404, detail="Image not found") + return FileResponse(path) diff --git a/demos/reasoning-notebook/backend/ontology.py b/demos/reasoning-notebook/backend/ontology.py new file mode 100644 index 00000000..b1b794da --- /dev/null +++ b/demos/reasoning-notebook/backend/ontology.py @@ -0,0 +1,213 @@ +"""LangGraph ontology consolidation agent. + +Scans all predicates and entity names, proposes normalizations +(e.g. "works for" / "employed by" -> "works_at"), and applies +the merges as IQL retract+insert operations. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from langchain_openai import ChatOpenAI +from pydantic import BaseModel, Field + +from inputlayer.integrations.langchain.params import iql_literal +from inputlayer.integrations.langgraph import InputLayerState + +from config import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL + +logger = logging.getLogger("reasoning_notebook.ontology") + + +# ── LLM structured output models ────────────────────────────────── + + +class PredicateMerge(BaseModel): + variants: list[str] = Field(description="Predicate names that mean the same thing") + canonical: str = Field(description="Single canonical predicate name to keep (snake_case)") + + +class EntityMerge(BaseModel): + variants: list[str] = Field(description="Entity names that refer to the same thing") + canonical: str = Field(description="Single canonical entity name to keep (lowercase)") + + +class MergeProposal(BaseModel): + predicate_merges: list[PredicateMerge] = Field( + default_factory=list, + description="Groups of synonymous predicates to unify", + ) + entity_merges: list[EntityMerge] = Field( + default_factory=list, + description="Groups of entity names that refer to the same thing", + ) + + +# ── State ────────────────────────────────────────────────────────── + + +class OntologyState(InputLayerState): + predicates: list[str] + entity_names: list[str] + proposal: dict[str, Any] + applied: dict[str, int] + iteration: int + max_iterations: int + status: str + + +# ── Pipeline (no LangGraph graph needed for this simple flow) ────── + + +def _get_llm() -> ChatOpenAI: + return ChatOpenAI( + base_url=LLM_BASE_URL, + api_key=LLM_API_KEY, + model=LLM_MODEL, + temperature=0, + ) + + +CONSOLIDATION_PROMPT = ( + "You are an ontology normalization agent. Given these predicates and entity " + "names from a knowledge graph, identify groups that should be merged.\n\n" + "Rules:\n" + "- Only merge predicates that truly mean the same relationship\n" + "- Pick a canonical name in snake_case (e.g. works_at, reports_to)\n" + "- Only merge entities that clearly refer to the same real-world thing\n" + "- Pick the most complete/common name as canonical (lowercase)\n" + "- If nothing needs merging, return empty lists\n" + "- Do NOT merge predicates that are merely related (e.g. manages vs reports_to)\n\n" + "Predicates: {predicates}\n\n" + "Entity names: {entities}" +) + + +async def consolidate_ontology(kg: Any) -> dict[str, Any]: + """Run one round of ontology consolidation. Returns summary of changes.""" + + # Step 1: Scan distinct predicates and entity names + pred_result = await kg.execute("?relationship(_, _, Predicate, _, _)") + predicates = sorted({row[0] for row in (pred_result.rows or [])}) + + ent_result = await kg.execute("?entity(_, Name, _, _, _)") + entity_names = sorted({row[0] for row in (ent_result.rows or [])}) + + if len(predicates) < 2 and len(entity_names) < 2: + return {"status": "nothing_to_consolidate", "predicates": predicates, "entities": entity_names} + + logger.info("Scanning: %d predicates, %d entities", len(predicates), len(entity_names)) + + # Step 2: LLM proposes merges + llm = _get_llm() + proposer = llm.with_structured_output(MergeProposal) + + try: + proposal = await proposer.ainvoke( + CONSOLIDATION_PROMPT.format( + predicates=", ".join(predicates), + entities=", ".join(entity_names), + ) + ) + except Exception: + logger.exception("Ontology consolidation LLM call failed") + return {"status": "llm_error"} + + pred_merges = 0 + entity_merges = 0 + + # Step 3: Apply predicate merges + for merge in proposal.predicate_merges: + variants_to_replace = [v for v in merge.variants if v != merge.canonical] + for old_pred in variants_to_replace: + # Find all relationships with the old predicate + rows = await kg.execute( + f"?relationship(Id, Subject, {iql_literal(old_pred)}, Object, Source)" + ) + if not rows.rows: + continue + for row in rows.rows: + rid, subject, obj, source = row[0], row[1], row[3], row[4] + # Retract old + await kg.execute( + f"-relationship({iql_literal(rid)}, {iql_literal(subject)}, " + f"{iql_literal(old_pred)}, {iql_literal(obj)}, {iql_literal(source)})" + ) + # Insert with canonical predicate + await kg.execute( + f"+relationship({iql_literal(rid)}, {iql_literal(subject)}, " + f"{iql_literal(merge.canonical)}, {iql_literal(obj)}, {iql_literal(source)})" + ) + pred_merges += 1 + + # Step 4: Apply entity merges + for merge in proposal.entity_merges: + variants_to_replace = [v for v in merge.variants if v != merge.canonical] + for old_name in variants_to_replace: + # Update entities + ent_rows = await kg.execute( + f"?entity(Id, {iql_literal(old_name)}, Kind, Desc, Source)" + ) + for row in (ent_rows.rows or []): + eid, kind, desc, source = row[0], row[2], row[3], row[4] + await kg.execute( + f"-entity({iql_literal(eid)}, {iql_literal(old_name)}, " + f"{iql_literal(kind)}, {iql_literal(desc)}, {iql_literal(source)})" + ) + await kg.execute( + f"+entity({iql_literal(eid)}, {iql_literal(merge.canonical)}, " + f"{iql_literal(kind)}, {iql_literal(desc)}, {iql_literal(source)})" + ) + entity_merges += 1 + + # Update relationships referencing old entity name (as subject) + subj_rows = await kg.execute( + f"?relationship(Id, {iql_literal(old_name)}, Pred, Obj, Source)" + ) + for row in (subj_rows.rows or []): + rid, pred, obj, source = row[0], row[2], row[3], row[4] + await kg.execute( + f"-relationship({iql_literal(rid)}, {iql_literal(old_name)}, " + f"{iql_literal(pred)}, {iql_literal(obj)}, {iql_literal(source)})" + ) + await kg.execute( + f"+relationship({iql_literal(rid)}, {iql_literal(merge.canonical)}, " + f"{iql_literal(pred)}, {iql_literal(obj)}, {iql_literal(source)})" + ) + + # Update relationships referencing old entity name (as object) + obj_rows = await kg.execute( + f"?relationship(Id, Subj, Pred, {iql_literal(old_name)}, Source)" + ) + for row in (obj_rows.rows or []): + rid, subj, pred, source = row[0], row[1], row[2], row[4] + await kg.execute( + f"-relationship({iql_literal(rid)}, {iql_literal(subj)}, " + f"{iql_literal(pred)}, {iql_literal(old_name)}, {iql_literal(source)})" + ) + await kg.execute( + f"+relationship({iql_literal(rid)}, {iql_literal(subj)}, " + f"{iql_literal(pred)}, {iql_literal(merge.canonical)}, {iql_literal(source)})" + ) + + logger.info( + "Consolidated: %d predicate renames, %d entity renames", + pred_merges, + entity_merges, + ) + + return { + "status": "done", + "predicate_merges": [ + {"variants": m.variants, "canonical": m.canonical} + for m in proposal.predicate_merges + ], + "entity_merges": [ + {"variants": m.variants, "canonical": m.canonical} + for m in proposal.entity_merges + ], + "predicates_renamed": pred_merges, + "entities_renamed": entity_merges, + } diff --git a/demos/reasoning-notebook/backend/pyproject.toml b/demos/reasoning-notebook/backend/pyproject.toml new file mode 100644 index 00000000..ebc1484d --- /dev/null +++ b/demos/reasoning-notebook/backend/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "reasoning-notebook" +version = "0.1.0" +description = "Reasoning Notebook demo — Obsidian-like notes with InputLayer, LangChain, and LangGraph" +requires-python = ">=3.10" +dependencies = [ + "fastapi>=0.115", + "uvicorn[standard]>=0.30", + "inputlayer-client-dev[langchain,langgraph]", + "langchain-openai>=0.3", + "python-multipart>=0.0.26", + "jupyter>=1.1.1", + "ipykernel>=7.2.0", + "langchain-anthropic>=1.4.2", +] + +[tool.uv.sources] +inputlayer-client-dev = { path = "../../../packages/inputlayer-py", editable = true } + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF"] diff --git a/demos/reasoning-notebook/backend/resolution.py b/demos/reasoning-notebook/backend/resolution.py new file mode 100644 index 00000000..2e9fd7dd --- /dev/null +++ b/demos/reasoning-notebook/backend/resolution.py @@ -0,0 +1,215 @@ +"""Entity resolution via vector similarity. + +Embeds entity names + descriptions into a simple vector space, +builds an HNSW index, and finds near-duplicates for merging. +""" + +from __future__ import annotations + +import hashlib +import logging +from typing import Any + +from inputlayer import HnswIndex, Relation, Vector +from inputlayer.integrations.langchain.params import iql_literal + +logger = logging.getLogger("reasoning_notebook.resolution") + +EMBED_DIM = 32 + + +# ── KG Schema ────────────────────────────────────────────────────── + + +class EntityEmbedding(Relation): + """Entity embedding for similarity search.""" + + id: str + entity_name: str + embedding: Vector + + +# ── Simple character n-gram embedder (no external service needed) ── + + +def _char_ngram_embed(text: str, dim: int = EMBED_DIM) -> list[float]: + """Deterministic character n-gram embedding. + + Hashes character trigrams into a fixed-size vector. + Not production quality, but works for demonstrating + the HNSW similarity search pattern. + """ + text = text.lower().strip() + vec = [0.0] * dim + for i in range(len(text) - 2): + trigram = text[i : i + 3] + h = int(hashlib.md5(trigram.encode()).hexdigest(), 16) + idx = h % dim + vec[idx] += 1.0 + + # Normalize + norm = sum(v * v for v in vec) ** 0.5 + if norm > 0: + vec = [v / norm for v in vec] + else: + vec = [1.0 / dim**0.5] * dim + return vec + + +# ── Resolution pipeline ─────────────────────────────────────────── + + +async def resolve_entities(kg: Any, threshold: float = 0.95) -> dict[str, Any]: + """Find and merge near-duplicate entities using vector similarity. + + 1. Get all unique entity names + 2. Embed each name + description + 3. Store embeddings and build HNSW index + 4. For each entity, find nearest neighbors above threshold + 5. Merge duplicates (keep the longer/more common name) + + Returns summary of merges performed. + """ + + # Step 1: Get all entities + ent_result = await kg.execute("?entity(Id, Name, Kind, Desc, Source)") + if not ent_result.rows or ent_result.columns == ["error"]: + return {"status": "no_entities"} + + # Group by name + name_info: dict[str, dict[str, Any]] = {} + for row in ent_result.rows: + data = dict(zip(ent_result.columns, row, strict=True)) + name = data["name"] + if name not in name_info: + name_info[name] = { + "kind": data["kind"], + "description": data["description"], + "count": 0, + } + name_info[name]["count"] += 1 + + if len(name_info) < 2: + return {"status": "too_few_entities", "count": len(name_info)} + + # Step 2: Define schema and create embeddings + await kg.define(EntityEmbedding) + + # Clear old embeddings + try: + await kg.execute("-entity_embedding(I, N, E) <- entity_embedding(I, N, E)") + except Exception: + pass + + # Insert embeddings + for name, info in name_info.items(): + embed_text = f"{name} {info['kind']} {info['description']}" + vec = _char_ngram_embed(embed_text) + emb = EntityEmbedding(id=f"emb_{name}", entity_name=name, embedding=vec) + await kg.insert(emb) + + # Step 3: Create HNSW index + try: + await kg.execute(".index drop entity_name_idx") + except Exception: + pass + + await kg.create_index( + HnswIndex( + name="entity_name_idx", + relation=EntityEmbedding, + column="embedding", + metric="cosine", + ) + ) + + # Step 4: Find near-duplicates + merges: list[dict[str, str]] = [] + merged_away: set[str] = set() + + names = sorted(name_info.keys()) + for name in names: + if name in merged_away: + continue + + vec = _char_ngram_embed(f"{name} {name_info[name]['kind']} {name_info[name]['description']}") + + try: + result = await kg.vector_search( + EntityEmbedding, vec, k=5, metric="cosine" + ) + except Exception: + continue + + for row in (result.rows or []): + match_data = {k.lower(): v for k, v in zip(result.columns, row, strict=True)} + match_name = match_data["entity_name"] + if match_name == name or match_name in merged_away: + continue + + # Compute similarity (cosine distance → similarity) + match_vec = _char_ngram_embed( + f"{match_name} {name_info[match_name]['kind']} {name_info[match_name]['description']}" + ) + sim = sum(a * b for a, b in zip(vec, match_vec)) + + if sim >= threshold: + # Keep the longer name as canonical + canonical = name if len(name) >= len(match_name) else match_name + variant = match_name if canonical == name else name + merges.append({"canonical": canonical, "variant": variant, "similarity": round(sim, 3)}) + merged_away.add(variant) + + # Step 5: Apply merges + applied = 0 + for merge in merges: + old_name = merge["variant"] + new_name = merge["canonical"] + + # Update entities + ent_rows = await kg.execute(f"?entity(Id, {iql_literal(old_name)}, Kind, Desc, Source)") + for row in (ent_rows.rows or []): + eid, kind, desc, source = row[0], row[2], row[3], row[4] + await kg.execute( + f"-entity({iql_literal(eid)}, {iql_literal(old_name)}, " + f"{iql_literal(kind)}, {iql_literal(desc)}, {iql_literal(source)})" + ) + await kg.execute( + f"+entity({iql_literal(eid)}, {iql_literal(new_name)}, " + f"{iql_literal(kind)}, {iql_literal(desc)}, {iql_literal(source)})" + ) + applied += 1 + + # Update relationships (subject) + subj_rows = await kg.execute(f"?relationship(Id, {iql_literal(old_name)}, Pred, Obj, Source)") + for row in (subj_rows.rows or []): + rid, pred, obj, source = row[0], row[2], row[3], row[4] + await kg.execute( + f"-relationship({iql_literal(rid)}, {iql_literal(old_name)}, " + f"{iql_literal(pred)}, {iql_literal(obj)}, {iql_literal(source)})" + ) + await kg.execute( + f"+relationship({iql_literal(rid)}, {iql_literal(new_name)}, " + f"{iql_literal(pred)}, {iql_literal(obj)}, {iql_literal(source)})" + ) + + # Update relationships (object) + obj_rows = await kg.execute(f"?relationship(Id, Subj, Pred, {iql_literal(old_name)}, Source)") + for row in (obj_rows.rows or []): + rid, subj, pred, source = row[0], row[1], row[2], row[4] + await kg.execute( + f"-relationship({iql_literal(rid)}, {iql_literal(subj)}, " + f"{iql_literal(pred)}, {iql_literal(old_name)}, {iql_literal(source)})" + ) + await kg.execute( + f"+relationship({iql_literal(rid)}, {iql_literal(subj)}, " + f"{iql_literal(pred)}, {iql_literal(new_name)}, {iql_literal(source)})" + ) + + logger.info("Entity resolution: %d merges applied", applied) + + return { + "status": "done", + "merges": merges, + "entities_renamed": applied, + } diff --git a/demos/reasoning-notebook/backend/schemas.py b/demos/reasoning-notebook/backend/schemas.py new file mode 100644 index 00000000..323c6b0e --- /dev/null +++ b/demos/reasoning-notebook/backend/schemas.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class NoteCreate(BaseModel): + title: str + content: str = "" + + +class NoteUpdate(BaseModel): + title: str | None = None + content: str | None = None + + +class NoteResponse(BaseModel): + id: str + title: str + content: str + created_at: int + updated_at: int + + +class ChatRequest(BaseModel): + message: str + history: list[dict[str, str]] = [] + + +class ChatResponse(BaseModel): + reply: str diff --git a/demos/reasoning-notebook/benchmarks/compare.py b/demos/reasoning-notebook/benchmarks/compare.py new file mode 100644 index 00000000..774802ab --- /dev/null +++ b/demos/reasoning-notebook/benchmarks/compare.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +"""Compare benchmark results and generate a report. + +Usage: + uv run python compare.py # terminal output, latest results + uv run python compare.py --html # open HTML report in browser + uv run python compare.py results/benchmark_*.json # specific file +""" + +from __future__ import annotations + +import json +import sys +import time +import webbrowser +from pathlib import Path + + +def load_results(path: Path) -> list[dict]: + return json.loads(path.read_text()) + + +def print_comparison(results: list[dict]) -> None: + inputs = sorted({r["input"] for r in results}) + models = sorted({r["model"] for r in results}) + + print(f"\n{'=' * 80}") + print(f" Benchmark Comparison — {len(results)} runs across {len(models)} models") + print(f"{'=' * 80}\n") + + for input_name in inputs: + input_results = [r for r in results if r["input"] == input_name] + input_type = input_results[0].get("type", "?") + print(f" Input: {input_name} ({input_type})") + print(f" {'Model':<28} {'Ent':>5} {'Rel':>5} {'Time':>7} {'Status':<12}") + print(f" {'-'*28} {'-'*5} {'-'*5} {'-'*7} {'-'*12}") + + for r in sorted(input_results, key=lambda x: x.get("time_seconds", 999)): + if not r["success"]: + print(f" {r['model']:<28} {'—':>5} {'—':>5} {'—':>7} FAILED") + continue + ents = r.get("entities_count", 0) + rels = r.get("relationships_count", 0) + t = f"{r['time_seconds']}s" + print(f" {r['model']:<28} {ents:>5} {rels:>5} {t:>7} OK") + + print(f"\n Entities extracted:") + for r in sorted(input_results, key=lambda x: x["model"]): + if not r["success"]: + continue + ent_names = sorted({e["name"] for e in r.get("entities", [])}) + print(f" {r['model']:<25} {', '.join(ent_names[:10])}") + + if input_type == "image": + print(f"\n Scene analysis:") + for r in sorted(input_results, key=lambda x: x["model"]): + if not r["success"]: + continue + print(f" {r['model']:<25}") + for field in ["scene", "emotion", "event_type", "aesthetic", "caption_seed"]: + val = r.get(field, "") + if val: + print(f" {field:<16} {val[:60]}") + + print() + + print(f"{'=' * 80}") + print(f" Overall Summary") + print(f"{'=' * 80}\n") + print(f" {'Model':<28} {'Avg Ent':>8} {'Avg Rel':>8} {'Avg Time':>9} {'Success':>8}") + print(f" {'-'*28} {'-'*8} {'-'*8} {'-'*9} {'-'*8}") + + for model in models: + model_results = [r for r in results if r["model"] == model] + successes = [r for r in model_results if r["success"]] + if not successes: + print(f" {model:<28} {'—':>8} {'—':>8} {'—':>9} {f'0/{len(model_results)}':>8}") + continue + avg_ents = sum(r.get("entities_count", 0) for r in successes) / len(successes) + avg_rels = sum(r.get("relationships_count", 0) for r in successes) / len(successes) + avg_time = sum(r["time_seconds"] for r in successes) / len(successes) + print( + f" {model:<28} {avg_ents:>8.1f} {avg_rels:>8.1f} {avg_time:>8.1f}s " + f"{f'{len(successes)}/{len(model_results)}':>8}" + ) + print() + + +# ── HTML Report ──────────────────────────────────────────────────── + + +def _esc(s: str) -> str: + return s.replace("&", "&").replace("<", "<").replace(">", ">") + + +def generate_html(results: list[dict], source_file: str) -> str: + inputs = sorted({r["input"] for r in results}) + models = sorted({r["model"] for r in results}) + ts = time.strftime("%Y-%m-%d %H:%M") + + html = f""" + + + + +LLM Extraction Benchmark + + + +

LLM Extraction Benchmark

+
{ts} — {len(results)} runs across {len(models)} models — {source_file}
+""" + + # Per-input tables + for input_name in inputs: + input_results = [r for r in results if r["input"] == input_name] + input_type = input_results[0].get("type", "?") + + html += f'

{_esc(input_name)} ({input_type})

\n' + html += '\n' + + for r in sorted(input_results, key=lambda x: x.get("time_seconds", 999)): + if not r["success"]: + err = _esc(r.get("error", "unknown")[:80]) + html += f'\n' + continue + ents = r.get("entities_count", 0) + rels = r.get("relationships_count", 0) + t = r["time_seconds"] + html += f'\n' + + html += '
ModelEntitiesRelationshipsTimeStatus
{_esc(r["model"])}———FAILED
{_esc(r["model"])}{ents}{rels}{t}sOK
\n' + + # Entities extracted + html += '

Entities Extracted

\n' + for r in sorted(input_results, key=lambda x: x["model"]): + if not r["success"]: + continue + ent_tags = "".join( + f'{_esc(e["name"])}{_esc(e.get("kind",""))}' + for e in sorted(r.get("entities", []), key=lambda e: e["name"]) + ) + html += f'
{_esc(r["model"])}
{ent_tags or "none"}
\n' + + # Image scene analysis + if input_type == "image": + html += '

Scene Analysis

\n' + for r in sorted(input_results, key=lambda x: x["model"]): + if not r["success"]: + continue + html += f'
{_esc(r["model"])}\n
\n' + for field in ["scene", "objects", "people", "emotion", "event_type", "aesthetic", "caption_seed", "cultural_context", "visible_text"]: + val = r.get(field, "") + if isinstance(val, list): + val = ", ".join(val) + if val: + html += f'{field}{_esc(str(val)[:120])}\n' + html += '
\n' + + # Overall summary + html += '

Overall Summary

\n' + html += '\n' + + max_ents = 1 + max_time = 1 + model_stats = {} + for model in models: + model_results = [r for r in results if r["model"] == model] + successes = [r for r in model_results if r["success"]] + if successes: + avg_ents = sum(r.get("entities_count", 0) for r in successes) / len(successes) + avg_time = sum(r["time_seconds"] for r in successes) / len(successes) + max_ents = max(max_ents, avg_ents) + max_time = max(max_time, avg_time) + model_stats[model] = (avg_ents, sum(r.get("relationships_count", 0) for r in successes) / len(successes), avg_time, len(successes), len(model_results)) + + for model in models: + if model not in model_stats: + model_results = [r for r in results if r["model"] == model] + html += f'\n' + continue + avg_ents, avg_rels, avg_time, succ, total = model_stats[model] + ent_pct = (avg_ents / max_ents) * 100 + time_pct = (avg_time / max_time) * 100 + html += ( + f'' + f'' + f'' + f'' + f'' + f'' + f'' + f'\n' + ) + + html += '
ModelAvg EntitiesAvg RelationshipsAvg TimeSuccessEntity BarTime Bar
{_esc(model)}———0/{len(model_results)}
{_esc(model)}{avg_ents:.1f}{avg_rels:.1f}{avg_time:.1f}s{succ}/{total}
\n' + + # ── Charts via Chart.js ── + # Prepare data for charts + successful_models = [m for m in models if m in model_stats] + + # Per-input breakdown data + chart_labels = json.dumps(successful_models) + + # Entities per input per model + input_datasets_ent = [] + input_datasets_time = [] + palette = ["#89b4fa", "#a6e3a1", "#f9e2af", "#f38ba8", "#cba6f7", "#fab387", "#94e2d5", "#f5c2e7"] + for idx, input_name in enumerate(inputs): + color = palette[idx % len(palette)] + ent_data = [] + time_data = [] + for model in successful_models: + r = next((x for x in results if x["model"] == model and x["input"] == input_name and x["success"]), None) + ent_data.append(r.get("entities_count", 0) if r else 0) + time_data.append(r.get("time_seconds", 0) if r else 0) + input_datasets_ent.append({ + "label": input_name, + "data": ent_data, + "backgroundColor": color, + "borderRadius": 4, + }) + input_datasets_time.append({ + "label": input_name, + "data": time_data, + "backgroundColor": color, + "borderRadius": 4, + }) + + # Relationship data + rel_data = [] + for model in successful_models: + avg = model_stats[model][1] + rel_data.append(round(avg, 1)) + + html += f""" +
+
+

Entities Extracted per Input

+ +
+
+

Extraction Time (seconds)

+ +
+
+ +
+
+

Avg Entities vs Relationships

+ +
+
+

Speed vs Quality

+ +
+
+ + + +""" + return html + + +if __name__ == "__main__": + results_dir = Path(__file__).parent / "results" + do_html = "--html" in sys.argv + args = [a for a in sys.argv[1:] if a != "--html"] + + if args: + path = Path(args[0]) + else: + files = sorted(results_dir.glob("benchmark_*.json")) + if not files: + print("No benchmark results found. Run run_benchmark.py first.") + sys.exit(1) + path = files[-1] + + print(f"Loading: {path}") + results = load_results(path) + print_comparison(results) + + if do_html: + html = generate_html(results, path.name) + out = results_dir / f"{path.stem}.html" + out.write_text(html) + print(f"\nHTML report: {out}") + webbrowser.open(f"file://{out.resolve()}") diff --git a/demos/reasoning-notebook/benchmarks/config.py b/demos/reasoning-notebook/benchmarks/config.py new file mode 100644 index 00000000..bcfac9a6 --- /dev/null +++ b/demos/reasoning-notebook/benchmarks/config.py @@ -0,0 +1,106 @@ +"""Benchmark configuration: models and test inputs.""" + +from __future__ import annotations + +import os + +# ── Models to benchmark ──────────────────────────────────────────── +# Each entry needs: name, provider, base_url, api_key, model +# api_key can be an env var name (resolved at runtime) + +MODELS = [ + # Local (LM Studio) + { + "name": "ministral-3-3b (local)", + "provider": "openai", + "base_url": "http://localhost:1234/v1", + "api_key": "lm-studio", + "model": "mistralai/ministral-3-3b", + "multimodal": True, + }, + # { + # "name": "deepseek-r1-8b (local)", + # "provider": "openai", + # "base_url": "http://localhost:1234/v1", + # "api_key": "lm-studio", + # "model": "deepseek/deepseek-r1-0528-qwen3-8b", + # "multimodal": False, + # }, + { + "name": "gemma-4-e4b (local)", + "provider": "openai", + "base_url": "http://localhost:1234/v1", + "api_key": "lm-studio", + "model": "google/gemma-4-e4b", + "multimodal": True, + }, + { + "name": "glm-4.6v-flash (local)", + "provider": "openai", + "base_url": "http://localhost:1234/v1", + "api_key": "lm-studio", + "model": "zai-org/glm-4.6v-flash", + "multimodal": True, + }, + { + "name": "qwen3.5-9b (local)", + "provider": "openai", + "base_url": "http://localhost:1234/v1", + "api_key": "lm-studio", + "model": "qwen/qwen3.5-9b", + "multimodal": True, + }, + # Cloud — OpenAI + { + "name": "gpt-4o-mini", + "provider": "openai", + "base_url": "https://api.openai.com/v1", + "api_key_env": "OPENAI_API_KEY", + "model": "gpt-4o-mini", + "multimodal": True, + }, + { + "name": "gpt-4o", + "provider": "openai", + "base_url": "https://api.openai.com/v1", + "api_key_env": "OPENAI_API_KEY", + "model": "gpt-4o", + "multimodal": True, + }, + # Cloud — Anthropic + { + "name": "claude-opus-4.7", + "provider": "anthropic", + "api_key_env": "ANTHROPIC_API_KEY", + "model": "claude-opus-4-7", + "multimodal": True, + }, + # { + # "name": "claude-mythos-preview", + # "provider": "anthropic", + # "api_key_env": "ANTHROPIC_API_KEY", + # "model": "claude-mythos-preview", + # "multimodal": True, + # }, + { + "name": "claude-sonnet-4.6", + "provider": "anthropic", + "api_key_env": "ANTHROPIC_API_KEY", + "model": "claude-sonnet-4-6", + "multimodal": True, + }, +] + + +def get_enabled_models() -> list[dict]: + """Return models that have their API keys available.""" + enabled = [] + for m in MODELS: + key_env = m.get("api_key_env") + if key_env: + key = os.environ.get(key_env) + if not key: + continue + m = {**m, "api_key": key} + enabled.append(m) + return enabled diff --git a/demos/reasoning-notebook/benchmarks/inputs/text_corporate.txt b/demos/reasoning-notebook/benchmarks/inputs/text_corporate.txt new file mode 100644 index 00000000..77d6550e --- /dev/null +++ b/demos/reasoning-notebook/benchmarks/inputs/text_corporate.txt @@ -0,0 +1 @@ +Alice is the CTO at Acme Corp. She manages both the ML team and the data engineering team. Bob is a senior ML engineer who reports to Alice. He specializes in recommendation systems using PyTorch and TensorFlow. Carol leads the data engineering team and collaborates with Bob on the recommendation pipeline. The pipeline uses Apache Spark for ETL and feeds data into the ML models. Carol reports to Alice. \ No newline at end of file diff --git a/demos/reasoning-notebook/benchmarks/inputs/text_narrative.txt b/demos/reasoning-notebook/benchmarks/inputs/text_narrative.txt new file mode 100644 index 00000000..a120c0b0 --- /dev/null +++ b/demos/reasoning-notebook/benchmarks/inputs/text_narrative.txt @@ -0,0 +1 @@ +The wind sharpened as we climbed the narrow path to the lighthouse, salt spray misting our faces with every gust. Its white tower rose against a bruised gray sky, the beam already sweeping lazily across the churning water below. Inside, the spiral staircase groaned under our steps, each iron rung echoing like a distant bell. At the top, the keeper pointed out a freighter on the horizon, barely visible through the haze, and told us the light had guided ships home for over a century. Walking back down, I felt smaller somehow, and grateful for it. \ No newline at end of file diff --git a/demos/reasoning-notebook/benchmarks/run_benchmark.py b/demos/reasoning-notebook/benchmarks/run_benchmark.py new file mode 100644 index 00000000..6e319550 --- /dev/null +++ b/demos/reasoning-notebook/benchmarks/run_benchmark.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Run extraction benchmarks across multiple LLMs. + +Usage: + uv run python run_benchmark.py # all enabled models, all inputs + uv run python run_benchmark.py --models local # only local models + uv run python run_benchmark.py --models cloud # only cloud models + uv run python run_benchmark.py --input text_corporate.txt # single input + uv run python run_benchmark.py --image-only # only image inputs + uv run python run_benchmark.py --text-only # only text inputs +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import sys +import time +from pathlib import Path +from typing import Any + +# Import benchmark config BEFORE adding backend to path +bench_dir = Path(__file__).parent +sys.path.insert(0, str(bench_dir)) +from config import get_enabled_models, MODELS # noqa: E402 +sys.path.pop(0) + +# Now add backend to path for langchain deps +sys.path.insert(0, str(bench_dir.parent / "backend")) + +from langchain_openai import ChatOpenAI # noqa: E402 +from pydantic import BaseModel, Field # noqa: E402 + + +# ── Extraction schemas (same as the app) ─────────────────────────── + + +class ExtractedEntity(BaseModel): + name: str = Field(description="Entity name, lowercase") + kind: str = Field(description="Type: person, organization, technology, concept, place, event, object, building, artwork, animal, role") + description: str = Field(description="One-sentence description") + + +class ExtractedRelationship(BaseModel): + subject: str = Field(description="Source entity name") + predicate: str = Field(description="Relationship type") + object: str = Field(description="Target entity name") + + +class TextExtraction(BaseModel): + entities: list[ExtractedEntity] = Field(default_factory=list) + relationships: list[ExtractedRelationship] = Field(default_factory=list) + + +class ImageAnalysis(BaseModel): + scene: str = Field(default="", description="Brief scene description") + objects: list[str] = Field(default_factory=list, description="Visible objects") + people: str = Field(default="none", description="People count and description") + emotion: str = Field(default="neutral", description="Emotional quality") + event_type: str = Field(default="", description="Type of event") + aesthetic: str = Field(default="", description="Visual style") + caption_seed: str = Field(default="", description="Short caption phrase") + cultural_context: str = Field(default="", description="Cultural context") + visible_text: str = Field(default="", description="Visible text in image") + entities: list[ExtractedEntity] = Field(default_factory=list) + relationships: list[ExtractedRelationship] = Field(default_factory=list) + + +# ── Prompts ──────────────────────────────────────────────────────── + + +TEXT_PROMPT = ( + "Extract all notable entities and their relationships from the following text.\n\n" + "Entity types: people (named or described), places, objects, organizations, " + "technologies, concepts, events, roles.\n\n" + "Rules:\n" + "- Normalize all entity names to lowercase\n" + "- Use short predicate names (works_at, manages, uses, located_at, part_of, reports_to)\n" + "- Every relationship's subject and object must match an entity name\n\n" + "Text:\n{content}" +) + +IMAGE_PROMPT = ( + "Analyze this image and extract structured information.\n\n" + "Provide: scene, objects (list), people (count/description or 'none'), " + "emotion, event_type, aesthetic, caption_seed, cultural_context, visible_text, " + "entities (name, kind, description), relationships (subject, predicate, object)." +) + + +# ── Benchmark runner ─────────────────────────────────────────────── + + +async def benchmark_text(model_cfg: dict, text: str, input_name: str) -> dict[str, Any]: + """Run text extraction benchmark for one model.""" + result: dict[str, Any] = { + "model": model_cfg["name"], + "input": input_name, + "type": "text", + "input_chars": len(text), + } + + try: + if model_cfg["provider"] == "anthropic": + from langchain_anthropic import ChatAnthropic + kwargs: dict[str, Any] = { + "api_key": model_cfg["api_key"], + "model": model_cfg["model"], + "max_tokens": 1024, + } + if "opus-4-7" not in model_cfg["model"] and "mythos" not in model_cfg["model"]: + kwargs["temperature"] = 0 + llm = ChatAnthropic(**kwargs) + else: + llm = ChatOpenAI( + base_url=model_cfg.get("base_url"), + api_key=model_cfg.get("api_key", ""), + model=model_cfg["model"], + temperature=0, + max_tokens=1024, + ) + + extractor = llm.with_structured_output(TextExtraction) + prompt = TEXT_PROMPT.format(content=text) + + start = time.time() + extraction = await extractor.ainvoke(prompt) + elapsed = time.time() - start + + result["success"] = True + result["time_seconds"] = round(elapsed, 2) + result["entities_count"] = len(extraction.entities) + result["relationships_count"] = len(extraction.relationships) + result["entities"] = [ + {"name": e.name, "kind": e.kind, "description": e.description} + for e in extraction.entities + ] + result["relationships"] = [ + {"subject": r.subject, "predicate": r.predicate, "object": r.object} + for r in extraction.relationships + ] + + except Exception as e: + result["success"] = False + result["error"] = f"{type(e).__name__}: {str(e)[:200]}" + result["time_seconds"] = 0 + + return result + + +async def benchmark_image(model_cfg: dict, image_path: Path, input_name: str) -> dict[str, Any]: + """Run image extraction benchmark for one model.""" + result: dict[str, Any] = { + "model": model_cfg["name"], + "input": input_name, + "type": "image", + "input_bytes": image_path.stat().st_size, + } + + if not model_cfg.get("multimodal"): + result["success"] = False + result["error"] = "Model does not support multimodal input" + result["time_seconds"] = 0 + return result + + img_b64 = base64.b64encode(image_path.read_bytes()).decode() + ext = image_path.suffix.lower() + mime = {".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png"}.get(ext, "image/jpeg") + + try: + if model_cfg["provider"] == "anthropic": + from langchain_anthropic import ChatAnthropic + kwargs: dict[str, Any] = { + "api_key": model_cfg["api_key"], + "model": model_cfg["model"], + "max_tokens": 1024, + } + if "opus-4-7" not in model_cfg["model"] and "mythos" not in model_cfg["model"]: + kwargs["temperature"] = 0 + llm = ChatAnthropic(**kwargs) + else: + llm = ChatOpenAI( + base_url=model_cfg.get("base_url"), + api_key=model_cfg.get("api_key", ""), + model=model_cfg["model"], + temperature=0, + max_tokens=1024, + ) + + from langchain_core.messages import HumanMessage + extractor = llm.with_structured_output(ImageAnalysis) + + start = time.time() + analysis = await extractor.ainvoke([ + HumanMessage(content=[ + {"type": "text", "text": IMAGE_PROMPT}, + {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{img_b64}"}}, + ]) + ]) + elapsed = time.time() - start + + result["success"] = True + result["time_seconds"] = round(elapsed, 2) + result["scene"] = analysis.scene + result["objects"] = analysis.objects + result["people"] = analysis.people + result["emotion"] = analysis.emotion + result["event_type"] = analysis.event_type + result["aesthetic"] = analysis.aesthetic + result["caption_seed"] = analysis.caption_seed + result["cultural_context"] = analysis.cultural_context + result["visible_text"] = analysis.visible_text + result["entities_count"] = len(analysis.entities) + result["relationships_count"] = len(analysis.relationships) + result["entities"] = [ + {"name": e.name, "kind": e.kind, "description": e.description} + for e in analysis.entities + ] + result["relationships"] = [ + {"subject": r.subject, "predicate": r.predicate, "object": r.object} + for r in analysis.relationships + ] + + except Exception as e: + result["success"] = False + result["error"] = f"{type(e).__name__}: {str(e)[:200]}" + result["time_seconds"] = 0 + + return result + + +# ── Main ─────────────────────────────────────────────────────────── + + +BOLD = "\033[1m" +DIM = "\033[2m" +GREEN = "\033[32m" +RED = "\033[31m" +YELLOW = "\033[33m" +CYAN = "\033[36m" +RESET = "\033[0m" + + +async def main(): + parser = argparse.ArgumentParser(description="Benchmark LLM extraction") + parser.add_argument("--models", choices=["all", "local", "cloud"], default="all") + parser.add_argument("--input", type=str, help="Specific input file name") + parser.add_argument("--text-only", action="store_true") + parser.add_argument("--image-only", action="store_true") + args = parser.parse_args() + + models = get_enabled_models() + if args.models == "local": + models = [m for m in models if "localhost" in m.get("base_url", "")] + elif args.models == "cloud": + models = [m for m in models if "localhost" not in m.get("base_url", "")] + + if not models: + print(f"{RED}No models available. Set OPENAI_API_KEY / ANTHROPIC_API_KEY for cloud models,") + print(f"or start LM Studio for local models.{RESET}") + print(f"\nConfigured models:") + for m in MODELS: + key_env = m.get("api_key_env", "") + status = "available" if not key_env else ("set" if key_env and __import__("os").environ.get(key_env) else f"missing {key_env}") + print(f" {m['name']}: {status}") + return + + inputs_dir = Path(__file__).parent / "inputs" + results_dir = Path(__file__).parent / "results" + results_dir.mkdir(exist_ok=True) + + # Collect inputs + text_inputs = [] + image_inputs = [] + for f in sorted(inputs_dir.iterdir()): + if args.input and f.name != args.input: + continue + if f.suffix == ".txt" and not args.image_only: + text_inputs.append(f) + elif f.suffix in (".jpg", ".jpeg", ".png") and not args.text_only: + image_inputs.append(f) + + print(f"\n{BOLD}{'=' * 60}{RESET}") + print(f"{BOLD} LLM Extraction Benchmark{RESET}") + print(f"{BOLD}{'=' * 60}{RESET}") + print(f"\n Models: {len(models)}") + print(f" Text inputs: {len(text_inputs)}") + print(f" Image inputs: {len(image_inputs)}") + print() + + all_results = [] + + for model in models: + print(f"{CYAN}{BOLD} Model: {model['name']}{RESET}") + + for text_file in text_inputs: + text = text_file.read_text().strip() + print(f" {DIM}Text: {text_file.name} ({len(text)} chars)...{RESET}", end=" ", flush=True) + result = await benchmark_text(model, text, text_file.name) + all_results.append(result) + + if result["success"]: + print(f"{GREEN}{result['entities_count']} entities, " + f"{result['relationships_count']} rels " + f"({result['time_seconds']}s){RESET}") + else: + print(f"{RED}FAILED: {result.get('error', '?')[:60]}{RESET}") + + for img_file in image_inputs: + print(f" {DIM}Image: {img_file.name}...{RESET}", end=" ", flush=True) + result = await benchmark_image(model, img_file, img_file.name) + all_results.append(result) + + if result["success"]: + print(f"{GREEN}{result['entities_count']} entities, " + f"{result['relationships_count']} rels " + f"({result['time_seconds']}s){RESET}") + print(f" {DIM}Scene: {result.get('scene', '?')[:60]}{RESET}") + print(f" {DIM}Caption: {result.get('caption_seed', '?')[:60]}{RESET}") + else: + print(f"{YELLOW}{result.get('error', '?')[:60]}{RESET}") + + print() + + # Save results + timestamp = time.strftime("%Y%m%d_%H%M%S") + output_file = results_dir / f"benchmark_{timestamp}.json" + output_file.write_text(json.dumps(all_results, indent=2)) + print(f" Results saved to: {output_file}") + + # Summary table + print(f"\n{BOLD}{'=' * 60}{RESET}") + print(f"{BOLD} Summary{RESET}") + print(f"{BOLD}{'=' * 60}{RESET}\n") + + # Group by input + inputs_seen = sorted({r["input"] for r in all_results}) + for input_name in inputs_seen: + print(f" {BOLD}{input_name}{RESET}") + print(f" {'Model':<25} {'Status':<8} {'Entities':<10} {'Rels':<8} {'Time':<8}") + print(f" {'-'*25} {'-'*8} {'-'*10} {'-'*8} {'-'*8}") + for r in all_results: + if r["input"] != input_name: + continue + status = f"{GREEN}OK{RESET}" if r["success"] else f"{RED}FAIL{RESET}" + ents = str(r.get("entities_count", "-")) + rels = str(r.get("relationships_count", "-")) + t = f"{r['time_seconds']}s" if r["success"] else "-" + print(f" {r['model']:<25} {status:<17} {ents:<10} {rels:<8} {t:<8}") + print() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/demos/reasoning-notebook/docker-compose.yml b/demos/reasoning-notebook/docker-compose.yml new file mode 100644 index 00000000..b927f1e8 --- /dev/null +++ b/demos/reasoning-notebook/docker-compose.yml @@ -0,0 +1,30 @@ +services: + engine: + build: + context: ../.. + dockerfile: Dockerfile + ports: + - "8080:8080" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 5s + timeout: 3s + retries: 10 + + backend: + build: ./backend + ports: + - "8000:8000" + environment: + - INPUTLAYER_URL=ws://engine:8080/ws + - FRONTEND_ORIGIN=http://localhost:5173 + depends_on: + engine: + condition: service_healthy + + frontend: + build: ./frontend + ports: + - "5173:5173" + depends_on: + - backend diff --git a/demos/reasoning-notebook/frontend/index.html b/demos/reasoning-notebook/frontend/index.html new file mode 100644 index 00000000..aac15d08 --- /dev/null +++ b/demos/reasoning-notebook/frontend/index.html @@ -0,0 +1,20 @@ + + + + + + Reasoning Notebook + + + +
+ + + diff --git a/demos/reasoning-notebook/frontend/package.json b/demos/reasoning-notebook/frontend/package.json new file mode 100644 index 00000000..29e9efbb --- /dev/null +++ b/demos/reasoning-notebook/frontend/package.json @@ -0,0 +1,33 @@ +{ + "name": "reasoning-notebook", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview" + }, + "dependencies": { + "@milkdown/core": "^7.20.0", + "@milkdown/ctx": "^7.20.0", + "@milkdown/plugin-listener": "^7.20.0", + "@milkdown/plugin-upload": "^7.20.0", + "@milkdown/preset-commonmark": "^7.20.0", + "@milkdown/preset-gfm": "^7.20.0", + "@milkdown/react": "^7.20.0", + "@milkdown/theme-nord": "^7.20.0", + "@milkdown/utils": "^7.20.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-force-graph-2d": "^1.29.1", + "react-markdown": "^10.1.0" + }, + "devDependencies": { + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", + "@vitejs/plugin-react": "^4.4.0", + "typescript": "~5.7.0", + "vite": "^6.3.0" + } +} diff --git a/demos/reasoning-notebook/frontend/src/App.tsx b/demos/reasoning-notebook/frontend/src/App.tsx new file mode 100644 index 00000000..4e80388e --- /dev/null +++ b/demos/reasoning-notebook/frontend/src/App.tsx @@ -0,0 +1,292 @@ +import { useCallback, useEffect, useState } from "react"; +import { + createNote, + deleteNote, + fetchHealth, + fetchNotes, + updateNote, + type HealthResponse, +} from "./api"; +import { ChatPanel } from "./components/ChatPanel"; +import { Editor } from "./components/Editor"; +import { ExtractionPanel } from "./components/ExtractionPanel"; +import { GraphView } from "./components/GraphView"; +import { Sidebar } from "./components/Sidebar"; +import type { Note } from "./types"; + +type View = "editor" | "graph" | "chat"; + +export function App() { + const [health, setHealth] = useState(null); + const [error, setError] = useState(null); + const [notes, setNotes] = useState([]); + const [activeId, setActiveId] = useState(null); + const [saveCount, setSaveCount] = useState(0); + const [view, setView] = useState("editor"); + + useEffect(() => { + fetchHealth() + .then(setHealth) + .catch((e) => setError(e.message)); + loadNotes(); + }, []); + + // Global keyboard shortcuts + useEffect(() => { + const handler = (e: KeyboardEvent) => { + const mod = e.metaKey || e.ctrlKey; + if (mod && e.key === "n") { + e.preventDefault(); + handleCreate(); + } else if (mod && e.key === "k") { + e.preventDefault(); + setView("chat"); + } else if (mod && e.key === "g") { + e.preventDefault(); + setView("graph"); + } else if (mod && e.key === "e") { + e.preventDefault(); + setView("editor"); + } + }; + window.addEventListener("keydown", handler); + return () => window.removeEventListener("keydown", handler); + }); + + const loadNotes = async () => { + try { + const list = await fetchNotes(); + list.sort((a, b) => b.updated_at - a.updated_at); + setNotes(list); + } catch { + /* backend not ready yet */ + } + }; + + const handleCreate = async () => { + const note = await createNote("Untitled"); + setNotes((prev) => [note, ...prev]); + setActiveId(note.id); + setView("editor"); + }; + + const handleDelete = async (id: string) => { + await deleteNote(id); + setNotes((prev) => prev.filter((n) => n.id !== id)); + if (activeId === id) setActiveId(null); + }; + + const handleSave = useCallback( + async (id: string, fields: { title?: string; content?: string }) => { + const updated = await updateNote(id, fields); + setNotes((prev) => + prev + .map((n) => (n.id === id ? updated : n)) + .sort((a, b) => b.updated_at - a.updated_at) + ); + setSaveCount((c) => c + 1); + }, + [] + ); + + const handleSelectFromGraph = (noteId: string) => { + setActiveId(noteId); + setView("editor"); + }; + + const activeNote = notes.find((n) => n.id === activeId) ?? null; + + return ( +
+
+

Reasoning Notebook

+
+ + + +
+ +
+
+ {view === "editor" && ( + <> + +
+ {activeNote ? ( +
+ setSaveCount((c) => c + 1)} + /> + +
+ ) : ( +

+ {notes.length === 0 + ? "Create a note to get started" + : "Select a note from the sidebar"} +

+ )} +
+ + )} + {view === "graph" && ( + + )} + {view === "chat" && } +
+
+ ); +} + +function StatusBadge({ + health, + error, +}: { + health: HealthResponse | null; + error: string | null; +}) { + if (error) { + return ( + + Disconnected + + ); + } + if (!health) { + return ( + + Connecting... + + ); + } + const ok = health.engine === "connected"; + return ( + + {ok ? "Connected" : "Engine error"} + + ); +} + +const styles: Record = { + container: { + minHeight: "100vh", + display: "flex", + flexDirection: "column", + }, + header: { + display: "flex", + alignItems: "center", + justifyContent: "space-between", + padding: "10px 24px", + borderBottom: "1px solid rgba(255,255,255,0.06)", + background: "#11111b", + flexShrink: 0, + gap: 16, + }, + title: { + fontSize: 16, + fontWeight: 600, + letterSpacing: -0.3, + }, + tabs: { + display: "flex", + gap: 2, + background: "rgba(255,255,255,0.04)", + borderRadius: 8, + padding: 2, + }, + tab: { + background: "none", + border: "none", + color: "#6c7086", + fontSize: 12, + fontWeight: 500, + padding: "6px 16px", + borderRadius: 6, + cursor: "pointer", + transition: "all 0.15s", + }, + tabActive: { + background: "rgba(137,180,250,0.12)", + color: "#89b4fa", + }, + badge: { + fontSize: 11, + fontWeight: 600, + padding: "4px 12px", + borderRadius: 6, + letterSpacing: 0.3, + flexShrink: 0, + }, + body: { + flex: 1, + display: "flex", + overflow: "hidden", + height: "calc(100vh - 49px)", + }, + main: { + flex: 1, + display: "flex", + alignItems: "center", + justifyContent: "center", + background: "#1e1e2e", + }, + editorContainer: { + display: "flex", + flexDirection: "column" as const, + flex: 1, + height: "100%", + }, + placeholder: { + color: "#6c7086", + fontSize: 14, + }, +}; diff --git a/demos/reasoning-notebook/frontend/src/api.ts b/demos/reasoning-notebook/frontend/src/api.ts new file mode 100644 index 00000000..df867a52 --- /dev/null +++ b/demos/reasoning-notebook/frontend/src/api.ts @@ -0,0 +1,239 @@ +import type { Note } from "./types"; + +const BASE = "/api"; + +export interface HealthResponse { + status: string; + engine: string; + kg: string; +} + +export async function fetchHealth(): Promise { + const res = await fetch(`${BASE}/health`); + if (!res.ok) throw new Error(`Health check failed: ${res.status}`); + return res.json(); +} + +export async function fetchNotes(): Promise { + const res = await fetch(`${BASE}/notes`); + if (!res.ok) throw new Error(`Failed to fetch notes: ${res.status}`); + return res.json(); +} + +export async function fetchNote(id: string): Promise { + const res = await fetch(`${BASE}/notes/${id}`); + if (!res.ok) throw new Error(`Failed to fetch note: ${res.status}`); + return res.json(); +} + +export async function createNote(title: string, content = ""): Promise { + const res = await fetch(`${BASE}/notes`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ title, content }), + }); + if (!res.ok) throw new Error(`Failed to create note: ${res.status}`); + return res.json(); +} + +export async function updateNote( + id: string, + fields: { title?: string; content?: string } +): Promise { + const res = await fetch(`${BASE}/notes/${id}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(fields), + }); + if (!res.ok) throw new Error(`Failed to update note: ${res.status}`); + return res.json(); +} + +export async function deleteNote(id: string): Promise { + const res = await fetch(`${BASE}/notes/${id}`, { method: "DELETE" }); + if (!res.ok) throw new Error(`Failed to delete note: ${res.status}`); +} + +export interface ExtractionResult { + entities: number; + relationships: number; + error?: string; +} + +export async function extractNote(id: string): Promise { + const res = await fetch(`${BASE}/notes/${id}/extract`, { method: "POST" }); + if (!res.ok) throw new Error(`Extraction failed: ${res.status}`); + return res.json(); +} + +export interface NoteEntities { + entities: Array<{ + id: string; + name: string; + kind: string; + description: string; + source_note_id: string; + }>; + relationships: Array<{ + id: string; + subject: string; + predicate: string; + object: string; + source_note_id: string; + }>; +} + +export async function fetchNoteEntities(id: string): Promise { + const res = await fetch(`${BASE}/notes/${id}/entities`); + if (!res.ok) throw new Error(`Failed to fetch entities: ${res.status}`); + return res.json(); +} + +export interface GraphData { + nodes: Array<{ + id: string; + name: string; + kind: string; + description: string; + source_note_id: string; + }>; + edges: Array<{ + id: string; + subject: string; + predicate: string; + object: string; + source_note_id: string; + derived?: boolean; + }>; +} + +export interface ImageSceneData { + image_id: string; + note_id: string; + scene: string; + objects: string; + people: string; + emotion: string; + event_type: string; + aesthetic: string; + caption_seed: string; + cultural_context: string; + visible_text: string; +} + +export async function fetchImageScenes(noteId: string): Promise { + const res = await fetch(`${BASE}/notes/${noteId}/scenes`); + if (!res.ok) return []; + return res.json(); +} + +export async function fetchGraph(): Promise { + const res = await fetch(`${BASE}/graph`); + if (!res.ok) throw new Error(`Failed to fetch graph: ${res.status}`); + return res.json(); +} + +export interface ConsolidationResult { + status: string; + predicate_merges?: Array<{ variants: string[]; canonical: string }>; + entity_merges?: Array<{ variants: string[]; canonical: string }>; + predicates_renamed?: number; + entities_renamed?: number; +} + +export interface ResolutionResult { + status: string; + merges?: Array<{ canonical: string; variant: string; similarity: number }>; + entities_renamed?: number; +} + +export async function resolveEntities(): Promise { + const res = await fetch(`${BASE}/ontology/resolve`, { method: "POST" }); + if (!res.ok) throw new Error(`Resolution failed: ${res.status}`); + return res.json(); +} + +export async function consolidateOntology(): Promise { + const res = await fetch(`${BASE}/ontology/consolidate`, { method: "POST" }); + if (!res.ok) throw new Error(`Consolidation failed: ${res.status}`); + return res.json(); +} + +export interface ChatMessage { + role: "user" | "assistant"; + content: string; +} + +// ── Images ── + +export interface ImageUploadResult { + image_id: string; + filename: string; + url: string; + description: string; + entities: number; + relationships: number; +} + +export async function uploadImage( + noteId: string, + file: File +): Promise { + const form = new FormData(); + form.append("file", file); + const res = await fetch(`${BASE}/notes/${noteId}/images`, { + method: "POST", + body: form, + }); + if (!res.ok) throw new Error(`Image upload failed: ${res.status}`); + return res.json(); +} + +// ── Provenance ── + +export interface ProofNode { + kind: string; + conclusion: { pred: string; args: string[] }; + children: string[]; + source: string | null; + rule_id: string | null; + bindings: Record | null; +} + +export interface ProofTreeData { + roots: string[]; + nodes: Record; + query: string | null; +} + +export interface WhyResponse { + columns: string[]; + rows: string[][]; + proof_trees: ProofTreeData[]; +} + +export async function fetchWhy(query: string): Promise { + const res = await fetch(`${BASE}/why`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ query }), + }); + if (!res.ok) throw new Error(`Why query failed: ${res.status}`); + return res.json(); +} + +// ── Chat ── + +export async function sendChat( + message: string, + history: ChatMessage[] +): Promise { + const res = await fetch(`${BASE}/chat`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ message, history }), + }); + if (!res.ok) throw new Error(`Chat failed: ${res.status}`); + const data = await res.json(); + return data.reply; +} diff --git a/demos/reasoning-notebook/frontend/src/components/ChatPanel.tsx b/demos/reasoning-notebook/frontend/src/components/ChatPanel.tsx new file mode 100644 index 00000000..48ec8f92 --- /dev/null +++ b/demos/reasoning-notebook/frontend/src/components/ChatPanel.tsx @@ -0,0 +1,213 @@ +import { useRef, useState } from "react"; +import { sendChat, type ChatMessage } from "../api"; + +export function ChatPanel() { + const [messages, setMessages] = useState([]); + const [input, setInput] = useState(""); + const [loading, setLoading] = useState(false); + const bottomRef = useRef(null); + + const handleSend = async () => { + const text = input.trim(); + if (!text || loading) return; + + const userMsg: ChatMessage = { role: "user", content: text }; + const updated = [...messages, userMsg]; + setMessages(updated); + setInput(""); + setLoading(true); + + try { + const reply = await sendChat(text, updated); + setMessages((prev) => [...prev, { role: "assistant", content: reply }]); + } catch { + setMessages((prev) => [ + ...prev, + { role: "assistant", content: "Failed to get a response. Is the LLM running?" }, + ]); + } finally { + setLoading(false); + setTimeout(() => bottomRef.current?.scrollIntoView({ behavior: "smooth" }), 50); + } + }; + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + handleSend(); + } + }; + + return ( +
+
+ Chat + Ask questions across your notes +
+
+ {messages.length === 0 && ( +
+

Ask anything about your notes

+

+ Try: "Who works at Acme Corp?" or "How are Alice and Bob connected?" +

+
+ )} + {messages.map((msg, i) => ( +
+
+ {msg.role === "user" ? "You" : "Assistant"} +
+
{msg.content}
+
+ ))} + {loading && ( +
+
Assistant
+
Thinking...
+
+ )} +
+
+
+