User Preference Learning: AI Agents That Adapt to Individual Users Over Time

Why Preference Learning Matters

A customer support agent that asks the same clarifying questions every session frustrates repeat users. A coding assistant that defaults to JavaScript when the user always writes Python wastes time. Preference learning enables agents to remember and adapt to each user's habits, communication style, and stated preferences, creating interactions that get better over time.

The challenge is extracting preferences from natural conversation — users rarely say "I prefer concise responses." Instead, they say "Can you just give me the answer?" and the agent must infer the preference and remember it.

Defining a User Profile

A user profile stores both explicit preferences (stated directly) and implicit preferences (inferred from behavior).

flowchart LR
    INPUT(["User intent"])
    PARSE["Parse plus<br/>classify"]
    PLAN["Plan and tool<br/>selection"]
    AGENT["Agent loop<br/>LLM plus tools"]
    GUARD{"Guardrails<br/>and policy"}
    EXEC["Execute and<br/>verify result"]
    OBS[("Trace and metrics")]
    OUT(["Outcome plus<br/>next action"])
    INPUT --> PARSE --> PLAN --> AGENT --> GUARD
    GUARD -->|Pass| EXEC --> OUT
    GUARD -->|Fail| AGENT
    AGENT --> OBS
    style AGENT fill:#4f46e5,stroke:#4338ca,color:#fff
    style GUARD fill:#f59e0b,stroke:#d97706,color:#1f2937
    style OBS fill:#ede9fe,stroke:#7c3aed,color:#1e1b4b
    style OUT fill:#059669,stroke:#047857,color:#fff

from dataclasses import dataclass, field
from datetime import datetime
from typing import Dict, List, Optional
from enum import Enum

class Confidence(Enum):
    LOW = "low"          # inferred from a single interaction
    MEDIUM = "medium"    # confirmed across multiple interactions
    HIGH = "high"        # explicitly stated by the user

@dataclass
class Preference:
    key: str             # "response_style", "programming_language", etc.
    value: str           # "concise", "python", etc.
    confidence: Confidence
    source: str          # how this was learned
    updated_at: datetime = field(default_factory=datetime.utcnow)
    observation_count: int = 1

@dataclass
class UserProfile:
    user_id: str
    preferences: Dict[str, Preference] = field(default_factory=dict)
    interaction_count: int = 0
    created_at: datetime = field(default_factory=datetime.utcnow)

    def set_preference(self, key: str, value: str, confidence: Confidence, source: str):
        if key in self.preferences:
            existing = self.preferences[key]
            existing.observation_count += 1
            # Upgrade confidence if we see the same preference repeatedly
            if existing.value == value and existing.observation_count >= 3:
                existing.confidence = Confidence.HIGH
            existing.value = value
            existing.updated_at = datetime.utcnow()
        else:
            self.preferences[key] = Preference(
                key=key, value=value, confidence=confidence, source=source
            )

    def get_preference(self, key: str, default: str = None) -> Optional[str]:
        pref = self.preferences.get(key)
        return pref.value if pref else default

    def to_prompt_context(self) -> str:
        """Format preferences for injection into the system prompt."""
        if not self.preferences:
            return ""
        lines = ["Known user preferences:"]
        for pref in self.preferences.values():
            lines.append(
                f"- {pref.key}: {pref.value} "
                f"(confidence: {pref.confidence.value})"
            )
        return "\n".join(lines)

Extracting Preferences from Conversation

Use the LLM itself to detect preferences in user messages. This is more reliable than rule-based extraction because it handles natural language variations.

import openai
import json

client = openai.OpenAI()

def extract_preferences(user_message: str, assistant_response: str) -> List[Dict]:
    """Use LLM to identify implicit and explicit preferences in a message pair."""
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{
            "role": "system",
            "content": (
                "Analyze this user-assistant exchange and extract any user preferences. "
                "Return a JSON array of objects with keys: "
                "'key' (preference category), 'value' (preference value), "
                "'explicit' (boolean, true if directly stated). "
                "Return [] if no preferences detected."
            ),
        }, {
            "role": "user",
            "content": (
                f"User said: {user_message}\n"
                f"Assistant responded: {assistant_response}"
            ),
        }],
        response_format={"type": "json_object"},
        max_tokens=300,
    )

    try:
        result = json.loads(response.choices[0].message.content)
        return result.get("preferences", [])
    except (json.JSONDecodeError, KeyError):
        return []

# Example usage
prefs = extract_preferences(
    user_message="Just give me the command, I don't need the explanation",
    assistant_response="Here is a detailed walkthrough of the process..."
)
# Returns: [{"key": "response_style", "value": "concise_commands_only", "explicit": true}]

Applying Preferences in Agent Responses

Once preferences are stored, inject them into the agent's system prompt so they influence every response.

class PersonalizedAgent:
    def __init__(self, base_system_prompt: str, profile_store):
        self.base_prompt = base_system_prompt
        self.profile_store = profile_store

    async def respond(self, user_id: str, message: str) -> str:
        profile = self.profile_store.load(user_id)
        profile.interaction_count += 1

        # Build personalized system prompt
        pref_context = profile.to_prompt_context()
        system_prompt = self.base_prompt
        if pref_context:
            system_prompt += f"\n\n{pref_context}"

        # Get response from LLM
        response = client.chat.completions.create(
            model="gpt-4o",
            messages=[
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": message},
            ],
        )
        assistant_msg = response.choices[0].message.content

        # Extract and store new preferences (async, non-blocking in production)
        new_prefs = extract_preferences(message, assistant_msg)
        for pref in new_prefs:
            confidence = Confidence.HIGH if pref.get("explicit") else Confidence.LOW
            profile.set_preference(
                key=pref["key"],
                value=pref["value"],
                confidence=confidence,
                source=f"interaction_{profile.interaction_count}",
            )

        self.profile_store.save(profile)
        return assistant_msg

Privacy and Data Handling

Preference learning must respect user privacy. Always follow these principles:

class PrivacyAwareProfileStore:
    """Profile store with privacy controls."""

    SENSITIVE_KEYS = {"health_info", "financial_data", "political_views"}

    def __init__(self, storage_path: str):
        self.storage_path = Path(storage_path)

    def save(self, profile: UserProfile):
        # Filter out sensitive categories
        safe_prefs = {
            k: v for k, v in profile.preferences.items()
            if k not in self.SENSITIVE_KEYS
        }
        profile.preferences = safe_prefs
        data = {
            "user_id": profile.user_id,
            "preferences": {
                k: {"value": v.value, "confidence": v.confidence.value,
                     "observation_count": v.observation_count}
                for k, v in safe_prefs.items()
            },
            "interaction_count": profile.interaction_count,
        }
        path = self.storage_path / f"{profile.user_id}.json"
        path.write_text(json.dumps(data, indent=2))

    def delete_profile(self, user_id: str):
        """Support right-to-be-forgotten requests."""
        path = self.storage_path / f"{user_id}.json"
        if path.exists():
            path.unlink()

FAQ

How do I handle conflicting preferences across sessions?

Track observation count and recency. If a user preferred verbose responses in 8 out of 10 sessions but asked for concise output in the last 2, weight the recent interactions more heavily. You can use exponential decay on the observation count to prioritize recent behavior.

Should preference extraction happen synchronously or asynchronously?

Run it asynchronously after sending the response. The user should not wait for the preference extraction LLM call. Queue the extraction as a background task and update the profile for the next interaction.

What happens when a user says "stop remembering things about me"?

Treat this as an explicit instruction to clear the profile. Delete stored preferences and set a meta-preference like {"data_retention": "none"} that prevents future preference extraction for that user.

---

#Personalization #UserPreferences #AdaptiveAgents #Privacy #AgenticAI #LearnAI #AIEngineering