Why DOM Selectors Keep Breaking Your Automation (And How AI Solves It)

// Your test, yesterday
await page.click('#login-button');
// ✓ Works perfectly

// Your test, today (after minor UI update)
await page.click('#login-button');
// ✗ Error: Element not found
// ✗ Test suite: 47 tests broken
// ✗ Your afternoon: Wasted fixing selectors

// CSS Selector
await page.click('.submit-btn');

// XPath
await page.click('//button[@class="submit-btn"]');

// ID
await page.click('#submit-button');

<!-- Production HTML -->
<button id="login" class="btn-primary" data-testid="login-btn">
  Sign In
</button>

// Your test
await page.click('#login');  // ✓ Works

<!-- Updated HTML -->
<button class="auth-button primary" data-action="authenticate">
  <span class="icon">🔐</span>
  <span class="text">Sign In</span>
</button>

// Your test
await page.click('#login');  // ✗ Element not found

Total test suites: 100
Avg tests per suite: 500
Total tests: 50,000

Selector-based automation results:
- Tests broken by UI changes: 35,000 (70%)
- Avg time to fix one test: 10 minutes
- Total repair time: 5,833 hours
- Cost at $100/hour: $583,300

Per company cost: $5,833/6 months = ~$12K/year

await page.click('#submit-btn-2023');

await page.click('.btn-primary.submit');

// XPath: "Click the 3rd button in the 2nd div"
await page.click('//div[2]/button[3]');

await page.click('text=Submit');

await page.click('[data-testid="submit-button"]');

<button data-testid="login-button">Sign In</button>

await page.click('[data-testid="login-button"]');

await page.click('//button[contains(text(), "Sign In")]');

// Multiple fallbacks
await page.click('#login, .login-btn, [data-testid="login"], button:contains("Login")');

// Find element by visual appearance
await page.clickByImage('login-button-screenshot.png');

// What the automation knows
const selector = '#login-button';

// What it does
const element = document.querySelector(selector);
if (element) {
  element.click();
} else {
  throw new Error('Element not found');
}

// If selector is wrong: Failed
// If selector is right: Success
// Resilience: 0%

// What the AI knows
const intent = {
  action: 'Click the login button',
  context: 'On a login page',
  purpose: 'Authenticate user'
};

// What it does
1. Analyze page structure
2. Find ALL buttons
3. Identify which one is most likely the login button:
   - Has login-related text?
   - In a login form?
   - Looks like a primary action?
   - Semantic role matches?
4. Click the best match

// If structure changes: Adapts
// If text changes: Still finds it
// If classes change: Doesn't care
// Resilience: 95%

class AIElementDetector {
  async findElement(intent: Intent, page: Page): Promise<Element> {
    // 1. Get page context
    const context = await this.analyzePageContext(page);

    // 2. Get all potential candidates
    const candidates = await this.getCandidateElements(page, intent.elementType);

    // 3. Score each candidate
    const scored = await Promise.all(
      candidates.map(async (candidate) => ({
        element: candidate,
        score: await this.scoreElement(candidate, intent, context)
      }))
    );

    // 4. Select best match
    const bestMatch = scored.reduce((best, current) =>
      current.score > best.score ? current : best
    );

    // 5. Validate selection
    if (bestMatch.score < 0.7) {
      throw new Error(`Low confidence: ${bestMatch.score}`);
    }

    return bestMatch.element;
  }

  private async scoreElement(
    element: Element,
    intent: Intent,
    context: Context
  ): Promise<number> {
    const scores = await Promise.all([
      this.scoreByText(element, intent),
      this.scoreByRole(element, intent),
      this.scoreByPosition(element, context),
      this.scoreByVisibility(element),
      this.scoreBySemanticMeaning(element, intent),
      this.scoreByVisionLLM(element, intent) // Uses vision model
    ]);

    // Weighted average
    return scores.reduce((sum, score, i) => {
      const weights = [0.25, 0.20, 0.15, 0.10, 0.15, 0.15];
      return sum + score * weights[i];
    }, 0);
  }

  private async scoreBySemanticMeaning(
    element: Element,
    intent: Intent
  ): Promise<number> {
    // Use LLM to understand semantic relationship
    const prompt = `
Intent: ${intent.description}
Element: ${await this.describeElement(element)}

Question: On a scale of 0-1, how likely is this element the one described by the intent?

Return only a number between 0 and 1.
    `;

    const score = await this.llm.generate(prompt);
    return parseFloat(score);
  }

  private async scoreByVisionLLM(
    element: Element,
    intent: Intent
  ): Promise<number> {
    // Capture screenshot
    const screenshot = await element.screenshot();

    // Ask vision model
    const analysis = await this.visionLLM.analyze(screenshot, {
      prompt: `Is this a ${intent.description}? Return confidence 0-1.`
    });

    return analysis.confidence;
  }
}

<div class="auth-container">
  <button class="btn-a">Cancel</button>
  <button class="btn-b primary">Sign In</button>
  <a href="/register">Create Account</a>
</div>

Candidates:
1. <button class="btn-a">Cancel</button>
   - Text score: 0.1 (not login-related)
   - Role score: 1.0 (is a button)
   - Position: 0.5 (first in form)
   - Visibility: 1.0 (visible)
   - Semantic: 0.2 (cancel action, not login)
   - Vision: 0.3 (secondary button style)
   → Total: 0.38

2. <button class="btn-b primary">Sign In</button>
   - Text score: 0.9 ("Sign In" = login)
   - Role score: 1.0 (is a button)
   - Position: 0.9 (primary position)
   - Visibility: 1.0 (visible)
   - Semantic: 0.95 (login action)
   - Vision: 0.95 (primary button style)
   → Total: 0.92 ✓ SELECTED

3. <a href="/register">Create Account</a>
   - Text score: 0.3 (related to auth)
   - Role score: 0.5 (is a link, not button)
   - Position: 0.3 (secondary action)
   - Visibility: 1.0 (visible)
   - Semantic: 0.4 (registration, not login)
   - Vision: 0.4 (tertiary action style)
   → Total: 0.42

Selected: <button class="btn-b primary"> with 92% confidence

await page.click('#login-btn');
// Knows: Element has id="login-btn"
// Doesn't know: Why this element matters
// Breaks when: ID changes

await aiAgent.click('the login button');
// Knows: Element is for authentication
// Knows: Should be prominent, button-like, login-related
// Survives: Class changes, ID changes, structure changes
// Only breaks if: Page no longer has login functionality

// Brittle selector-based
await page.click('#username');
await page.type('#username', '[email protected]');
await page.click('#password');
await page.type('#password', 'password123');
await page.click('button[type="submit"]');
await page.waitForSelector('.dashboard');

// Resilient intent-based
await aiAgent.do({
  action: 'Log in',
  inputs: {
    username: '[email protected]',
    password: 'password123'
  },
  expectedOutcome: 'See dashboard'
});

class IntentNavigator {
  async performAction(intent: ActionIntent): Promise<Result> {
    // 1. Understand intent
    const plan = await this.planAction(intent);

    // 2. Find elements by intent, not selectors
    const elements = await this.findElementsByIntent(plan);

    // 3. Execute action
    const result = await this.executeWithValidation(elements, plan);

    // 4. Verify outcome
    const validated = await this.validateOutcome(intent.expectedOutcome);

    return {
      success: validated.success,
      confidence: validated.confidence
    };
  }

  private async findElementsByIntent(
    plan: ActionPlan
  ): Promise<Map<string, Element>> {
    const elements = new Map();

    for (const [role, intent] of Object.entries(plan.elements)) {
      const element = await this.aiDetector.findElement({
        description: intent.description,
        role: intent.role,
        context: plan.context
      });

      elements.set(role, element);
    }

    return elements;
  }

  private async planAction(intent: ActionIntent): Promise<ActionPlan> {
    const prompt = `
Action intent: ${intent.action}
Required inputs: ${JSON.stringify(intent.inputs)}
Expected outcome: ${intent.expectedOutcome}

Create a plan:
1. What elements are needed?
2. What should we look for in each element?
3. What's the sequence of actions?
4. How do we validate success?

Return JSON plan.
    `;

    return await this.llm.generate(prompt);
  }
}

// Day 1: Works
test('User can log in', async () => {
  await page.click('#login');
  await page.type('#email', '[email protected]');
  await page.type('#pass', 'password');
  await page.click('.submit');
});

// Day 30: After UI update
test('User can log in', async () => {
  await page.click('#login');  // ✗ Not found
  // Test fails
  // Manual fix required
});

// Day 1: Works
test('User can log in', async () => {
  await aiAgent.do({
    action: 'Navigate to login and authenticate',
    inputs: { email: '[email protected]', password: 'password' },
    validate: 'User is logged in'
  });
});

// Day 30: After UI update
test('User can log in', async () => {
  await aiAgent.do({
    action: 'Navigate to login and authenticate',
    inputs: { email: '[email protected]', password: 'password' },
    validate: 'User is logged in'
  });
  // ✓ Still works
  // AI found new element locations automatically
});

// Day 60: After complete redesign
test('User can log in', async () => {
  await aiAgent.do({
    action: 'Navigate to login and authenticate',
    inputs: { email: '[email protected]', password: 'password' },
    validate: 'User is logged in'
  });
  // ✓ Still works
  // No code changes needed
});

class SelfHealingAgent {
  private knowledgeBase = new Map<string, ElementPattern>();

  async findElement(intent: Intent): Promise<Element> {
    // Try cached successful patterns first
    const cachedPattern = this.knowledgeBase.get(intent.id);
    if (cachedPattern) {
      try {
        const element = await this.tryPattern(cachedPattern);
        if (element) {
          this.recordSuccess(intent, cachedPattern);
          return element;
        }
      } catch {
        // Pattern no longer works, try AI
      }
    }

    // Use AI to find element
    const element = await this.aiDetector.findElement(intent);

    // Learn new pattern
    const newPattern = await this.extractPattern(element, intent);
    this.knowledgeBase.set(intent.id, newPattern);

    return element;
  }

  private async extractPattern(
    element: Element,
    intent: Intent
  ): Promise<ElementPattern> {
    // Extract generalizable patterns
    return {
      intent: intent.id,
      characteristics: {
        role: await element.getAttribute('role'),
        text: await element.textContent(),
        position: await this.getRelativePosition(element),
        visualFeatures: await this.extractVisualFeatures(element)
      },
      confidence: 1.0,
      lastSuccessful: Date.now()
    };
  }

  private recordSuccess(intent: Intent, pattern: ElementPattern) {
    // Reinforce successful pattern
    pattern.confidence = Math.min(1.0, pattern.confidence + 0.1);
    pattern.lastSuccessful = Date.now();
  }
}

Tests broken by changes:
- Week 2: 12/50 (24%)
- Week 4: 23/50 (46%)
- Week 8: 35/50 (70%)
- Week 12: 41/50 (82%)
- Week 24: 48/50 (96%)

Maintenance time:
- Total hours fixing selectors: 320 hours
- Avg time per fix: 25 minutes
- Tests eventually abandoned: 15/50 (30%)

Tests broken by changes:
- Week 2: 0/50 (0%)
- Week 4: 1/50 (2%)
- Week 8: 1/50 (2%)
- Week 12: 3/50 (6%)
- Week 24: 4/50 (8%)

Maintenance time:
- Total hours fixing: 8 hours
- Avg time per fix: 5 minutes (only for fundamental changes)
- Tests abandoned: 0/50 (0%)

4 sites that broke:
1. Complete functionality removal (login removed, switched to SSO)
2. Paywall added (content no longer accessible)
3. Site shutdown (404)
4. Fundamental redesign + login now requires 2FA (new workflow)

Valid failures: 4/4 (100%)
False failures: 0/4 (0%)

48 sites that broke:
1. CSS class name changes: 23
2. ID changes: 18
3. Structure changes: 15
4. Data attribute removal: 12
5. Text changes (i18n): 8
6. Dynamic IDs introduced: 6
7. Framework migration: 4
... many tests broken by multiple issues

Valid failures: 4 (functionality changed)
False failures: 44 (still works, selectors outdated)

# Find all selectors in your tests
grep -r "querySelector\|click\|type" tests/

# Common patterns:
# page.click('#id')
# page.click('.class')
# page.click('[data-testid="..."]')

High risk (will break soon):
- ID selectors (#...)
- Class selectors (...)
- Positional selectors (nth-child)

Medium risk:
- Data attributes ([data-testid])
- Text selectors (text=...)

Low risk (but still brittle):
- Semantic roles ([role="button"])

await page.click('#login-button');

await aiAgent.click('the login button');

await page.type('#email-input', '[email protected]');

await aiAgent.type('email field', '[email protected]');

await page.click('button.submit-form');

await aiAgent.click('the button that submits the form');

# Install
npm install @onpiste/agent

# Or use the Chrome extension
# https://chromewebstore.google.com/detail/onpiste/hmojfgaobpbggbfcaijjghjimbbjfnei

import { Agent } from '@onpiste/agent';

const agent = new Agent({
  llm: {
    provider: 'openai',
    model: 'gpt-4o'
  }
});

// Intent-based automation
await agent.click('login button');
await agent.fill('email', '[email protected]');
await agent.fill('password', 'secret123');
await agent.click('submit');

// Validates automatically
await agent.validate('user is logged in');

class IntentBasedAutomation {
  constructor(
    private llm: LLMProvider,
    private page: Page
  ) {}

  async click(description: string): Promise<void> {
    // 1. Find element by description
    const element = await this.findByIntent({
      action: 'click',
      target: description
    });

    // 2. Click
    await element.click();

    // 3. Validate action worked
    await this.validateClick(element);
  }

  private async findByIntent(intent: Intent): Promise<Element> {
    // Get page structure
    const structure = await this.getPageStructure();

    // Ask LLM to identify element
    const prompt = `
Find the element matching this description: "${intent.target}"

Page structure:
${structure}

Return the best matching element's index.
    `;

    const response = await this.llm.generate(prompt);
    const elements = await this.page.$$('button, a, input');

    return elements[response.index];
  }
}

// Keep old tests running
describe('Login (traditional)', () => {
  test('user can log in', async () => {
    await page.click('#login-btn');
    // ...
  });
});

// Add new AI tests alongside
describe('Login (AI)', () => {
  test('user can log in', async () => {
    await aiAgent.click('login button');
    // ...
  });
});

// Compare results for 2 weeks
// Migrate when confident

// Track resilience metrics
const metrics = {
  testsRun: 0,
  selectorBasedFailures: 0,
  intentBasedFailures: 0,
  falseFailures: 0,
  maintenanceTime: 0
};

// Alert on pattern
if (metrics.selectorBasedFailures > 10) {
  console.log('High selector failure rate - time to migrate');
}

Test suite: 1,000 tests
Selector-based: 100%

Annual costs:
- Initial development: 500 hours @ $100/hr = $50,000
- Maintenance (selector fixes): 800 hours @ $100/hr = $80,000
- False failure investigation: 400 hours @ $100/hr = $40,000
- Test abandonment (rewrite): 200 hours @ $100/hr = $20,000

Total annual cost: $190,000

Effective cost per test: $190/year

Test suite: 1,000 tests
Intent-based: 100%

Annual costs:
- Initial development: 300 hours @ $100/hr = $30,000
- AI API costs: $5,000/year
- Maintenance (real failures only): 80 hours @ $100/hr = $8,000
- False failure investigation: 0 hours = $0
- Test abandonment: 0 hours = $0

Total annual cost: $43,000

Effective cost per test: $43/year

Savings: $147,000/year (77% reduction)