Content Moderation

This example shows how to use AI AutoEvals for content moderation, automatically flagging low-scoring AI-generated content for human review.

Overview

Automatic Evaluation: Evaluate all AI-generated content
Score Thresholds: Define thresholds for different moderation actions
Workflow Integration: Integrate with Drupal’s content moderation
Notifications: Alert moderators when content needs review

Step 1: Configure Evaluation

Create a strict evaluation set for content moderation:

Navigate to /admin/content/ai-autoevals/sets
Click “Add Evaluation Set”
Configure:
- Label: “Content Moderation”
- Fact Extraction Method: “AI Generated”
- Custom Knowledge: Add your content guidelines
- Choice Scores:
  - A (Exact Match): 1.0
  - B (Superset): 0.8
  - C (Subset): 0.4
  - D (Disagreement): 0.0
Save

Optional: Configure Exclusion Keywords

To skip moderation of test or internal content, configure exclusion keywords:

Per-Set Exclusions:

Query Exclusion Keywords (skip moderation when queries contain these):
```
test
debug
staging
internal
```
Response Exclusion Keywords (skip moderation when responses contain these):
```
mock
placeholder
TBD
N/A
error
```

Global Exclusions (module settings):

Navigate to /admin/config/ai/autoevals and configure:

Global Query Exclusion Keywords: Skip ALL evaluations when queries contain these
Global Response Exclusion Keywords: Skip ALL evaluations when responses contain these

This ensures test content and placeholder data don’t trigger moderation workflows.

Step 2: Create Event Subscriber

Create an event subscriber to handle low-scoring content:

<?php

namespace Drupal\content_moderation_ai\EventSubscriber;

use Drupal\ai_autoevals\Event\PostEvaluationEvent;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Drupal\Core\Logger\LoggerChannelFactoryInterface;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;

/**
 * Event subscriber for content moderation.
 */
class ContentModerationSubscriber implements EventSubscriberInterface {

  /**
   * Score thresholds for moderation actions.
   */
  const THRESHOLD_APPROVE = 0.8;
  const THRESHOLD_REVIEW = 0.5;
  const THRESHOLD_REJECT = 0.3;

  public function __construct(
    protected EntityTypeManagerInterface $entityTypeManager,
    protected LoggerChannelFactoryInterface $loggerFactory
  ) {}

  /**
   * {@inheritdoc}
   */
  public static function getSubscribedEvents(): array {
    return [
      PostEvaluationEvent::EVENT_NAME => ['moderateContent', 0],
    ];
  }

  /**
   * Moderates content based on evaluation score.
   */
  public function moderateContent(PostEvaluationEvent $event): void {
    $evaluation = $event->getEvaluationResult();
    $score = $event->getScore();

    // Only moderate if score is available
    if ($score === null) {
      return;
    }

    $metadata = $evaluation->getMetadata();

    // Get related content node if available
    if (empty($metadata['node_id'])) {
      return;
    }

    /** @var \Drupal\node\NodeInterface $node */
    $node = $this->entityTypeManager->getStorage('node')->load($metadata['node_id']);

    if (!$node || !$node->hasField('moderation_state')) {
      return;
    }

    // Moderate based on score
    if ($score >= self::THRESHOLD_APPROVE) {
      $this->approveContent($node, $evaluation);
    }
    elseif ($score >= self::THRESHOLD_REVIEW) {
      $this->flagForReview($node, $evaluation, 'medium');
    }
    elseif ($score >= self::THRESHOLD_REJECT) {
      $this->flagForReview($node, $evaluation, 'high');
    }
    else {
      $this->rejectContent($node, $evaluation);
    }
  }

  /**
   * Approves content automatically.
   */
  protected function approveContent($node, $evaluation): void {
    $node->set('moderation_state', 'published');
    $node->save();

    $this->loggerFactory->get('content_moderation_ai')->info(
      'Content @nid automatically approved (score: @score)',
      [
        '@nid' => $node->id(),
        '@score' => $evaluation->getScore(),
      ]
    );
  }

  /**
   * Flags content for review.
   */
  protected function flagForReview($node, $evaluation, $priority): void {
    $node->set('moderation_state', 'needs_review');
    $node->set('field_review_priority', $priority);
    $node->set('field_evaluation_score', $evaluation->getScore());
    $node->set('field_evaluation_id', $evaluation->id());
    $node->save();

    $this->loggerFactory->get('content_moderation_ai')->warning(
      'Content @nid flagged for @priority review (score: @score)',
      [
        '@nid' => $node->id(),
        '@priority' => $priority,
        '@score' => $evaluation->getScore(),
      ]
    );

    // Send notification to moderators
    $this->sendModeratorNotification($node, $evaluation, $priority);
  }

  /**
   * Rejects low-quality content.
   */
  protected function rejectContent($node, $evaluation): void {
    $node->set('moderation_state', 'draft');
    $node->set('field_rejection_reason', 'Low quality AI content detected');
    $node->set('field_evaluation_score', $evaluation->getScore());
    $node->set('field_evaluation_id', $evaluation->id());
    $node->save();

    $this->loggerFactory->get('content_moderation_ai')->error(
      'Content @nid rejected (score: @score)',
      [
        '@nid' => $node->id(),
        '@score' => $evaluation->getScore(),
      ]
    );

    // Send urgent notification
    $this->sendUrgentNotification($node, $evaluation);
  }

  /**
   * Sends notification to moderators.
   */
  protected function sendModeratorNotification($node, $evaluation, $priority): void {
    $mail_manager = \Drupal::service('plugin.manager.mail');
    $to = $this->getModeratorEmails($priority);

    $params = [
      'node' => $node,
      'evaluation' => $evaluation,
      'priority' => $priority,
      'url' => $node->toUrl('canonical', ['absolute' => TRUE])->toString(),
    ];

    $mail_manager->mail('content_moderation_ai', 'review_notification', $to, 'en', $params);
  }

  /**
   * Sends urgent notification.
   */
  protected function sendUrgentNotification($node, $evaluation): void {
    $mail_manager = \Drupal::service('plugin.manager.mail');
    $to = $this->getModeratorEmails('urgent');

    $params = [
      'node' => $node,
      'evaluation' => $evaluation,
      'url' => $node->toUrl('canonical', ['absolute' => TRUE])->toString(),
    ];

    $mail_manager->mail('content_moderation_ai', 'urgent_notification', $to, 'en', $params);
  }

  /**
   * Gets moderator email addresses.
   */
  protected function getModeratorEmails($priority): string {
    $config = \Drupal::config('content_moderation_ai.settings');

    switch ($priority) {
      case 'urgent':
        return $config->get('urgent_email') ?? 'admin@example.com';

      case 'high':
        return $config->get('high_priority_email') ?? 'moderators@example.com';

      case 'medium':
      default:
        return $config->get('review_email') ?? 'moderators@example.com';
    }
  }

}

services:
  content_moderation_ai.subscriber:
    class: Drupal\content_moderation_ai\EventSubscriber\ContentModerationSubscriber
    arguments:
      - '@entity_type.manager'
      - '@logger.factory'
    tags:
      - { name: 'event_subscriber' }

Step 3: Track AI Content

Ensure AI-generated content is tracked with the node ID:

<?php

/**
 * Example: Create content with AI and track it.
 */
function create_ai_content($question, $answer) {
  // Create node
  $node = Node::create([
    'type' => 'article',
    'title' => substr($question, 0, 50),
    'body' => $answer,
    'moderation_state' => 'draft',
  ]);
  $node->save();

  // Make AI request with tracking
  $provider = \Drupal::service('ai.provider')->createInstance('amazeeio');
  $response = $provider->chat($question, 'gpt-4', [
    'ai_autoevals:track' => TRUE,
    'node_id' => $node->id(),
  ]);

  return $node;
}

Step 4: Create Moderation Dashboard

Create a custom dashboard for moderators:

<?php

namespace Drupal\content_moderation_ai\Controller;

use Drupal\Core\Controller\ControllerBase;
use Drupal\Core\Entity\EntityTypeManagerInterface;

/**
 * Controller for moderation dashboard.
 */
class ModerationDashboardController extends ControllerBase {

  public function __construct(
    protected EntityTypeManagerInterface $entityTypeManager
  ) {}

  /**
   * Displays moderation dashboard.
   */
  public function dashboard(): array {
    // Get nodes flagged for review
    $nodes = $this->entityTypeManager->getStorage('node')
      ->getQuery()
      ->accessCheck(FALSE)
      ->condition('moderation_state', 'needs_review')
      ->sort('field_review_priority', 'DESC')
      ->sort('changed', 'DESC')
      ->range(0, 50)
      ->execute();

    $nodes = $this->entityTypeManager->getStorage('node')->loadMultiple($nodes);

    // Get high priority items
    $high_priority = array_filter($nodes, function($node) {
      return $node->get('field_review_priority')->value === 'high';
    });

    return [
      '#theme' => 'content_moderation_dashboard',
      '#high_priority' => $high_priority,
      '#all_items' => $nodes,
    ];
  }

}

Create the template templates/content-moderation-dashboard.html.twig:

{#
/**
 * @file
 * Content moderation dashboard template.
 */
#}
<div class="moderation-dashboard">
  <h2>{{ 'Content Moderation Dashboard'|t }}</h2>

  <div class="priority-sections">
    <div class="high-priority">
      <h3>{{ 'High Priority'|t }}</h3>
      <ul>
        {% for node in high_priority %}
          <li>
            <a href="{{ path('entity.node.canonical', {'node': node.id}) }}">
              {{ node.label }}
            </a>
            <span class="score">{{ 'Score: @score'|t({'@score': node.field_evaluation_score.value}) }}</span>
          </li>
        {% endfor %}
      </ul>
    </div>

    <div class="all-items">
      <h3>{{ 'All Items'|t }}</h3>
      <table>
        <thead>
          <tr>
            <th>{{ 'Title'|t }}</th>
            <th>{{ 'Score'|t }}</th>
            <th>{{ 'Priority'|t }}</th>
            <th>{{ 'Actions'|t }}</th>
          </tr>
        </thead>
        <tbody>
          {% for node in all_items %}
            <tr>
              <td>
                <a href="{{ path('entity.node.canonical', {'node': node.id}) }}">
                  {{ node.label }}
                </a>
              </td>
              <td>{{ node.field_evaluation_score.value|number_format(2) }}</td>
              <td>{{ node.field_review_priority.value }}</td>
              <td>
                <a href="{{ path('entity.node.edit_form', {'node': node.id}) }}">
                  {{ 'Edit'|t }}
                </a>
              </td>
            </tr>
          {% endfor %}
        </tbody>
      </table>
    </div>
  </div>
</div>

Step 5: Configure Moderation Thresholds

Create configuration form for thresholds:

<?php

namespace Drupal\content_moderation_ai\Form;

use Drupal\Core\Form\ConfigFormBase;
use Drupal\Core\Form\FormStateInterface;

/**
 * Configuration form for content moderation settings.
 */
class ModerationSettingsForm extends ConfigFormBase {

  /**
   * {@inheritdoc}
   */
  protected function getEditableConfigNames(): array {
    return ['content_moderation_ai.settings'];
  }

  /**
   * {@inheritdoc}
   */
  public function getFormId(): string {
    return 'content_moderation_ai_settings';
  }

  /**
   * {@inheritdoc}
   */
  public function buildForm(array $form, FormStateInterface $form_state): array {
    $config = $this->config('content_moderation_ai.settings');

    $form['thresholds'] = [
      '#type' => 'details',
      '#title' => $this->t('Score Thresholds'),
      '#open' => TRUE,
    ];

    $form['thresholds']['approve_threshold'] = [
      '#type' => 'number',
      '#title' => $this->t('Approve Threshold'),
      '#description' => $this->t('Content with scores at or above this value will be automatically approved.'),
      '#default_value' => $config->get('approve_threshold') ?? 0.8,
      '#min' => 0,
      '#max' => 1,
      '#step' => 0.05,
    ];

    $form['thresholds']['review_threshold'] = [
      '#type' => 'number',
      '#title' => $this->t('Review Threshold'),
      '#description' => $this->t('Content with scores between this value and the approve threshold will be flagged for review.'),
      '#default_value' => $config->get('review_threshold') ?? 0.5,
      '#min' => 0,
      '#max' => 1,
      '#step' => 0.05,
    ];

    $form['thresholds']['reject_threshold'] = [
      '#type' => 'number',
      '#title' => $this->t('Reject Threshold'),
      '#description' => $this->t('Content with scores below this value will be rejected.'),
      '#default_value' => $config->get('reject_threshold') ?? 0.3,
      '#min' => 0,
      '#max' => 1,
      '#step' => 0.05,
    ];

    $form['notifications'] = [
      '#type' => 'details',
      '#title' => $this->t('Notification Emails'),
      '#open' => TRUE,
    ];

    $form['notifications']['urgent_email'] = [
      '#type' => 'email',
      '#title' => $this->t('Urgent Notification Email'),
      '#description' => $this->t('Email address for urgent notifications (rejected content).'),
      '#default_value' => $config->get('urgent_email') ?? '',
    ];

    $form['notifications']['high_priority_email'] = [
      '#type' => 'email',
      '#title' => $this->t('High Priority Notification Email'),
      '#description' => $this->t('Email address for high priority review notifications.'),
      '#default_value' => $config->get('high_priority_email') ?? '',
    ];

    $form['notifications']['review_email'] = [
      '#type' => 'email',
      '#title' => $this->t('Review Notification Email'),
      '#description' => $this->t('Email address for standard review notifications.'),
      '#default_value' => $config->get('review_email') ?? '',
    ];

    return parent::buildForm($form, $form_state);
  }

  /**
   * {@inheritdoc}
   */
  public function submitForm(array &$form, FormStateInterface $form_state): void {
    $this->config('content_moderation_ai.settings')
      ->set('approve_threshold', $form_state->getValue('approve_threshold'))
      ->set('review_threshold', $form_state->getValue('review_threshold'))
      ->set('reject_threshold', $form_state->getValue('reject_threshold'))
      ->set('urgent_email', $form_state->getValue('urgent_email'))
      ->set('high_priority_email', $form_state->getValue('high_priority_email'))
      ->set('review_email', $form_state->getValue('review_email'))
      ->save();

    parent::submitForm($form, $form_state);
  }

}

Best Practices

Define Clear Thresholds

Set thresholds based on your quality requirements:
- 0.8+: Automatically approve high-quality content
- 0.5-0.8: Flag for manual review
- Below 0.5: Require human intervention

Provide Context to Moderators

Include evaluation details with flagged content:

$node->set('field_evaluation_details', [
  'score' => $evaluation->getScore(),
  'facts' => $evaluation->getFacts(),
  'analysis' => $evaluation->getAnalysis(),
]);

Implement Escalation

Escalate low-scoring content to senior moderators:

if ($score < 0.3) {
  $node->set('field_reviewer', 'senior_moderator');
}

Track Moderation Decisions

Track human decisions for model improvement:

// When moderator approves/rejects
$node->set('field_moderator_decision', 'approved');
$node->set('field_moderator_notes', $notes);

Next Steps

A/B Testing - Test different moderation strategies
Custom Fact Extractors - Domain-specific evaluation
Event System - Event system guide