Performance

This guide covers performance optimization strategies for AI AutoEvals.

Overview

AI AutoEvals uses asynchronous processing and caching to minimize performance impact. However, there are several strategies you can use to optimize performance further.

Queue Processing

Process During Off-Peak Hours

Schedule queue processing for low-traffic times:

# In crontab
0 2 * * * drush queue:run ai_autoevals_evaluation_worker

Or use Drupal’s cron:

/**
 * Implements hook_cron().
 */
function my_module_cron(): void {
  // Only run queue processing during off-peak hours (2-4 AM)
  $hour = (int) date('H');
  if ($hour >= 2 && $hour < 4) {
    $queue = \Drupal::queue('ai_autoevals_evaluation_worker');
    $queue->createQueue();
    $queue->claimItem();
  }
}

Increase Queue Worker Limit

The default time limit is 60 seconds per cron run. Increase if needed:

# In ai_autoevals.services.yml or your module's services.yml
services:
  my_module.evaluation_queue_worker:
    class: Drupal\my_module\Plugin\QueueWorker\MyEvaluationWorker
    arguments: ['@ai_autoevals.evaluation_manager']
    tags:
      - { name: queue_worker, cron: { time: 300 } }  # 5 minutes instead of 1

Batch Queue Processing

Process evaluations in batches:

<?php

namespace Drupal\my_module\Commands;

use Drush\Commands\DrushCommands;

/**
 * Custom drush commands.
 */
class MyModuleCommands extends DrushCommands {

  /**
   * Process evaluations in batches.
   *
   * @command my_module:process-evaluations
   * @aliases my:pe
   * @usage my_module:process-evaluations --limit=100
   */
  public function processEvaluations($options = ['limit' => 50]): void {
    $limit = $options['limit'];
    $processed = 0;

    while ($processed < $limit) {
      $item = $this->processQueueItem();
      if (!$item) {
        break;
      }
      $processed++;
    }

    $this->output()->writeln("Processed {$processed} evaluations.");
  }

  /**
   * Process a single queue item.
   */
  protected function processQueueItem(): bool {
    $queue = \Drupal::queue('ai_autoevals_evaluation_worker');
    $item = $queue->claimItem(30);

    if (!$item) {
      return FALSE;
    }

    // Process item
    try {
      // ... processing logic ...
      $queue->deleteItem($item);
      return TRUE;
    }
    catch (\Exception $e) {
      $queue->releaseItem($item);
      throw $e;
    }
  }

}

Caching

Fact Extraction Cache

Fact extraction results are cached by default. Ensure caching is enabled:

// Check cache status
$cache = \Drupal::cache('ai_autoevals_facts');

Clear cache only when necessary:

// Clear cache for specific evaluation set
\Drupal::cache('ai_autoevals_facts')->deleteTags(['ai_autoevals:facts:my_set_id']);

// Clear all fact cache
\Drupal::cache('ai_autoevals_facts')->deleteAll();

Custom Cache Tuning

Adjust cache settings in services.yml:

services:
  cache.ai_autoevals_facts:
    class: Drupal\Core\Cache\CacheBackendInterface
    arguments: ['ai_autoevals_facts']
    factory: cache_factory:get
    tags:
      - { name: cache.bin, default: 86400 }  # Cache for 24 hours

Database Optimization

Add Database Indexes

Add indexes for frequently queried fields:

<?php

/**
 * Implements hook_entity_base_field_info_alter().
 */
function my_module_entity_base_field_info_alter(&$fields, \Drupal\Core\Entity\EntityTypeInterface $entity_type): void {
  if ($entity_type->id() === 'ai_autoevals_evaluation_result') {
    if (isset($fields['status'])) {
      $fields['status']->addConstraint('EntityIndex');
    }
    if (isset($fields['score'])) {
      $fields['score']->addConstraint('EntityIndex');
    }
  }
}

Or add indexes via schema:

<?php

/**
 * Implements hook_schema_alter().
 */
function my_module_schema_alter(&$schema): void {
  if (isset($schema['ai_autoevals_evaluation_result'])) {
    $schema['ai_autoevals_evaluation_result']['fields']['status']['indexes']['status'] = ['status'];
    $schema['ai_autoevals_evaluation_result']['fields']['score']['indexes']['score'] = ['score'];
  }
}

Optimize Queries

Use entity query efficiently:

// Good - Use indexed fields
$ids = \Drupal::entityQuery('ai_autoevals_evaluation_result')
  ->accessCheck(FALSE)
  ->condition('status', 'completed')
  ->condition('score', 0.5, '>=')
  ->range(0, 100)
  ->execute();

// Bad - Query without conditions
$evaluations = \Drupal::entityTypeManager()
  ->getStorage('ai_autoevals_evaluation_result')
  ->loadMultiple();  // Loads ALL evaluations!

Clean Up Old Data

Implement data retention:

<?php

/**
 * Implements hook_cron().
 */
function my_module_cron(): void {
  $retention_days = \Drupal::config('ai_autoevals.settings')->get('retention_days') ?? 90;

  if ($retention_days > 0) {
    $threshold = strtotime("-{$retention_days} days");
    $storage = \Drupal::entityTypeManager()->getStorage('ai_autoevals_evaluation_result');

    $ids = $storage->getQuery()
      ->accessCheck(FALSE)
      ->condition('created', $threshold, '<')
      ->range(0, 100)  // Process in batches
      ->execute();

    if (!empty($ids)) {
      $entities = $storage->loadMultiple($ids);
      $storage->delete($entities);
    }
  }
}

Evaluation Configuration

Use Faster Models

Use faster models for evaluations:

// In evaluation set configuration
$evaluationSet->set('model_id', 'gpt-3.5-turbo');  // Faster than GPT-4
$evaluationSet->save();

Reduce Context Depth

Lower conversation context depth:

// In evaluation set configuration
$evaluationSet->set('context_depth', 1);  // Only include last turn
$evaluationSet->save();

Reduce Max Tokens

Reduce tokens in evaluation prompt:

<?php

/**
 * Implements hook_ai_autoevals_evaluation_prompt_alter().
 */
function my_module_ai_autoevals_evaluation_prompt_alter(array &$prompt, $evaluationSet): void {
  // Reduce max tokens for faster responses
  if ($evaluationSet->id() === 'my_fast_evaluation') {
    $prompt['max_tokens'] = 100;  // Reduce from default
  }
}

Batch Operations

Optimize Batch Size

Use smaller batch sizes for re-evaluations:

<?php

/**
 * Implements hook_ai_autoevals_batch_size_alter().
 */
function my_module_ai_autoevals_batch_size_alter(&$size, $operation): void {
  if ($operation === 'reevaluate') {
    $size = 20;  // Process 20 at a time instead of 50
  }
}

Use Queues for Large Operations

For very large operations, use custom queues:

<?php

namespace Drupal\my_module\Plugin\QueueWorker;

use Drupal\Core\Queue\QueueWorkerBase;

/**
 * Queue worker for batch re-evaluations.
 *
 * @QueueWorker(
 *   id = "my_module_batch_reevaluation",
 *   title = @Translation("Batch Re-evaluation Worker"),
 *   cron = {"time" = 120}
 * )
 */
class BatchReEvaluationWorker extends QueueWorkerBase {

  /**
   * {@inheritdoc}
   */
  public function processItem($data): void {
    $evaluation_ids = $data['evaluation_ids'];
    $new_evaluation_set_id = $data['new_evaluation_set_id'];

    // Process in smaller chunks
    $chunks = array_chunk($evaluation_ids, 10);
    foreach ($chunks as $chunk) {
      // Re-evaluate each chunk
    }
  }

}

Memory Management

Limit Memory Usage

Prevent memory issues during batch processing:

<?php

// Process evaluations in chunks to limit memory
$chunk_size = 100;
$total = $this->getTotalEvaluations();

for ($offset = 0; $offset < $total; $offset += $chunk_size) {
  $ids = $this->getEvaluationIds($offset, $chunk_size);

  foreach ($ids as $id) {
    $evaluation = EvaluationResult::load($id);
    // Process evaluation
    unset($evaluation);  // Free memory
  }

  // Clear static cache
  \Drupal::entityTypeManager()->getStorage('ai_autoevals_evaluation_result')->resetCache();
}

Use Static Caching Wisely

Use static cache for frequently accessed data:

<?php

class EvaluationCache {

  protected static $cache = [];

  public static function getEvaluationSet($id): ?EvaluationSet {
    if (!isset(static::$cache[$id])) {
      static::$cache[$id] = EvaluationSet::load($id);
    }
    return static::$cache[$id];
  }

  public static function clear(): void {
    static::$cache = [];
  }

}

Monitoring

Track Performance Metrics

Monitor queue processing time:

<?php

namespace Drupal\my_module\EventSubscriber;

use Drupal\ai_autoevals\Event\PostEvaluationEvent;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;

class PerformanceTrackingSubscriber implements EventSubscriberInterface {

  protected static $processingTimes = [];

  public static function getSubscribedEvents(): array {
    return [
      PostEvaluationEvent::EVENT_NAME => ['trackPerformance', 100],
    ];
  }

  public function trackPerformance(PostEvaluationEvent $event): void {
    $evaluation = $event->getEvaluationResult();
    $created = $evaluation->getCreatedTime();
    $now = time();
    $processingTime = $now - $created;

    static::$processingTimes[] = $processingTime;

    // Log slow evaluations
    if ($processingTime > 60) {
      \Drupal::logger('my_module')->warning(
        'Slow evaluation @id took @seconds seconds',
        ['@id' => $evaluation->id(), '@seconds' => $processingTime]
      );
    }
  }

  public static function getAverageProcessingTime(): float {
    if (empty(static::$processingTimes)) {
      return 0.0;
    }
    return array_sum(static::$processingTimes) / count(static::$processingTimes);
  }

}

Monitor Queue Size

Track queue size to detect backlogs:

<?php

/**
 * Implements hook_cron().
 */
function my_module_cron(): void {
  $queue = \Drupal::queue('ai_autoevals_evaluation_worker');
  $count = $queue->numberOfItems();

  \Drupal::logger('my_module')->info(
    'Evaluation queue size: @count',
    ['@count' => $count]
  );

  // Alert if queue is too large
  if ($count > 1000) {
    \Drupal::logger('my_module')->alert(
      'Evaluation queue is large: @count items pending',
      ['@count' => $count]
    );
  }
}

Best Practices

1. Process Queue Regularly

Ensure the evaluation queue is processed regularly to avoid backlog:

# Add to crontab
*/5 * * * * drush queue:run ai_autoevals_evaluation_worker

2. Use Indexes

Add indexes to frequently queried fields for faster lookups.

3. Limit Data Retention

Implement data retention to prevent database bloat.

4. Monitor Performance

Track queue size, processing time, and success rates.

5. Use Appropriate Models

Choose evaluation models based on your speed vs accuracy requirements.

6. Cache Strategically

Cache fact extraction results to reduce API calls.

7. Batch Large Operations

Process large operations in batches to avoid timeouts and memory issues.

Next Steps

• API Reference - Complete service documentation
• Examples - Real-world implementations
• Extending - Extension guide