Batch Processing

const API_KEY = process.env.SERPWATCH_API_KEY;
const BASE_URL = "https://engine.v2.serpwatch.io";

// Define keywords
const keywords = [
  "project management software",
  "task management tools",
  "team collaboration software"
];

// Build batch request with shared settings
const batch = keywords.map(keyword => ({
  keyword,
  location_name: "United States",
  iso_code: "US",
  depth: 10,
  device: "desktop",
  language_code: "en"
}));

// Submit batch
const response = await fetch(
  `${BASE_URL}/api/v2/serp/crawl/google/batch`,
  {
    method: "POST",
    headers: {
      "Authorization": `Bearer ${API_KEY}`,
      "Content-Type": "application/json"
    },
    body: JSON.stringify(batch)
  }
);

const tasks = await response.json();
console.log(`Created ${tasks.length} tasks`);

for (const task of tasks) {
  console.log(`  ${task.id}: ${task.keyword} - ${task.status}`);
}

function chunkArray(arr, chunkSize) {
  const chunks = [];
  for (let i = 0; i < arr.length; i += chunkSize) {
    chunks.push(arr.slice(i, i + chunkSize));
  }
  return chunks;
}

const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));

async function submitLargeBatch(keywords, batchSize = 100, delayMs = 1000) {
  /**
   * Submit a large list of keywords in smaller batches.
   */
  const allTasks = [];
  const chunks = chunkArray(keywords, batchSize);

  for (let i = 0; i < chunks.length; i++) {
    const chunk = chunks[i];
    console.log(`Submitting batch ${i + 1} (${chunk.length} keywords)...`);

    const batch = chunk.map(keyword => ({
      keyword,
      location_name: "United States",
      iso_code: "US",
      depth: 10,
      device: "desktop"
    }));

    const response = await fetch(
      `${BASE_URL}/api/v2/serp/crawl/google/batch`,
      {
        method: "POST",
        headers: {
          "Authorization": `Bearer ${API_KEY}`,
          "Content-Type": "application/json"
        },
        body: JSON.stringify(batch)
      }
    );

    if (response.ok) {
      const tasks = await response.json();
      allTasks.push(...tasks);
      console.log(`  Created ${tasks.length} tasks`);
    } else {
      console.error(`  Error: ${response.status}`);
    }

    // Delay between batches
    if (delayMs > 0 && i < chunks.length - 1) {
      await sleep(delayMs);
    }
  }

  return allTasks;
}

// Example: Submit 500 keywords
const largeKeywordList = Array.from({ length: 500 }, (_, i) => `keyword ${i}`);
const allTasks = await submitLargeBatch(largeKeywordList, 100);
console.log(`\nTotal tasks created: ${allTasks.length}`);

async function getTaskResult(taskId) {
  const response = await fetch(
    `${BASE_URL}/api/v2/serp/crawl/${taskId}`,
    { headers: { "Authorization": `Bearer ${API_KEY}` } }
  );
  return response.json();
}

async function collectResults(taskIds, concurrency = 10, timeoutMs = 300000) {
  /**
   * Collect results for multiple tasks with concurrency control.
   */
  const results = new Map();
  const pending = new Set(taskIds);
  const startTime = Date.now();

  while (pending.size > 0 && (Date.now() - startTime) < timeoutMs) {
    // Process in batches with concurrency limit
    const batch = Array.from(pending).slice(0, concurrency);

    const batchResults = await Promise.all(
      batch.map(async (taskId) => {
        try {
          const result = await getTaskResult(taskId);
          return { taskId, result };
        } catch (e) {
          console.error(`Error checking ${taskId}:`, e.message);
          return { taskId, result: null };
        }
      })
    );

    for (const { taskId, result } of batchResults) {
      if (result && ["success", "completed", "error"].includes(result.status)) {
        results.set(taskId, result);
        pending.delete(taskId);
      }
    }

    if (pending.size > 0) {
      console.log(`Progress: ${results.size}/${taskIds.length} (${pending.size} pending)`);
      await sleep(3000);
    }
  }

  return results;
}

// Usage
const taskIds = allTasks.map(task => task.id);
const results = await collectResults(taskIds);
console.log(`Collected ${results.size} results`);