rustyseo-toolkit

git clone https://github.com/mascanho/RustySEO-Headless.git
cd RustySEO-Headless
cargo build --release

// Example: Trigger shallow crawl programmatically
// Located in: src/components/Crawler.tsx
 
interface CrawlConfig {
  url: string;
  followLinks: boolean;
  respectRobotsTxt: boolean;
  userAgent: string;
}
 
const shallowCrawl = async (config: CrawlConfig) => {
  const result = await invoke('shallow_crawl', {
    url: config.url,
    followLinks: false,
    respectRobotsTxt: true
  });
  return result;
};

// Example: Deep crawl with concurrency control
const deepCrawl = async (baseUrl: string, maxPages: number = 100) => {
  const result = await invoke('deep_crawl', {
    url: baseUrl,
    maxPages: maxPages,
    concurrency: 5, // Concurrent requests
    delay: 1000 // Milliseconds between requests
  });
  return result;
};

// Example: Configure PageSpeed Insights API
// Stored in local SQLite database
 
interface APIConfig {
  provider: 'pagespeed' | 'gemini' | 'gsc' | 'ga4';
  apiKey: string;
  enabled: boolean;
}
 
const setAPIKey = async (config: APIConfig) => {
  await invoke('save_api_config', {
    provider: config.provider,
    apiKey: config.apiKey,
    enabled: true
  });
};
 
// Usage
await setAPIKey({
  provider: 'pagespeed',
  apiKey: process.env.GOOGLE_PAGESPEED_API_KEY || '',
  enabled: true
});

# .env.local
GOOGLE_PAGESPEED_API_KEY=your_pagespeed_key_here
GOOGLE_GEMINI_API_KEY=your_gemini_key_here
GOOGLE_GSC_CLIENT_ID=your_oauth_client_id
GOOGLE_GSC_CLIENT_SECRET=your_oauth_client_secret
GOOGLE_GA4_PROPERTY_ID=your_ga4_property_id

// Example: Parse and analyze server logs
interface LogAnalysisConfig {
  logPath: string;
  logType: 'nginx' | 'apache';
  startDate?: string;
  endDate?: string;
}
 
const analyzeLogs = async (config: LogAnalysisConfig) => {
  const analysis = await invoke('parse_logs', {
    path: config.logPath,
    logType: config.logType,
    filters: {
      startDate: config.startDate,
      endDate: config.endDate
    }
  });
  
  // Returns: bot traffic, crawl frequency, errors, popular pages
  return analysis;
};
 
// Usage
const logData = await analyzeLogs({
  logPath: '/var/log/nginx/access.log',
  logType: 'nginx',
  startDate: '2024-01-01',
  endDate: '2024-01-31'
});
 
console.log(logData.botTraffic); // Googlebot, Bingbot stats
console.log(logData.errorPages); // 404, 500 errors

// Example: Extract SEO metrics from crawl
interface SEOMetrics {
  title: string;
  metaDescription: string;
  h1Tags: string[];
  h2Tags: string[];
  canonicalUrl: string;
  openGraphTags: Record<string, string>;
  structuredData: object[];
  imageCount: number;
  internalLinks: number;
  externalLinks: number;
  wordCount: number;
  loadTime: number;
}
 
const analyzePage = async (url: string): Promise<SEOMetrics> => {
  const crawlResult = await invoke('shallow_crawl', { url });
  
  return {
    title: crawlResult.title,
    metaDescription: crawlResult.meta_description,
    h1Tags: crawlResult.h1_tags,
    h2Tags: crawlResult.h2_tags,
    canonicalUrl: crawlResult.canonical,
    openGraphTags: crawlResult.og_tags,
    structuredData: crawlResult.structured_data,
    imageCount: crawlResult.images.length,
    internalLinks: crawlResult.internal_links.length,
    externalLinks: crawlResult.external_links.length,
    wordCount: crawlResult.word_count,
    loadTime: crawlResult.load_time_ms
  };
};

// Example: Export crawl data to various formats
const exportData = async (format: 'csv' | 'excel' | 'pdf' | 'sheets') => {
  await invoke('export_crawl_data', {
    format: format,
    crawlId: currentCrawlId,
    outputPath: `./exports/crawl-${Date.now()}.${format}`
  });
};
 
// CSV Export
await exportData('csv');
 
// Google Sheets (requires GSC OAuth)
await exportData('sheets');

// Example: Generate content topics using AI
interface TopicRequest {
  seed: string;
  count: number;
  model: 'gemini' | 'ollama';
}
 
const generateTopics = async (request: TopicRequest) => {
  const topics = await invoke('ai_generate_topics', {
    seed: request.seed,
    count: request.count,
    provider: request.model
  });
  return topics;
};
 
// Usage with Google Gemini
const topics = await generateTopics({
  seed: 'organic gardening tips',
  count: 10,
  model: 'gemini'
});
 
// Returns: ["Companion Planting Guide", "Composting 101", ...]

// Example: Cluster keywords using ML
const clusterKeywords = async (keywords: string[]) => {
  const clusters = await invoke('cluster_keywords', {
    keywords: keywords,
    minClusterSize: 5,
    algorithm: 'kmeans'
  });
  
  return clusters;
};
 
const keywords = ["seo tools", "seo software", "website crawler", "site audit"];
const result = await clusterKeywords(keywords);
// Returns: { cluster1: ["seo tools", "seo software"], cluster2: ["website crawler", "site audit"] }

// Example: Query AI about crawled pages
const askChatbot = async (question: string, crawlContext: string) => {
  const response = await invoke('ai_chat', {
    question: question,
    context: crawlContext,
    provider: 'gemini'
  });
  return response;
};
 
// Usage
const answer = await askChatbot(
  "What are the main SEO issues on this page?",
  JSON.stringify(currentPageData)
);

// Example: Fetch GSC data
interface GSCQuery {
  siteUrl: string;
  startDate: string;
  endDate: string;
  dimensions?: ('query' | 'page' | 'country' | 'device')[];
}
 
const fetchGSCData = async (query: GSCQuery) => {
  // Requires OAuth2 authentication
  const data = await invoke('gsc_fetch_data', {
    siteUrl: query.siteUrl,
    startDate: query.startDate,
    endDate: query.endDate,
    dimensions: query.dimensions || ['query', 'page']
  });
  
  return data.rows; // { query, clicks, impressions, ctr, position }
};
 
// Usage
const gscData = await fetchGSCData({
  siteUrl: 'https://example.com',
  startDate: '2024-01-01',
  endDate: '2024-01-31',
  dimensions: ['query', 'page']
});

// Example: Generate and validate structured data
interface SchemaConfig {
  type: 'Article' | 'Product' | 'LocalBusiness' | 'FAQ';
  data: Record<string, any>;
}
 
const generateSchema = (config: SchemaConfig) => {
  const schema = {
    "@context": "https://schema.org",
    "@type": config.type,
    ...config.data
  };
  
  return JSON.stringify(schema, null, 2);
};
 
// Usage
const articleSchema = generateSchema({
  type: 'Article',
  data: {
    headline: 'Complete SEO Guide',
    author: { "@type": "Person", name: "John Doe" },
    datePublished: '2024-01-01',
    image: 'https://example.com/image.jpg'
  }
});
 
// Validate schema
const validateSchema = async (schemaJson: string) => {
  const validation = await invoke('validate_schema', {
    schema: schemaJson
  });
  return validation.isValid;
};

// Example: Query crawl history
const getCrawlHistory = async (limit: number = 50) => {
  const history = await invoke('get_crawl_history', { limit });
  return history;
};
 
// Clear crawl logs
const clearLogs = async () => {
  await invoke('clear_crawl_logs');
};

// Example: Create and track SEO tasks
interface SEOTask {
  title: string;
  description: string;
  priority: 'low' | 'medium' | 'high';
  dueDate?: string;
}
 
const createTask = async (task: SEOTask) => {
  await invoke('create_task', {
    title: task.title,
    description: task.description,
    priority: task.priority,
    dueDate: task.dueDate
  });
};
 
// Keyboard shortcut: CTRL + T

// Example: Convert and optimize images
interface ImageOptimizeConfig {
  inputPath: string;
  outputPath: string;
  format: 'webp' | 'avif' | 'jpeg';
  quality: number; // 1-100
}
 
const optimizeImage = async (config: ImageOptimizeConfig) => {
  await invoke('optimize_image', {
    input: config.inputPath,
    output: config.outputPath,
    format: config.format,
    quality: config.quality
  });
};

const setUserAgent = async (userAgent: string) => {
  await invoke('set_user_agent', { userAgent });
};
 
// Respect robots.txt
const setRobotsTxt = async (respect: boolean) => {
  await invoke('set_robots_txt_respect', { respect });
};

# Keyboard shortcuts
CTRL + /           # Clear cache
CTRL + Shift + /   # Full app reset

// Add delay between requests
const crawlWithDelay = async (urls: string[], delayMs: number = 2000) => {
  for (const url of urls) {
    await invoke('shallow_crawl', { url });
    await new Promise(resolve => setTimeout(resolve, delayMs));
  }
};

# Install RustySEO-Headless
cd RustySEO-Headless
cargo run --release -- --url https://example.com --max-pages 100000

Authorized redirect URIs:
http://localhost:3000/api/auth/callback/google

// Switch to Gemini
const config = {
  provider: 'gemini',
  apiKey: process.env.GOOGLE_GEMINI_API_KEY
};

# Basic crawl
rustyseo-headless --url https://example.com
 
# Deep crawl with limits
rustyseo-headless --url https://example.com --max-pages 500 --concurrency 10
 
# Parse logs
rustyseo-headless --parse-logs /var/log/nginx/access.log --log-type nginx
 
# Export to CSV
rustyseo-headless --url https://example.com --export-csv output.csv

-- Crawl results table
CREATE TABLE crawls (
  id INTEGER PRIMARY KEY,
  url TEXT NOT NULL,
  title TEXT,
  meta_description TEXT,
  status_code INTEGER,
  load_time_ms INTEGER,
  word_count INTEGER,
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
 
-- SEO tasks table
CREATE TABLE tasks (
  id INTEGER PRIMARY KEY,
  title TEXT NOT NULL,
  description TEXT,
  priority TEXT,
  status TEXT DEFAULT 'pending',
  due_date TEXT,
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);