{
  "id": "acbb28f2-a22f-453b-8025-5dc4c5da3a9f",
  "name": "Robots.txt Review",
  "nodes": [
    {
      "parameters": {
        "content": "# Reviewed Harshith.com template\n\nReads one public robots.txt file and preserves ordered user-agent groups with their allow and disallow directives for human review. It does not decide whether a site should be indexed.\n\nThis export is inactive and contains no credentials, deployment IDs, webhook IDs, or pinned data. Review the visible URL before each run.",
        "height": 300,
        "width": 430,
        "color": 7
      },
      "id": "708056fe-c8d1-4b3e-8f20-324f432c8121",
      "name": "Read Me First",
      "type": "n8n-nodes-base.stickyNote",
      "typeVersion": 1,
      "position": [
        -760,
        -300
      ]
    },
    {
      "parameters": {},
      "id": "baa01ab5-4105-442d-bbba-7c73a4fa42c3",
      "name": "Run Manually",
      "type": "n8n-nodes-base.manualTrigger",
      "typeVersion": 1,
      "position": [
        -620,
        80
      ]
    },
    {
      "parameters": {
        "jsCode": "return [{ json: { url: 'https://www.harshith.com/robots.txt', maxLines: 500 } }];"
      },
      "id": "dfaa8b70-be5f-4aba-8e6f-75e9ca040614",
      "name": "Set Robots URL",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        -380,
        80
      ]
    },
    {
      "parameters": {
        "url": "={{ $json.url }}",
        "options": {
          "timeout": 10000,
          "redirect": {
            "redirect": {
              "followRedirects": false
            }
          },
          "response": {
            "response": {
              "fullResponse": true,
              "neverError": true,
              "responseFormat": "text"
            }
          }
        }
      },
      "retryOnFail": true,
      "maxTries": 3,
      "waitBetweenTries": 1000,
      "id": "52472783-9f34-45cf-a072-d0b1bb23707f",
      "name": "Read Robots.txt",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        -100,
        80
      ]
    },
    {
      "parameters": {
        "mode": "runOnceForEachItem",
        "jsCode": "const payload = $json;\nconst source = $('Set Robots URL').first().json;\nconst text = String(payload.body || payload.data || '');\nconst allLines = text.split(/\\r?\\n/).map((line) => line.replace(/#.*$/, '').trim()).filter(Boolean);\nconst lines = allLines.slice(0, source.maxLines);\nconst groups = [];\nconst sitemaps = [];\nlet invalidUserAgentCount = 0;\nlet current = null;\nconst pushCurrent = () => { if (current && current.userAgents.length) groups.push(current); current = null; };\nfor (const line of lines) {\n  const separator = line.indexOf(':');\n  if (separator < 0) continue;\n  const directive = line.slice(0, separator).trim().toLowerCase();\n  const value = line.slice(separator + 1).trim();\n  if (directive === 'sitemap') { if (value) sitemaps.push(value); continue; }\n  if (directive === 'user-agent') {\n    if (!value) { pushCurrent(); invalidUserAgentCount += 1; continue; }\n    if (current && (current.allows.length || current.disallows.length)) pushCurrent();\n    current ||= { userAgents: [], allows: [], disallows: [] };\n    current.userAgents.push(value);\n    continue;\n  }\n  if (!current) continue;\n  if (directive === 'allow') current.allows.push(value);\n  if (directive === 'disallow') current.disallows.push(value);\n}\npushCurrent();\nconst truncated = allLines.length > source.maxLines;\nreturn { json: { robotsUrl: source.url, statusCode: Number(payload.statusCode || 0), groupCount: groups.length, groups, sitemaps, lineCount: lines.length, invalidUserAgentCount, truncated, needsReview: Number(payload.statusCode || 0) !== 200 || groups.length === 0 || invalidUserAgentCount > 0 || truncated, checkedAt: new Date().toISOString() } };"
      },
      "id": "a8bce270-77a6-4fcf-91cd-b39386d30a12",
      "name": "Build Robots Report",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        180,
        80
      ]
    }
  ],
  "connections": {
    "Run Manually": {
      "main": [
        [
          {
            "node": "Set Robots URL",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Set Robots URL": {
      "main": [
        [
          {
            "node": "Read Robots.txt",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Read Robots.txt": {
      "main": [
        [
          {
            "node": "Build Robots Report",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  },
  "active": false,
  "settings": {
    "executionOrder": "v1",
    "saveDataErrorExecution": "none",
    "saveDataSuccessExecution": "none",
    "saveManualExecutions": false,
    "executionTimeout": 60
  },
  "tags": []
}