# Group 1
User-agent: Googlebot
Disallow: /nogooglebot/

# Group 2
User-agent: *
Allow: /

# Recommended baseline for AI agent access

# OpenAI: allow search and user agents, manage training separately
User-agent: OAI-SearchBot
Allow: /

User-agent: openai_chat_gpt_user
Allow: /

User-agent: GPTBot
Allow: / # Change to Disallow: / if you want to opt out of training

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Apple
User-agent: Applebot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Google AI (separate from Googlebot)
User-agent: Google-Extended
Allow: / # Change to Disallow: / to opt out of Gemini/AI Overviews training

Sitemap: https://admin.smc.edu/sitemap.xml

Disallow: /_qa
Disallow: /_resources/
Disallow: /_showcase/
Disallow: /_sandbox/
Disallow: /_training/
Disallow: /_uat/
Disallow: /smc-community/
Disallow: /search-1/
Disallow: /*.inc$
Disallow: */_props.pcf$

Disallow: /*?d=*&m=*
Disallow: /administration/information-technology/academic-computing/student-it-help/answer/
Disallow: /news/in-focus/2018/vol-iv-issue-5/05-02-welcome-back.php
Disallow: /news/in-focus/2018/vol-iv-issue-1/01-04-greg.php

Disallow: /academics/classes/2018-19/
Disallow: /academics/classes/2019-20/
Disallow: /academics/classes/2020-21/
Disallow: /academics/classes/2021-22/
Disallow: /academics/classes/2022-23/
Disallow: /academics/classes/2023-24/
Disallow: /academics/classes/2024-25/

Disallow: /administration/marketing/a1-archive/
Disallow: /administration/information-technology/academic-computing/peterson_lee/