# Useno robots.txt
#
# THIS FILE IS NOT A SECURITY CONTROL, and nothing below should be read as one.
# It is a request that well-behaved crawlers respect, and it is obeyed entirely
# at the crawler's discretion. Every path listed here that actually needs
# protecting is protected by something else: the API routes require a Supabase
# access token checked server-side, the admin surfaces sit behind their own
# auth gate, and the job feed is rate-limited and metered per account.
# Disallow lines exist so that compliant crawlers do not waste our capacity or
# index things that should not be in a search result — not to keep anyone out.
#
# See Section 4 of the Terms of Use (/terms) for the terms governing automated
# access, which is where the actual prohibition lives.

User-agent: *

# ── Protected and internal APIs ──────────────────────────────────────────────
# Authenticated job search, the job feed, resume processing, billing and the
# admin data endpoints. All of these require credentials; none of them has
# anything to offer an index.
Disallow: /api/

# ── Data files and exports ───────────────────────────────────────────────────
# The public projections under /data are JSON for pages to consume, not pages.
# The full corpus has not been served here since 2026-08-11 (it answers 410),
# and operator CSVs 404.
Disallow: /data/
Disallow: /public/data/

# ── Authenticated application routes ─────────────────────────────────────────
# A crawler cannot sign in, so these render as empty shells or login redirects
# and indexing them puts a blank page in a search result under a real title.
Disallow: /student-discover
Disallow: /for-you
Disallow: /dashboard
Disallow: /explore
Disallow: /student-ai-search
Disallow: /student-applications
Disallow: /student-contacts
Disallow: /student-contact-detail
Disallow: /student-hub
Disallow: /student-insights
Disallow: /student-community
Disallow: /student-community-post
Disallow: /student-resume
Disallow: /student-resume-editor
Disallow: /student-resume-match
Disallow: /student-resume-report
Disallow: /settings
Disallow: /onboarding
Disallow: /content-studio

# ── Account and auth flows ───────────────────────────────────────────────────
# These carry one-time tokens in their query strings. Keeping them out of an
# index keeps them out of the places indexes end up.
Disallow: /reset-password
Disallow: /email-verified
Disallow: /early-access

# ── Internal, admin and operator surfaces ────────────────────────────────────
Disallow: /ops
Disallow: /ops.html
Disallow: /ops-demo
Disallow: /ops-demo.html
Disallow: /admin-job-repository
Disallow: /admin-job-repository.html
Disallow: /growth-analytics
Disallow: /growth-analytics.html
Disallow: /outreach
Disallow: /outreach.html
Disallow: /recruiters
Disallow: /recruiters.html
Disallow: /university
Disallow: /university-login
Disallow: /university-queue
Disallow: /university-results
Disallow: /university-employer
Disallow: /university-settings
Disallow: /university-setup
Disallow: /university-recommendations
Disallow: /student-sidebar-preview

# ── Everything else is public and meant to be found ──────────────────────────
# The marketing pages, the resource guides, the internship index and the
# individual job pages are acquisition surfaces. Indexing them is the point.
Allow: /

Sitemap: https://www.useno.app/sitemap.xml