User-agent: *
Allow: /

# -- Main site -- non-public paths -----------------------------------------
Disallow: /_content-writing-services/
Disallow: /bk-services/
Disallow: /content-writing-services-usa/
Disallow: /404
Disallow: /old-website/
Disallow: /samples/
Disallow: /samplescontent/
Disallow: /contact-customer-new.php
Disallow: /dev/

# -- /blog -- pagination (page numbers) ------------------------------------
Disallow: /blog/page/
Disallow: /blog/category/*/page/
Disallow: /blog/tag/*/page/
Disallow: /blog/author/*/page/

# -- /us/blog -- pagination (page numbers) ---------------------------------
Disallow: /us/blog/page/
Disallow: /us/blog/category/*/page/
Disallow: /us/blog/author/*/page/

# -- /blog -- WordPress system paths ---------------------------------------
Disallow: /blog/wp-admin/
Disallow: /blog/wp-*.php
Disallow: /blog/wp-includes/
Disallow: /blog/wp-content/
Disallow: /blog/wp-json/
Disallow: /blog/xmlrpc.php

# -- /us/blog -- WordPress system paths ------------------------------------
Disallow: /us/blog/wp-admin/
Disallow: /us/blog/wp-*.php
Disallow: /us/blog/wp-includes/
Disallow: /us/blog/wp-content/
Disallow: /us/blog/wp-json/
Disallow: /us/blog/xmlrpc.php

# -- Media / video embed files ---------------------------------------------
# /static/uploads/v/*.mp4 | /public/video/*.mp4 | /media/mp4/*.mp4 | /upload/media/video/*.mp4
Disallow: /static/
Disallow: /public/video/
Disallow: /media/mp4/
Disallow: /upload/

# -- UTM tracking parameters (duplicate content) ---------------------------
Disallow: /*?utm_source=

# -- Legacy .html / .php files (old URLs now returning 404) ---------------
Disallow: /sitemap.php
Disallow: /about.html
Disallow: /services.html
Disallow: /graphic-design-agency-nyc.php

# -- Broken / ghost paths (wrong nested paths crawled from bad links) ------
Disallow: /services/services/
Disallow: /services/content-whale-com-services/
Disallow: /services/service-policies
Disallow: /website-content-writing/
Disallow: /in/

# -- Duplicate /us/ regional service pages (same content as main site) -----
Disallow: /us/about-us/
Disallow: /us/ebook-writing-services
Disallow: /us/case-study-writing-services
Disallow: /us/magazine-article-writer
Disallow: /us/thought-leadership-marketing-agency
Disallow: /us/white-paper-writing-services

# -- Non-content URLs ------------------------------------------------------
Disallow: /blog/feed/
Disallow: /blog/?s=

Sitemap: https://content-whale.com/sitemap-index.xml