# ---------------------------------------------------------- # Applicable Parties # ---------------------------------------------------------- # Tell ALL robots to heed these rules # ---------------------------------------------------------- User-agent: * Allow: * # ---------------------------------------------------------- # Troublesome Content # ---------------------------------------------------------- # Disallow crawling of content that is recursive, or # otherwise too much for the robots # ---------------------------------------------------------- # Disallow: /some/path/pattern/* # ---------------------------------------------------------- # Misbehaving Robots # ---------------------------------------------------------- # Call out the robot(s) by name with User-agent # Disallow as much as neccessary # ---------------------------------------------------------- # User-agent: someBadBot # Disallow: * # ---------------------------------------------------------- # Sitemap(s) # ---------------------------------------------------------- # Multiple Sitemap URLs are permitted # one per line, using the format # Sitemap: https://www.wm.edu/[folder]/sitemap.xml # ---------------------------------------------------------- Sitemap: https://www.wm.edu/sitemap.xml