User-Agent: *
Crawl-delay:60
# These are API pages or user actions that won't have any interesting data on them.
# login/logout/home are banned as robots shouldn't really be able to log in.
Disallow: /ajax/
Disallow: /api.php
Disallow: /change-pw.php
Disallow: /contact/
Disallow: /contact-artist.php
Disallow: /CreateAccount.php
Disallow: /email-clip.php
Disallow: /forgot.php
Disallow: /home.php
Disallow: /login.php
Disallow: /logout.php
Disallow: /player-redirect.php
Disallow: /rate.php
Disallow: /search.php
Disallow: /set.php
Disallow: /tags.php
Disallow: /upload.php
Disallow: /verify.php
# the same files, but without .php at the end, since the URL space is visible that way too.
Disallow: /api
Disallow: /change-pw
Disallow: /contact-artist
Disallow: /CreateAccount
Disallow: /email-clip
Disallow: /forgot
Disallow: /home
Disallow: /login
Disallow: /logout
Disallow: /player-redirect
Disallow: /rate
# FIXME (maybe): this kills search-results too
Disallow: /search
Disallow: /set
Disallow: /tags
Disallow: /upload
Disallow: /verify